From 682da242ff2b3b59b706e66ea21eacc45a921a02 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:11:43 +0100 Subject: [PATCH 1/7] refactor(planner): read compiled models' lineage and analysis natively in planner readers Fixture metadata, fixture reads, expected fixture types, schema findings and migration column readers use the typed analysis accessors instead of building Python projections: inferred rows, star flags, lineage facts with transform and confidence values, and proven dynamic-pivot inputs. Lane B's accessors are merged in; lineage facts and proven inputs are added. --- .../bindings/_helpers/analysis/model_reads.rs | 75 +++++++++++++++++++ .../compiled_project/compiled_model_view.rs | 19 +++++ .../_helpers/planner_changes/detect.rs | 29 +++---- .../planner_migrations/column_entries.rs | 8 +- .../planner_migrations/compatibility.rs | 3 +- .../_helpers/planner_strategy/constants.rs | 4 + .../planner_strategy/fixture_metadata.rs | 24 ++++-- .../planner_strategy/fixture_reads.rs | 47 ++++-------- .../planner_strategy/fixture_types.rs | 31 +++----- 9 files changed, 157 insertions(+), 83 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index a15ec21969..b705e2d294 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -6,6 +6,19 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ DeclaredRow, InferredRow, LineageInput, NativeCompiledModel, }; +/// `(output, transform kind, confidence, [(resource type, resource, column)])` of one lineage output. +pub(crate) type LineageFactRow = (String, String, String, Vec<(String, String, String)>); + +const TRANSFORM_VALUES: [&str; 6] = [ + "direct", + "cast", + "expression", + "aggregation", + "star", + "constant", +]; +const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; + /// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { if let Ok(view) = model.downcast::() { @@ -117,3 +130,65 @@ pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { let proof: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; Ok(!proof.is_none() && proof.getattr("output_proven")?.is_truthy()?) } + +/// Each lineage output with its transform kind and confidence values, `None` without lineage. +pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Ok(view) = model.downcast::() { + let Some(rows) = view.get().lineage_rows(model.py())? else { + return Ok(None); + }; + let mut facts: Vec = Vec::with_capacity(rows.len()); + for row in rows { + facts.push(( + row.output_column, + coded(&TRANSFORM_VALUES, row.transform_code)?, + coded(&CONFIDENCE_VALUES, row.confidence_code)?, + row.sources, + )); + } + return Ok(Some(facts)); + } + let columns: Bound<'_, PyAny> = model.getattr("fast_lineage_columns")?; + if columns.is_none() { + return Ok(None); + } + let mut facts: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut sources: Vec<(String, String, String)> = Vec::new(); + for source in output.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + sources.push(( + source.getattr("resource_type")?.extract()?, + source.getattr("resource_name")?.extract()?, + source.getattr("column_name")?.extract()?, + )); + } + facts.push(( + output.getattr("output_column")?.extract()?, + output.getattr("transform_kind")?.str()?.extract()?, + output.getattr("confidence")?.str()?.extract()?, + sources, + )); + } + Ok(Some(facts)) +} + +fn coded(values: &[&str], code: u8) -> PyResult { + values + .get(usize::from(code)) + .map(|value| (*value).to_owned()) + .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("tuple index out of range")) +} + +/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. +pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Ok(view) = model.downcast::() { + return view.get().proven_inputs(model.py()); + } + let proof: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; + if proof.is_none() || !proof.getattr("output_proven")?.is_truthy()? { + return Ok(None); + } + proof.getattr("input_relations")?.extract().map(Some) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index ac38fc0f72..4d8e5f3080 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -309,6 +309,25 @@ impl NativeCompiledModel { &self.overlay.poisoned } + /// The kept lineage rows with their transform and confidence codes, or `None`. + pub(crate) fn lineage_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled.analysis.lineage_rows().map(<[LineageRow]>::to_vec)) + }) + } + + /// The input relations of a dynamic pivot proof that proved its outputs, else `None`. + pub(crate) fn proven_inputs(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .proof + .as_ref() + .filter(|proof| proof.output_proven) + .map(|proof| proof.input_relations.clone())) + }) + } + fn model_file_attribute(&self, py: Python<'_>, name: &str) -> PyResult> { self.read(py, |record, py| { Ok(record.model_file.bind(py).getattr(name)?.unbind()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 274e121ff8..ac0c65cc87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -22,9 +22,11 @@ use sqlbuild_analysis::planner_changes::models::{ SchemaFindingRow, }; +use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -542,26 +544,11 @@ fn model_schema_findings<'py>( dynamic = Some(families); } } - let inferred: Option)>> = { - let columns: Bound<'py, PyAny> = model.getattr("inferred_columns")?; - if columns.is_none() { - None - } else { - let mut items: Vec<(String, Option)> = Vec::new(); - for column in columns.try_iter()? { - let column: Bound<'py, PyAny> = column?; - items.push(( - column.getattr("name")?.extract()?, - column.getattr("type")?.extract()?, - )); - } - Some(items) - } - }; + let inferred: Option)>> = inferred_rows(model)?.map(typed_names); if yml.is_empty() && inferred.as_ref().is_none_or(Vec::is_empty) { return Ok(Vec::new()); } - let inferred_schema_complete: bool = !model.getattr("fast_lineage_has_star")?.is_truthy()?; + let inferred_schema_complete: bool = !star_flags(model)?.0; schema_findings_of( SchemaColumns { yml, @@ -573,6 +560,14 @@ fn model_schema_findings<'py>( ) } +fn typed_names(rows: Vec) -> Vec<(String, Option)> { + let mut columns: Vec<(String, Option)> = Vec::with_capacity(rows.len()); + for (name, column_type, _) in rows { + columns.push((name, column_type)); + } + columns +} + fn column_pairs(columns: &Bound<'_, PyAny>) -> PyResult> { let mut pairs: Vec<(String, String)> = Vec::new(); for column in columns.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 9f5685f3ca..04ddc289a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -1,5 +1,6 @@ //! Materialize column decisions from native proofs and live evidence. +use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -52,11 +53,8 @@ pub(super) fn decide<'py>( outputs.push(projection?.getattr("key")?.extract::()?); } } else { - let inferred = candidate.model.getattr("inferred_columns")?; - if !inferred.is_none() { - for column in inferred.try_iter()? { - outputs.push(lower(&column?.getattr("name")?)?); - } + for (name, _, _) in inferred_rows(&candidate.model)?.unwrap_or_default() { + outputs.push(name.to_lowercase()); } } let relation = host::entry(py, ExecutorHost::MigrationRelationForLocation)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 86b6de5fd3..37862b2218 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -1,5 +1,6 @@ //! Convert existing planner contracts for origin compatibility checks. +use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -166,7 +167,7 @@ fn check_migration_compatibility<'py>( ], )?; let schema = model.getattr("schema_entry")?; - let mut has_columns = model.getattr("inferred_columns")?.is_truthy()?; + let mut has_columns = inferred_rows(&model)?.is_some_and(|rows| !rows.is_empty()); if !schema.is_none() { for column in schema.getattr("columns")?.try_iter()? { has_columns |= !column?.getattr("type")?.is_none(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index 4646fba279..aa811c5350 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -13,3 +13,7 @@ pub(crate) const MODEL_REFERENCE: &str = "ref"; pub(crate) const SEED_REFERENCE: &str = "seed"; pub(crate) const SOURCE_REFERENCE: &str = "source"; pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; +/// The lineage confidence whose reads fixtures must provide. +pub(crate) const HIGH_CONFIDENCE: &str = "high"; +/// The lineage transform kind that passes an input column's type through. +pub(crate) const DIRECT_TRANSFORM: &str = "direct"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index 8839867676..c9aeb5bcf1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -4,7 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyTuple; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata}; #[pyfunction] @@ -13,26 +17,30 @@ pub(crate) fn model_fixture_metadata<'py>( model: &Bound<'py, PyAny>, ) -> PyResult> { let py = model.py(); - let inferred = model.getattr("inferred_columns")?; + let inferred: Option> = inferred_rows(model)?; let schema = model.getattr("schema_entry")?; let declared = if schema.is_none() { PyTuple::empty(py).into_any() } else { schema.getattr("columns")? }; - let inferred_columns = if inferred.is_none() { - Vec::new() - } else { - columns(&inferred, true)? - }; + let mut inferred_columns: Vec = Vec::new(); + for (name, sql_type, _) in inferred.iter().flatten() { + inferred_columns.push(FixtureColumn { + name: name.clone(), + folded_name: python_casefold(name), + sql_type: sql_type.clone(), + nullable: None, + }); + } let merged = merge_metadata(inferred_columns, columns(&declared, false)?); - let authoritative = !model.getattr("fast_lineage_has_star")?.is_truthy()? + let authoritative = !star_flags(model)?.0 && (model .getattr("config")? .getattr("values")? .call_method1("get", ("contract",))? .eq("enforced")? - || !inferred.is_none()); + || inferred.is_some()); metadata(py, &merged, authoritative) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index ce01565e3c..704d9f03fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -13,10 +13,12 @@ use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; +use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; use crate::bindings::_helpers::planner_strategy::fixture_values::columns; use crate::bindings::_helpers::planner_strategy::models::FixtureReadCacheHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use crate::bindings::types::CompilerDetach; @@ -118,26 +120,15 @@ pub(crate) fn required_fixture_columns<'py>( fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let mut reads: Vec = Vec::new(); - let facts = model.getattr("fast_lineage_columns")?; - if facts.is_none() { - return Ok(reads); - } - for fact in facts.try_iter()? { - let fact = fact?; - if !fact.getattr("confidence")?.eq(registry::class( - model.py(), - RecordId::ColumnLineageConfidence, - )? - .getattr("HIGH")?)? - { + for (_, _, confidence, sources) in lineage_facts(model)?.unwrap_or_default() { + if confidence != HIGH_CONFIDENCE { continue; } - for source in fact.getattr("upstream_columns")?.try_iter()? { - let source = source?; + for (resource_type, resource_name, column_name) in sources { reads.push(ColumnRead { - resource_type: source.getattr("resource_type")?.extract()?, - resource_name: source.getattr("resource_name")?.extract()?, - column_name: source.getattr("column_name")?.extract()?, + resource_type, + resource_name, + column_name, }); } } @@ -149,16 +140,14 @@ fn model_stars( schema: &[FixtureStarRelation], adapter: &Bound<'_, PyAny>, ) -> PyResult> { - if !model.getattr("fast_lineage_has_star")?.is_truthy()? { + if !star_flags(model)?.0 { return Ok(Vec::new()); } let mut deps: Vec<(String, String)> = Vec::new(); - for dependency in model.getattr("deps")?.try_iter()? { - let dependency = dependency?; - let key = ( - dependency.getattr("resource_type")?.extract::()?, - dependency.getattr("name")?.extract::()?, - ); + for key in model + .getattr("dep_rows")? + .extract::>()? + { if schema.iter().any(|relation| relation.key == key) { deps.push(key); } @@ -175,13 +164,7 @@ fn model_stars( }); } } - let contract = model.getattr("dynamic_column_contract")?; - let inputs: Option> = - if !contract.is_none() && contract.getattr("output_proven")?.is_truthy()? { - Some(contract.getattr("input_relations")?.extract()?) - } else { - None - }; + let inputs: Option> = proven_inputs(model)?; let dialect: Option = if inputs.is_none() && selected.iter().any(|relation| !relation.columns.is_empty()) { adapter diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index beb34dfd93..3715f6bfb7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -9,9 +9,10 @@ use sqlbuild_analysis::planner_fixtures::models::{ FixtureTypeModel, FixtureTypeProjection, FixtureTypeRead, FixtureTypeState, }; -use crate::bindings::_helpers::planner_strategy::fixture_values::folded; -use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::records::registry; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::bindings::_helpers::analysis::model_reads::lineage_facts; +use crate::bindings::_helpers::planner_strategy::constants::DIRECT_TRANSFORM; #[pyfunction] #[pyo3(signature = (*, base_expected_types, ordered_model_names, model_map))] @@ -77,30 +78,20 @@ pub(crate) fn expected_fixture_types<'py>( } fn projections(model: &Bound<'_, PyAny>) -> PyResult> { - let lineage = model.getattr("fast_lineage_columns")?; let mut projections: Vec = Vec::new(); - if lineage.is_none() { - return Ok(projections); - } - let direct = registry::class(model.py(), RecordId::ColumnTransformKind)?.getattr("DIRECT")?; - for column in lineage.try_iter()? { - let column = column?; - if !column.getattr("transform_kind")?.eq(&direct)? { + for (output_column, transform, _, sources) in lineage_facts(model)?.unwrap_or_default() { + if transform != DIRECT_TRANSFORM { continue; } - let mut upstream: Vec = Vec::new(); - for source in column.getattr("upstream_columns")?.try_iter()? { - let source = source?; + let mut upstream: Vec = Vec::with_capacity(sources.len()); + for (resource_type, resource_name, column_name) in sources { upstream.push(FixtureTypeRead { - key: ( - source.getattr("resource_type")?.extract()?, - source.getattr("resource_name")?.extract()?, - ), - column: folded(&source.getattr("column_name")?)?, + key: (resource_type, resource_name), + column: python_casefold(&column_name), }); } projections.push(FixtureTypeProjection { - output_column: folded(&column.getattr("output_column")?)?, + output_column: python_casefold(&output_column), upstream, }); } From 637944e8892d30628515f16a24b4053f6afdbcc5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:36:25 +0100 Subject: [PATCH 2/7] refactor(planner): read model names, keys, destinations and SQL through typed accessors Native compiled models keep their name in the compiled record instead of deriving it from the model file's path on every read. Planner, migration, scenario, SQL-test and janitor readers take a model's name, key, destination and query SQL through the typed model reads, which answer native views directly and fall back to attributes for hand-built records. --- .../bindings/_helpers/analysis/model_reads.rs | 45 +++++++++++++++++++ .../bindings/_helpers/compile_driver/diff.rs | 5 ++- .../compiled_project/compiled_model_view.rs | 12 +++-- .../_helpers/janitor/janitor_old_names.rs | 5 ++- .../_helpers/planner_changes/detect.rs | 11 ++--- .../_helpers/planner_changes/renames.rs | 7 +-- .../_helpers/planner_entries/clone_plans.rs | 13 +++--- .../_helpers/planner_entries/command_plans.rs | 3 +- .../_helpers/planner_entries/display_plan.rs | 11 ++--- .../_helpers/planner_entries/model_entries.rs | 8 +++- .../_helpers/planner_identity/identities.rs | 9 ++-- .../_helpers/planner_identity/metadata.rs | 3 +- .../planner_migrations/column_candidates.rs | 5 ++- .../planner_migrations/column_entries.rs | 11 ++--- .../planner_migrations/column_objects.rs | 5 ++- .../planner_migrations/column_overlay.rs | 5 ++- .../planner_migrations/column_planning.rs | 5 ++- .../planner_migrations/compatibility.rs | 8 +++- .../planner_migrations/fingerprint.rs | 3 +- .../_helpers/planner_migrations/matching.rs | 9 ++-- .../planner_migrations/model_handover.rs | 13 +++--- .../_helpers/planner_migrations/old_names.rs | 9 ++-- .../_helpers/planner_phases/full_refresh.rs | 5 ++- .../planner_phases/output_assembly.rs | 3 +- .../_helpers/planner_phases/retention.rs | 27 +++++------ .../_helpers/planner_phases/scopes.rs | 3 +- .../planner_resolution/cursor_model.rs | 13 +++--- .../planner_resolution/cursor_overrides.rs | 6 ++- .../planner_resolution/cursor_ranges.rs | 5 ++- .../planner_resolution/orchestration.rs | 7 +-- .../bindings/_helpers/planner_scenario/cli.rs | 5 ++- .../_helpers/planner_scenario/execution.rs | 5 ++- .../_helpers/planner_scenario/graph.rs | 5 ++- .../planner_strategy/cursor_inputs.rs | 3 +- .../planner_strategy/fixture_context.rs | 3 +- .../planner_strategy/fixture_reads.rs | 5 ++- .../_helpers/planner_strategy/lineage.rs | 5 ++- .../planner_strategy/model_entries.rs | 18 ++++---- .../_helpers/planner_strategy/strategy.rs | 3 +- .../_helpers/planner_warehouse/cursors.rs | 5 ++- .../planner_warehouse/semantic_sources.rs | 3 +- .../_helpers/planner_warehouse/snapshot.rs | 11 +++-- .../planner_warehouse/source_deferral.rs | 5 ++- .../planner_warehouse/source_freshness.rs | 3 +- .../_helpers/planner_warehouse/watermarks.rs | 3 +- .../_helpers/sql_test_glue/cursor_window.rs | 5 ++- .../_helpers/sql_test_glue/plan_assembly.rs | 3 +- .../_helpers/sql_test_glue/plan_rendering.rs | 9 ++-- .../_helpers/sql_test_glue/planning.rs | 7 +-- 49 files changed, 247 insertions(+), 133 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index b705e2d294..747d81f21a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -192,3 +192,48 @@ pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { + if let Ok(view) = model.downcast::() { + return view.get().name(model.py()); + } + model.getattr("name")?.extract() +} + +/// The model's compiled query SQL. +pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + return Ok(view + .get() + .query_sql(model.py())? + .into_bound(model.py()) + .into_any()); + } + model.getattr("query_sql") +} + +/// The model's `CompiledObjectKey`. +pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + return Ok(view.get().key(model.py())?.into_bound(model.py())); + } + model.getattr("key") +} + +/// The model's destination `CompiledRelationLocation`. +pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + return Ok(view.get().destination(model.py())?.into_bound(model.py())); + } + model.getattr("destination") +} + +/// The model's name as a Python string. +pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + let name: String = view.get().name(model.py())?; + return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); + } + model.getattr("name") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index 19874d97ce..a23568749f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -12,6 +12,7 @@ use sqlbuild_analysis::diff_selection::errors::SelectorError; use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_diff_selection; use sqlbuild_analysis::diff_selection::models::DiffGraph; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -165,13 +166,13 @@ fn diff_model_names<'py>( let mut names: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - let key: Bound<'py, PyAny> = model.getattr("key")?; + let key: Bound<'py, PyAny> = model_reads::model_key(&model)?; let chosen: bool = match keys.indexes.get_item(&key)? { Some(index) => selected.contains(&index.extract::()?), None => false, }; if chosen && key.getattr("resource_type")?.eq(&model_type)? { - names.push(model.getattr("name")?); + names.push(model_reads::name_object(&model)?); } } PyTuple::new(py, names) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index 4d8e5f3080..511b6b8033 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -72,6 +72,8 @@ pub(crate) struct DestinationFact { /// What assembly and the semantic checks finished for one compiled model. pub(crate) struct CompiledRecord { + /// The model's name, its file stem. + name: String, /// `(resource type, name)` of each dependency. deps: Vec<(String, String)>, destination: DestinationFact, @@ -156,6 +158,7 @@ pub(crate) fn compiled_record( let config: &Bound<'_, PyAny> = input.config.bind(py); let values: Bound<'_, PyDict> = config.getattr(VALUES_ATTRIBUTE)?.downcast_into()?; Ok(CompiledRecord { + name: name.to_owned(), deps, destination: DestinationFact { database: text_value(&values, DATABASE_KEY)?, @@ -363,7 +366,7 @@ fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { #[pymethods] impl NativeCompiledModel { #[getter] - fn key(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| { let compiled: &CompiledRecord = compiled_of(record)?; compiled @@ -406,7 +409,7 @@ impl NativeCompiledModel { } #[getter] - fn query_sql(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn query_sql(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, _| Ok(record.query_sql.clone_ref(py))) } @@ -416,7 +419,7 @@ impl NativeCompiledModel { } #[getter] - fn destination(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { self.compiled(py, |compiled, py| { self.projections .destination @@ -750,6 +753,9 @@ pub(crate) fn model_graph_row( } fn model_name(record: &ModelInputRecord, py: Python<'_>) -> PyResult { + if let Some(compiled) = &record.compiled { + return Ok(compiled.name.clone()); + } record .model_file .bind(py) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index 031744d937..f9944cedda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -7,6 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::janitor_planning::main::migration_origin_keys::migration_origin_keys; use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_name_request_matches; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::janitor::janitor_facts::{ SchemaRelations, Warehouse, model_destinations, @@ -228,7 +229,7 @@ fn project_destinations<'py>(project: &Bound<'py, PyAny>) -> PyResult = project.py(); let destinations: Bound<'py, PyDict> = PyDict::new(py); for (key, model) in model_destinations(project)? { - destinations.set_item(key.to_python(py)?, model.getattr("name")?)?; + destinations.set_item(key.to_python(py)?, model_reads::name_object(&model)?)?; } Ok(destinations) } @@ -369,7 +370,7 @@ pub(crate) fn pending_migration_origins( if listed.contains(&destination.folded()) { continue; } - let model_name: String = model.getattr("name")?.extract()?; + let model_name: String = model_reads::model_name(&model)?; for origin in migration_origin_keys(&raw, destination.database.as_deref(), &schemas) { let folded: Folded = Key { database: origin.database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index ac0c65cc87..e9baf1cdaf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -1,5 +1,6 @@ //! Detect how each selected model and function changed since its recorded build. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -275,7 +276,7 @@ fn scope_model_changes<'py>( if model.is_none() { continue; } - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let expected: (Option, Option) = ( lookup(&version_hashes, &name)?, lookup(&metadata_jsons, &name)?, @@ -299,7 +300,7 @@ pub(crate) fn model_change<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let snapshot: &Bound<'py, PyAny> = &pass.snapshot; - let model_name: Bound<'py, PyAny> = model.getattr("name")?; + let model_name: Bound<'py, PyAny> = model_reads::name_object(model)?; let columns_hash: String = model_declared_columns_hash(model)?; let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { Some(text) => text, @@ -339,7 +340,7 @@ pub(crate) fn model_change<'py>( py, &model_name.extract::()?, ( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(model)?.extract::()?, dialect.extract::>()?.as_deref(), ), )?) @@ -383,7 +384,7 @@ pub(crate) fn model_change<'py>( let renames: Bound<'py, PyAny> = snapshot.getattr("reference_renames")?; let origins: Option> = match (&compiled_query_hash, &recorded) { (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( - (model.getattr("query_sql")?.extract()?, &fingerprint), + (model_reads::query_sql(model)?.extract()?, &fingerprint), &renames, (&recorded.definition_hash, &dialect), )?, @@ -497,7 +498,7 @@ fn log_query_check( &log_sql, &[ ("logger", logger.clone()), - ("sql", model.getattr("query_sql")?), + ("sql", model_reads::query_sql(model)?), ("action", text("compiled_query")), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index dc1bbb01b7..47462cf513 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -10,6 +10,7 @@ use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_ref use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, @@ -240,7 +241,7 @@ fn changed_referencing_query( let dialect: Option = snapshot.getattr("column_dialect")?.extract()?; for model in selected { let recorded: Bound<'_, PyAny> = - recorded_models.call_method1("get", (model.getattr("name")?,))?; + recorded_models.call_method1("get", (model_reads::name_object(&model)?,))?; if recorded.is_none() { continue; } @@ -254,9 +255,9 @@ fn changed_referencing_query( if references_models && model_query_hash( py, - &model.getattr("name")?.extract::()?, + &model_reads::model_name(&model)?, ( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(&model)?.extract::()?, dialect.as_deref(), ), )? != recorded.getattr("definition_hash")?.extract::()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index d8f8bdae34..d12fd6e241 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -1,5 +1,6 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, @@ -145,8 +146,8 @@ fn model_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model.getattr("key")?), - ("name", model.getattr("name")?), + ("key", model_reads::model_key(model)?), + ("name", model_reads::name_object(model)?), ("relative_path", model.getattr("relative_path")?), ("materialization_type", materialization_type), ("action", planner_type(py, RecordId::PlanAction, action)?), @@ -154,7 +155,7 @@ fn model_entry<'py>( "reason", planner_type(py, RecordId::PlanReason, "NO_CHANGE")?, ), - ("destination", model.getattr("destination")?), + ("destination", model_reads::destination(model)?), ("fingerprint_query_sql", string(py, "")), ("resolved_sql", resolved_sql), ("logical_ddl", string(py, "")), @@ -173,7 +174,7 @@ fn clone_model_entries<'py>( let entries = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - if !selected.contains(model.getattr("key")?)? || disabled_model(&model)? { + if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = native( @@ -237,7 +238,7 @@ fn clone_model_entries<'py>( string(py, "") }; entries.set_item( - model.getattr("key")?, + model_reads::model_key(&model)?, model_entry(&model, materialization_type, resolved_sql)?, )?; } @@ -415,7 +416,7 @@ fn origin_model_entries<'py>( let mut entries: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - if !selected_names.contains(model.getattr("name")?)? || disabled_model(&model)? { + if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = native( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index d8d83ae37b..3edf007fad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -1,5 +1,6 @@ //! Static plans for commands that do not build models, and direct seed identity reads. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, @@ -155,7 +156,7 @@ fn build_audit_command_plan<'py>( for model in project.getattr("models")?.try_iter()? { let model = model?; materializations.set_item( - model.getattr("name")?, + model_reads::name_object(&model)?, native( py, NativeAttr::GetMaterializationType, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index a778e2173f..b56f6ed178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,5 +1,6 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, @@ -59,7 +60,7 @@ fn display_entry<'py>( ], )? .is_truthy()?; - let query_sql = model.getattr("query_sql")?; + let query_sql = model_reads::query_sql(model)?; let custom_name = if materialization_type.eq(CUSTOM)? { config(model, "materialized")? } else { @@ -69,8 +70,8 @@ fn display_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model.getattr("key")?), - ("name", model.getattr("name")?), + ("key", model_reads::model_key(model)?), + ("name", model_reads::name_object(model)?), ("relative_path", model.getattr("relative_path")?), ( "action", @@ -93,7 +94,7 @@ fn display_entry<'py>( }, )?, ), - ("destination", model.getattr("destination")?), + ("destination", model_reads::destination(model)?), ("fingerprint_query_sql", query_sql.clone()), ("resolved_sql", query_sql), ("logical_ddl", string(py, "")), @@ -142,7 +143,7 @@ fn build_display_only_sqlbuild_plan<'py>( let mut entries: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - if selected.contains(model.getattr("name")?)? { + if selected.contains(model_reads::name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index efef439137..d0bf74899e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,5 +1,6 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, imported, keywords, }; @@ -239,8 +240,11 @@ fn plan_model_from_change<'py>( } else { cursor_bounds }; - let warnings = - with_cursor_safety_warnings(&assembled.get_item(1)?, &model.getattr("name")?, &bounds)?; + let warnings = with_cursor_safety_warnings( + &assembled.get_item(1)?, + &model_reads::name_object(model)?, + &bounds, + )?; Ok((assembled.get_item(0)?, warnings)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 4b795d067c..9ecd1ba24a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -14,6 +14,7 @@ use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::grap use sqlbuild_analysis::planner_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -149,7 +150,7 @@ fn build_direct_model_version_identities<'py>( if model.is_none() { continue; } - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(&model)?; let metadata_json: String = model_version_identity_metadata_json( &model, Some(function_hashes.as_any()), @@ -159,7 +160,7 @@ fn build_direct_model_version_identities<'py>( py, &name, ( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(&model)?.extract::()?, dialect.as_deref(), ), )?; @@ -276,7 +277,7 @@ fn with_honest_model_write_hashes<'py>( .try_iter()? { let model: Bound<'py, PyAny> = model?; - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { @@ -323,7 +324,7 @@ fn with_honest_model_write_hashes<'py>( if model.is_none() { continue; } - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; if local_hashes.call_method1("get", (&name,))?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 5ac2e6cc45..35d06a6406 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -14,6 +14,7 @@ use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::functio use sqlbuild_analysis::planner_identity::main::stable_identity_hash::stable_identity_hash; use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords, within}; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::planner_identity::hashing::function_definition_hash; @@ -102,7 +103,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( let signature: Bound<'py, PyDict> = execution_signature(model, &config_values, hook_version_hashes)?; version_identity_metadata_json( - &model.getattr("name")?, + &model_reads::name_object(model)?, &config_values, ( Some(local_function_hashes.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index b209ee41a1..4ba5097a0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -1,5 +1,6 @@ //! Gather live candidate evidence without changing adapter inspection ownership. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, @@ -34,7 +35,7 @@ pub(super) fn candidates<'py>( if !materialized.eq("incremental")? && !materialized.eq("snapshot")? { continue; } - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; if args.value("full_refresh_model_names")?.contains(&name)? || !snapshot.getattr("existing_relations")?.contains(&name)? { @@ -86,7 +87,7 @@ pub(super) fn candidates<'py>( py, NativeAttr::ParseQueryShape, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(&model)?), ("dialect", dialect.clone()), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 04ddc289a4..9a8539396a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -1,5 +1,6 @@ //! Materialize column decisions from native proofs and live evidence. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -58,7 +59,7 @@ pub(super) fn decide<'py>( } } let relation = host::entry(py, ExecutorHost::MigrationRelationForLocation)? - .call1((candidate.model.getattr("destination")?,))?; + .call1((model_reads::destination(&candidate.model)?,))?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; @@ -96,8 +97,8 @@ pub(super) fn decide<'py>( py, RecordId::ColumnMigrationPlanEntry, &[ - ("model_name", candidate.model.getattr("name")?), - ("destination", candidate.model.getattr("destination")?), + ("model_name", model_reads::name_object(&candidate.model)?), + ("destination", model_reads::destination(&candidate.model)?), ( "origin_column", if let Some(origin) = origin { @@ -153,7 +154,7 @@ pub(super) fn checked<'py>( warnings: Vec::new(), }); } - let destination = candidate.model.getattr("destination")?; + let destination = model_reads::destination(&candidate.model)?; let qualified = destination.getattr("qualified_name")?; let relation = if physical.is_truthy()? { physical.clone() @@ -225,7 +226,7 @@ pub(super) fn hints<'py>( py, NativeAttr::RenameHints, &[ - ("model_name", candidate.model.getattr("name")?), + ("model_name", model_reads::name_object(&candidate.model)?), ("previous", candidate.previous.clone()), ("current", candidate.current.clone()), ("excluded", excluded.call_method1("union", (claimed,))?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 50b4e856d7..e7f00d1aa4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,5 +1,6 @@ //! Boundary conversions for the column migration planner. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::records::ids::{NativeAttr, NativeItem, RecordId}; @@ -57,8 +58,8 @@ pub(super) fn definition_hash<'py>( model.py(), NativeAttr::ModelDefinitionHash, &[ - ("model_name", model.getattr("name")?), - ("query_sql", model.getattr("query_sql")?), + ("model_name", model_reads::name_object(model)?), + ("query_sql", model_reads::query_sql(model)?), ("dialect", snapshot.getattr("column_dialect")?), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 0204ab771f..2aa146b874 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,5 +1,6 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -27,7 +28,7 @@ pub(super) fn overlay<'py>( let stored = snapshot.getattr("fingerprints")?; let fingerprints = imported(py, "builtins", "dict")?.call1((stored.getattr("models")?,))?; for candidate in candidates { - let name = candidate.model.getattr("name")?; + let name = model_reads::name_object(&candidate.model)?; let renames = PyDict::new(py); let pending = PyDict::new(py); let mut blocked = false; @@ -83,7 +84,7 @@ pub(super) fn overlay<'py>( replace( &candidate.fingerprint, &[ - ("definition", candidate.model.getattr("query_sql")?), + ("definition", model_reads::query_sql(&candidate.model)?), ( "definition_hash", definition_hash(&candidate.model, snapshot)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 902ac4cfec..45dca7498f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -1,5 +1,6 @@ //! Host column migration planning while Python owns warehouse inspection. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::replace; @@ -63,7 +64,7 @@ fn plan_column_migrations<'py>( )?; let schemas = PySet::empty(py)?; for candidate in &candidates { - let schema = candidate.model.getattr("destination")?.getattr("schema")?; + let schema = model_reads::destination(&candidate.model)?.getattr("schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -92,7 +93,7 @@ fn plan_column_migrations<'py>( )?; let physical = args .value("physical_relations")? - .call_method1("get", (candidate.model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(&candidate.model)?,))?; let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { hints.push(hint?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 37862b2218..da8768b2ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -1,5 +1,6 @@ //! Convert existing planner contracts for origin compatibility checks. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -136,9 +137,12 @@ fn check_migration_compatibility<'py>( ); } let relations = PyDict::new(py); - relations.set_item(model.getattr("name")?, relation)?; + relations.set_item(model_reads::name_object(&model)?, relation)?; let columns = PyDict::new(py); - columns.set_item(model.getattr("name")?, arguments.value("origin_columns")?)?; + columns.set_item( + model_reads::name_object(&model)?, + arguments.value("origin_columns")?, + )?; let snapshot = call_with( ®istry::class(py, RecordId::WarehouseSnapshot)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 1a667a1e29..0ce873333a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -1,5 +1,6 @@ //! Migration identity binding with unchanged metadata and hash format. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -101,7 +102,7 @@ fn with_migration_fingerprints<'py>( let fingerprint = call_with( &cache.getattr("fingerprint")?, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(&model)?), ("metadata_json", metadata), ("ref_identities", PyDict::new(py).into_any()), ("dialect", arguments.value("dialect")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index a8c06036a3..f8106545b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -1,5 +1,6 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -14,7 +15,7 @@ fn variants<'py>( model: &Bound<'py, PyAny>, maps: Bound<'py, PyAny>, ) -> PyResult> { - let name = model.getattr("name")?; + let name = model_reads::name_object(model)?; let metadata = arguments.value("metadata_jsons")?.get_item(name)?; let cache = arguments.value("fingerprints")?.getattr("fingerprint")?; let mut hashes: Vec = Vec::new(); @@ -22,7 +23,7 @@ fn variants<'py>( let hash: Option = call_with( &cache, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(model)?), ("metadata_json", metadata.clone()), ("ref_identities", map?), ("dialect", arguments.value("dialect")?), @@ -67,7 +68,7 @@ fn match_migration_fingerprints<'py>( let mut names: Vec = Vec::new(); for model in arguments.value("models")?.try_iter()? { let model = model?; - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(&model)?; if new_names.contains(&name) { names.push(name.clone()); } @@ -131,7 +132,7 @@ fn migration_destination_fingerprints<'py>( let result = PyDict::new(py); for request in arguments.value("requests")?.try_iter()? { let model = request?.getattr("model")?; - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; if arguments .value("metadata_jsons")? .call_method1("get", (&name,))? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index 80672798c7..e46ef09a7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -1,5 +1,6 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_migrations::column_objects::{native, replace, text}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; @@ -17,7 +18,7 @@ pub(super) fn applies( let own = snapshot .getattr("fingerprints")? .getattr("models")? - .call_method1("get", (model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(model)?,))?; Ok(own.is_none() || presumed_utc(&own.getattr("ts")?)? < presumed_utc(&event.getattr("created_at")?)?) } @@ -44,18 +45,18 @@ pub(super) fn handover<'py>( if let Ok(payload) = payload.cast::() && payload.contains(key)? { - payload.set_item(key, model.getattr("name")?)?; + payload.set_item(key, model_reads::name_object(model)?)?; let kwargs = PyDict::new(py); kwargs.set_item("sort_keys", true)?; kwargs.set_item("separators", (",", ":"))?; kwargs.set_item("default", imported(py, "builtins", "str")?)?; metadata = json.getattr("dumps")?.call((payload,), Some(&kwargs))?; } - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; replace( fingerprint, &[ - ("node_name", model.getattr("name")?), + ("node_name", model_reads::name_object(model)?), ("target_database", destination.getattr("database")?), ("target_schema", destination.getattr("schema")?), ("target_name", destination.getattr("name")?), @@ -89,14 +90,14 @@ pub(super) fn equivalent<'py>( replace( &handover, &[ - ("definition", model.getattr("query_sql")?), + ("definition", model_reads::query_sql(model)?), ( "definition_hash", native( py, NativeAttr::ComputeQueryHash, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(model)?), ("dialect", dialect.clone()), ], )?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index f8116bae5b..8ebfdd6972 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -1,5 +1,6 @@ //! Existing old-name planner models at the native decision boundary. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -109,7 +110,7 @@ fn old_name_claim_errors<'py>( let mut errors: Vec> = Vec::new(); for model in arguments.value("selected")?.try_iter()? { let model = model?; - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; let existing = arguments .value("snapshot")? .getattr("existing_relations")? @@ -122,7 +123,7 @@ fn old_name_claim_errors<'py>( if !kind.eq("view")? { continue; } - let destination = relation(&model.getattr("destination")?)?; + let destination = relation(&model_reads::destination(&model)?)?; for view in arguments.value("views")?.try_iter()? { let view = view?; if view.getattr("destination_model")?.eq(&name)? @@ -177,11 +178,11 @@ fn old_name_view_skip_reason<'py>( let mut reuser: Option = None; for model in arguments.value("models")?.try_iter()? { let model = model?; - if relation(&model.getattr("destination")?)? + if relation(&model_reads::destination(&model)?)? .call_method1("matches", (&old,))? .is_truthy()? { - reuser = Some(model.getattr("name")?.extract()?); + reuser = Some(model_reads::name_object(&model)?.extract()?); break; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index 63b451f15c..5e578431fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -1,5 +1,6 @@ //! Per-model full-refresh resolution and full-rebuild refusals. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -40,7 +41,7 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( for model in project.getattr("models")?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { - names.push(model.getattr("name")?); + names.push(model_reads::name_object(&model)?); } } frozenset(project.py(), &names) @@ -80,7 +81,7 @@ pub(crate) fn check_full_rebuild_allowed( { return Ok(()); } - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(model)?; Err(planner_error( py, &format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 5ce7e353c0..a2a54769be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,5 +1,6 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_phases::objects::{ @@ -188,7 +189,7 @@ pub(crate) fn with_storage_policies<'py>( let models = PyDict::new(py); for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } entries = native( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index ff3d379738..0559d5227e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -1,5 +1,6 @@ //! Warehouse-aware time-travel retention and Snowflake table-type planning. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -65,7 +66,7 @@ fn selected_models<'py>( let mut models: Vec<_> = Vec::new(); for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { let model = model?; - if names.contains(&model.getattr("name")?.extract::()?) { + if names.contains(&model_reads::model_name(&model)?) { models.push(model); } } @@ -98,7 +99,7 @@ pub(crate) fn plan_table_types<'py>( .getattr("declared")? .is_truthy()? { - declared.push(model.getattr("name")?.extract::()?); + declared.push(model_reads::model_name(model)?); } } if !declared.is_empty() && !snowflake { @@ -135,7 +136,7 @@ pub(crate) fn plan_table_types<'py>( if !backed.is_truthy()? { continue; } - let relation = relations.call_method1("get", (model.getattr("name")?,))?; + let relation = relations.call_method1("get", (model_reads::name_object(&model)?,))?; if relation.is_none() { continue; } @@ -151,7 +152,7 @@ pub(crate) fn plan_table_types<'py>( continue; } let downgrade = actual == Some(false) && desired_transient; - let destination = model.getattr("destination")?; + let destination = model_reads::destination(&model)?; let limit: i64 = adapter .call_method0("maximum_identifier_length")? .extract()?; @@ -160,7 +161,7 @@ pub(crate) fn plan_table_types<'py>( py, RecordId::TableTypePlanEntry, &[ - ("model_name", model.getattr("name")?), + ("model_name", model_reads::name_object(&model)?), ("destination", destination), ("copy_name", text(py, ©_name(&target_name, limit))), ("desired_type", table_type.getattr("value")?.getattr("value")?), @@ -317,7 +318,7 @@ fn relation_retention<'py>( model: &Bound<'py, PyAny>, ) -> PyResult>> { let py = runtime.py(); - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(model)?; let config = model.getattr("config")?; let retention = config.getattr("time_travel_retention")?; if retention.getattr("desired_days")?.is_none() || retention.getattr("source")?.is_none() { @@ -328,7 +329,7 @@ fn relation_retention<'py>( )); } let desired_days: i64 = retention.getattr("desired_days")?.extract()?; - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; let schema = destination.getattr("schema")?; if schema.is_none() { return Err(planner_error( @@ -341,7 +342,7 @@ fn relation_retention<'py>( py, AdapterHost::RetentionRequest, &[ - ("request_id", model.getattr("name")?), + ("request_id", model_reads::name_object(model)?), ( "scope", member(py, AdapterHost::RetentionScope, "relation")?, @@ -354,7 +355,7 @@ fn relation_retention<'py>( )?; let mut base = RetentionEntry { request: request.clone(), - model_names: tuple(py, vec![model.getattr("name")?])?, + model_names: tuple(py, vec![model_reads::name_object(model)?])?, state: None, source: retention.getattr("source")?.getattr("value")?, }; @@ -363,7 +364,7 @@ fn relation_retention<'py>( let relations = warehouse .getattr("snapshot")? .getattr("existing_relations")?; - if !relations.contains(model.getattr("name")?)? { + if !relations.contains(model_reads::name_object(model)?)? { if snowflake && desired_days > 1 && table_type.eq(TRANSIENT_TABLE_TYPE)? { return Err(transient_error(py, &name, desired_days, "set ")); } @@ -374,7 +375,7 @@ fn relation_retention<'py>( flatten(&changes)?, )?]); } - let relation = relations.get_item(model.getattr("name")?)?; + let relation = relations.get_item(model_reads::name_object(model)?)?; let relation_transient = relation.getattr("is_transient")?; let lookup = call_with( &host::entry(py, AdapterHost::RelationLookup)?.getattr("key")?, @@ -465,7 +466,7 @@ fn bigquery_retention<'py>( for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { let model = model?; let retention = model.getattr("config")?.getattr("time_travel_retention")?; - let destination = model.getattr("destination")?; + let destination = model_reads::destination(&model)?; if retention.getattr("unmanaged")?.is_truthy()? || retention.getattr("desired_days")?.is_none() || destination.getattr("schema")?.is_none() @@ -482,7 +483,7 @@ fn bigquery_retention<'py>( } let mut scopes: BTreeSet<(String, String, Option)> = BTreeSet::new(); for model in models { - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; let database: Option = destination.getattr("database")?.extract()?; let schema: String = destination.getattr("schema")?.str()?.extract()?; scopes.insert((database.clone().unwrap_or_default(), schema, database)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 4338918063..2a4666fb15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,5 +1,6 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_phases::objects::{native, replace, tuple}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use pyo3::IntoPyObject; @@ -66,7 +67,7 @@ pub(crate) fn resolve_planner_scopes<'py>( } let models = PyDict::new(py); for (name, model) in full.getattr("models_by_name")?.downcast::()?.iter() { - if stale_keys.contains(model.getattr("key")?)? { + if stale_keys.contains(model_reads::model_key(&model)?)? { models.set_item(name, model)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 7a41207e0b..ea106be136 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -1,5 +1,6 @@ //! Model cursor orchestration over live warehouse snapshot facts. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::model_config::config_values::{ config_cursor_bound, config_str as authored_config_str, @@ -42,11 +43,11 @@ fn unbounded(py: Python<'_>) -> PyResult> { fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { let relation = snapshot .getattr("existing_relations")? - .call_method1("get", (model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(model)?,))?; if relation.is_none() { return Ok(false); } - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; let key = call_with( &host::entry(model.py(), AdapterHost::RelationLookup)?.getattr("key")?, &[ @@ -61,7 +62,7 @@ fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { let target = snapshot.getattr("target_relation")?; Ok(!target.is_none() - && target.ne(model.getattr("destination")?.getattr("qualified_name")?)?) + && target.ne(model_reads::destination(model)?.getattr("qualified_name")?)?) } #[pyfunction] @@ -118,7 +119,7 @@ fn compute_model_cursor_bounds<'py>( let snapshot = args.value("snapshot")?; let mut cursor_snapshot = snapshot .getattr("cursor_snapshots")? - .call_method1("get", (model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(&model)?,))?; let roles = call_with( ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, &[("model", model.clone())], @@ -147,7 +148,7 @@ fn compute_model_cursor_bounds<'py>( let refusal = call_with( ®istry::native(py, NativeAttr::EmptyInputRebuildRefusal)?, &[ - ("model_name", model.getattr("name")?), + ("model_name", model_reads::name_object(&model)?), ("input_names", empty.clone()), ("cursor_type", cursor_type.clone()), ], @@ -183,7 +184,7 @@ fn compute_model_cursor_bounds<'py>( return Err(PyErr::from_value(call_with( ®istry::native(py, NativeAttr::UnreadableCursorInputsError)?, &[ - ("model_name", model.getattr("name")?), + ("model_name", model_reads::name_object(&model)?), ("cursor_type", cursor_type), ("input_names", names), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 33ba90bdd1..403c5728ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -1,5 +1,6 @@ //! Selected-model inspection and untyped override conversion. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_resolution::values::{method, string}; @@ -45,7 +46,10 @@ fn type_cursor_overrides<'py>( } let kind = config("cursor_type")?; if kind.eq("integer")? || kind.eq("timestamp")? { - models.push((model.getattr("name")?.extract()?, kind.extract()?)); + models.push(( + model_reads::name_object(&model)?.extract()?, + kind.extract()?, + )); } } let mut integer_literals = true; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index c13ec8d03f..da8a171299 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -1,5 +1,6 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -353,7 +354,7 @@ fn model_cursor_snapshot<'py>( ) -> PyResult> { snapshot .getattr("cursor_snapshots")? - .call_method1("get", (model.getattr("name")?,)) + .call_method1("get", (model_reads::name_object(model)?,)) } fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { @@ -566,7 +567,7 @@ pub(crate) fn model_cursor_overrides<'py>( } } let session = CursorSession::new(py)?; - let name = model.getattr("name")?.str()?; + let name = model_reads::name_object(model)?.str()?; let cursor_start = config_bound(model, "cursor_start")?; let cursor_end = config_bound(model, "cursor_end")?; if !resolved_start.is_none() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 862b1f7b5e..c9498d0cc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -1,5 +1,6 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -205,7 +206,7 @@ pub(crate) fn resolve_model_sql<'py>( )?; let inclusive = append_cursor_inclusive(&model)?; let mut sql = resolution.resolve( - model.getattr("query_sql")?, + model_reads::query_sql(&model)?, &[ ("cursor_bounds", bounds.clone()), ("cursor_filter_inputs", roles.getattr("filter_inputs")?), @@ -217,7 +218,7 @@ pub(crate) fn resolve_model_sql<'py>( ®istry::native(py, NativeAttr::ResolveCursorIntrinsics)?, &[("sql", sql.clone())], )?; - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(&model)?; if intrinsics.get_item(1)?.is_truthy()? { if bounds.is_none() { let cause = args.value("full_rebuild_cause")?; @@ -359,7 +360,7 @@ pub(crate) fn resolve_model_hook_entries<'py>( external: external_sql_reference_resolver, }; let values = model.getattr("config")?.getattr("values")?; - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(model)?; let mut hooks: Vec> = Vec::new(); for key in ["pre_hooks", "post_hooks"] { hooks.push(resolution.hook( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 86669773d4..9e7835d0c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -1,5 +1,6 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; @@ -122,12 +123,12 @@ fn namespace<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } for field in ["target_model_names", "model_names"] { for name in graph.getattr(field)?.try_iter()? { if let Some(model) = models.get_item(name?)? { - let destination = model.getattr("destination")?; + let destination = model_reads::destination(&model)?; return Ok(( destination.getattr("database")?, destination.getattr("schema")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index ba484b2854..e6c452e83f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -1,5 +1,6 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -116,11 +117,11 @@ pub(crate) fn build_scenario_execution_plan<'py>( let destination = required_target( &context.relations, "model_locations", - &model.getattr("name")?.extract::()?, + &model_reads::model_name(&model)?, "model", )?; let query = resolve_model_dbt_refs( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(&model)?.extract::()?, &context.relations, )?; let record = registry::native(py, NativeAttr::CompiledModelRecord)?.call1((&model,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 4be6a925ee..e550a8d14e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -1,5 +1,6 @@ //! Convert project views into pure scenario dependency facts. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; @@ -41,7 +42,9 @@ pub(crate) fn plan_scenario_graph<'py>( .try_iter()? .map(|dep| native_key(&dep?)) .collect::>()?; - input.models.insert(model.getattr("name")?.extract()?, deps); + input + .models + .insert(model_reads::name_object(&model)?.extract()?, deps); } for source in project.getattr("sources")?.try_iter()? { input diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 40cc22382d..2497ab3b8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -7,6 +7,7 @@ use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::call_with; @@ -74,7 +75,7 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; let source_map = or_empty(request.optional("source_map")?, py)?; let warehouse_columns: Bound<'py, PyAny> = request.get("source_warehouse_columns")?; - let model_name: String = text(&model.getattr("name")?)?; + let model_name: String = model_reads::model_name(&model)?; for role in ["filter_inputs", "watermark_inputs"] { for item in roles.getattr(role)?.call_method0("items")?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 3d2187cabe..33f67645b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -5,6 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_metadata; use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata, record}; use crate::bindings::_helpers::records::ids::RecordId; @@ -22,7 +23,7 @@ pub(crate) fn build_relation_fixture_context<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, &model)?; + models.set_item(model_reads::name_object(&model)?, &model)?; } models } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 704d9f03fc..4d1332cabd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; @@ -79,7 +80,7 @@ pub(crate) fn required_fixture_columns<'py>( models.push(FixtureReadModel { name, key: FixtureReadKey { - sql: model.getattr("query_sql")?.extract()?, + sql: model_reads::query_sql(&model)?.extract()?, references, dialect: dialect.clone(), }, @@ -174,7 +175,7 @@ fn model_stars( } else { None }; - let sql: String = model.getattr("query_sql")?.extract()?; + let sql: String = model_reads::query_sql(model)?.extract()?; model .py() .compiler_detach(|| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 9fc4edac60..6c14b8c730 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -12,6 +12,7 @@ use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_refere use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::model_config::config_values::config_str; use crate::bindings::_helpers::planner_strategy::microbatches::{ @@ -126,7 +127,7 @@ pub(crate) fn resolve_lineage_reference<'py>( models_by_name, functions_by_name, }; - let name: String = text(&model.getattr("name")?)?; + let name: String = model_reads::model_name(model)?; lineage_reference(&graph, (&name, references(model)?), input_name) .map_err(|failure| lineage_error(py, failure)) } @@ -151,7 +152,7 @@ fn reject_capped_watermark_inputs<'py>( models_by_name, functions_by_name, }; - let name: String = text(&model.getattr("name")?)?; + let name: String = model_reads::model_name(model)?; let inputs: Vec = watermark_names(model)?; match capped_watermark_input(&graph, (&name, references(model)?), inputs) { Ok(None) => Ok(()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 0b69c266ef..05a255c4fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -9,6 +9,7 @@ use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_l use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; use sqlbuild_analysis::planner_strategy::models::LookbackConfig; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; @@ -165,7 +166,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult( let py: Python<'py> = request.py(); let adapter: Bound<'py, PyAny> = request.get("adapter")?; let model: Bound<'py, PyAny> = request.get("model")?; - let destination: Bound<'py, PyAny> = model.getattr("destination")?; + let destination: Bound<'py, PyAny> = model_reads::destination(&model)?; let qualified: Bound<'py, PyAny> = destination.getattr("qualified_name")?; let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { qualified @@ -332,7 +333,7 @@ fn model_warnings<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let model: Bound<'py, PyAny> = request.get("model")?; - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let mut warnings: Vec> = call_with( ®istry::native(py, NativeAttr::BuildModelWarnings)?, &[ @@ -490,7 +491,7 @@ pub(crate) fn assemble_model_plan_entry<'py>( let values: Bound<'py, PyAny> = model_values(&model)?; let change: Bound<'py, PyAny> = request.get("change_result")?; let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let policy: Bound<'py, PyAny> = on_schema_change(&model)?; let schema_actions: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveSchemaActions)?, @@ -522,10 +523,11 @@ pub(crate) fn assemble_model_plan_entry<'py>( (&warehouse_columns, ddl_cursor_bounds(&request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); - for field in ["key", "name", "relative_path", "destination"] { - entry.set_item(field, model.getattr(field)?)?; - } - entry.set_item("fingerprint_query_sql", model.getattr("query_sql")?)?; + entry.set_item("key", model_reads::model_key(&model)?)?; + entry.set_item("name", &name)?; + entry.set_item("relative_path", model.getattr("relative_path")?)?; + entry.set_item("destination", model_reads::destination(&model)?)?; + entry.set_item("fingerprint_query_sql", model_reads::query_sql(&model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; for field in PASSED_FIELDS { entry.set_item(field, request.get(field)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index cf13bb91c8..0a2682167c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -1,5 +1,6 @@ //! Per-model plan actions, schema actions and plan warnings. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; @@ -80,7 +81,7 @@ pub(crate) fn resolve_model_plan_action<'py>( let py: Python<'py> = model.py(); let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; let strategy = ModelStrategy { - name: text(&model.getattr("name")?)?, + name: model_reads::model_name(model)?, disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, materialization: model_materialization(model)?, incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index f316e0e19d..5fff2d990a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -1,5 +1,6 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{ @@ -346,7 +347,7 @@ fn collect_models<'py>( let mut models: Vec<_> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - let name_value = model.getattr("name")?; + let name_value = model_reads::name_object(&model)?; let model_name: String = name_value.extract()?; if !inputs.selected_keys.is_none() && !inputs.selected_keys.contains(model_key(py, &name_value)?)? @@ -390,7 +391,7 @@ fn collect_models<'py>( None => None, }; let target_info = existing_relations.get_item(&name_value)?; - let qualified = model.getattr("destination")?.getattr("qualified_name")?; + let qualified = model_reads::destination(&model)?.getattr("qualified_name")?; if let Some(redirected) = redirected.filter(|value| !value.is_none()) { target_tag = Some(format!("{model_name}__target__max")); target_relation = Some(redirected.extract::()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index f91f79d345..64ecf9b14b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -1,5 +1,6 @@ //! Complete missing source metadata in bounded, database-grouped batches. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; @@ -192,7 +193,7 @@ pub(crate) fn semantic_source_columns<'py>( .getattr("config")? .getattr("values")? .call_method1("get", ("sql_analysis",))?; - if !selected_keys.contains(model.getattr("key")?)? + if !selected_keys.contains(model_reads::model_key(&model)?)? || analysis.is(pyo3::types::PyBool::new(py, false)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index 40177197f8..d0a9af43c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -1,5 +1,6 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; @@ -193,7 +194,11 @@ fn metadata_name_filter( for key in selected.try_iter()? { let name = key?.getattr("name")?; if let Some(model) = models.get_item(&name)? { - names.insert(model.getattr("destination")?.getattr("name")?.extract()?); + names.insert( + model_reads::destination(&model)? + .getattr("name")? + .extract()?, + ); names.extend(model_upstream_names( &model, (&models, &seeds), @@ -461,7 +466,7 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> let mut any_migration = false; for model in project.getattr("models")?.try_iter()? { let model = model?; - if !names.contains(&model.getattr("name")?.extract::()?) { + if !names.contains(&model_reads::model_name(&model)?) { all_selected = false; } if model @@ -847,7 +852,7 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( &snapshot, &[( "target_relation", - model.getattr("destination")?.getattr("qualified_name")?, + model_reads::destination(&model)?.getattr("qualified_name")?, )], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index d31120fc7b..b55cad2956 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -1,5 +1,6 @@ //! Source read deferral for managed source loaders. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -83,10 +84,10 @@ fn selected_managed_refs( let mut names: BTreeSet<_> = BTreeSet::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - if !selected.contains(model.getattr("key")?)? { + if !selected.contains(model_reads::model_key(&model)?)? { continue; } - names.extend(sql_source_reads(&model.getattr("query_sql")?, managed)?); + names.extend(sql_source_reads(&model_reads::query_sql(&model)?, managed)?); names.extend(hook_source_reads(&model, managed)?); } for function in project.getattr("functions")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 6fa37b875c..cb56a20b1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -1,5 +1,6 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::freshness::planning::{ @@ -93,7 +94,7 @@ fn state_schemas( if model.is_none() { continue; } - let schema = model.getattr("destination")?.getattr("schema")?; + let schema = model_reads::destination(&model)?.getattr("schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index ab046e7c97..f9c629d6fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -1,5 +1,6 @@ //! Validate reliable watermark column contracts before warehouse inspection. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; use crate::bindings::_helpers::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, @@ -62,7 +63,7 @@ pub(crate) fn validate_watermark_column<'py>( let py = model.py(); let kind = reference.getattr("ref_kind")?; let ref_name: String = reference.getattr("ref_name")?.extract()?; - let model_name: String = model.getattr("name")?.extract()?; + let model_name: String = model_reads::model_name(model)?; let lowered = |value: &str| -> PyResult { text(py, value).call_method0("lower")?.extract() }; let (names, types) = if kind.eq("ref")? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs index 872e7c9fc0..536be5e414 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -8,6 +8,7 @@ use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::attachments::cursor_intrinsics::{ compile_input_error, resolve_cursor_intrinsics, }; @@ -86,7 +87,7 @@ pub(super) fn declared_window_cursor_models<'py>( let py = test.py(); let mut cursor_models: Vec> = Vec::new(); for model in chain_models { - if uses_cursor_intrinsics(py, &model.getattr("query_sql")?.extract::()?)? { + if uses_cursor_intrinsics(py, &model_reads::query_sql(&model)?.extract::()?)? { cursor_models.push(model); } } @@ -255,7 +256,7 @@ fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { _ => None, }; Ok(ModelCursor { - name: model.getattr("name")?.extract()?, + name: model_reads::name_object(model)?.extract()?, cursor_type, type_name, grain, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index f88373bfe9..926142499f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -74,7 +75,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } let result_class = registry::class(py, RecordId::SqlTestPlanResult)?; let mut results: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs index 350760a6b0..ed52d26a49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs @@ -1,5 +1,6 @@ //! Adapter rendering and declared cursor-window boundaries for native test planning. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; @@ -26,7 +27,7 @@ pub(super) fn models<'py>( let result = PyDict::new(py); for (index, model) in project.getattr("models")?.try_iter()?.enumerate() { let model = model?; - let sql = model.getattr("query_sql")?; + let sql = model_reads::query_sql(&model)?; if mentions(&sql.extract::()?) { result.set_item( index, @@ -46,7 +47,7 @@ pub(super) fn overrides<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } let tests: Vec> = tests.extract()?; let mut indexes: Vec = Vec::new(); @@ -116,12 +117,12 @@ pub(super) fn overrides<'py>( } } for model in declared_window_cursor_models(test, chain_models)? { - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; if !rendered.contains(&name)? { rendered.set_item( name, render_test_cursor_intrinsics( - model.getattr("query_sql")?, + model_reads::query_sql(&model)?, &model, adapter, Some(test), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index 43b10157f0..e7bf16eaf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::compiler::models::{ SqlTestPlanTest, }; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; @@ -99,7 +100,7 @@ fn resolve_compiled_sql_test_chains( .iter() .map(|model| { Ok(SqlTestChainModel { - name: model.getattr("name")?.extract()?, + name: model_reads::name_object(model)?.extract()?, model_dependencies: model_dependencies(model)?, }) }) @@ -165,10 +166,10 @@ fn plan_model( rendered_sql: Option, ) -> PyResult { Ok(SqlTestPlanModel { - name: model.getattr("name")?.extract()?, + name: model_reads::name_object(model)?.extract()?, query_sql: match rendered_sql { Some(sql) => sql, - None => model.getattr("query_sql")?.extract()?, + None => model_reads::query_sql(model)?.extract()?, }, model_dependencies: model_dependencies(model)?, }) From 9b5a60091a354f720fe301037d32fa4c0a4e7361 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:58:40 +0100 Subject: [PATCH 3/7] refactor(planner): call per-model plan checks directly and read the plan context once Per-model planning validates cursor inputs and capped watermark inputs, resolves the materialization, describes and checks full rebuilds and builds cursor input relations through direct native calls instead of looking each one up on the native module. The model plan context's fields are read once per plan instead of once or more per model. --- .../_helpers/planner_changes/actions.rs | 2 +- .../_helpers/planner_entries/model_entries.rs | 152 ++++++++++-------- .../_helpers/planner_strategy/lineage.rs | 2 +- .../_helpers/planner_strategy/strategy.rs | 2 +- 4 files changed, 86 insertions(+), 72 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index 052731e1ce..fee4441401 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -52,7 +52,7 @@ fn resolve_model_actions<'py>( /// Incremental replacements, whose bounds must ignore the destination maximum. #[pyfunction] #[pyo3(signature = (*, materialization_type, action))] -fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { +pub(crate) fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index d0bf74899e..2b7b859fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -4,11 +4,14 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, imported, keywords, }; +use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_entries::values::{ COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, boolean, error_of, native, none, planner_type, replaced, string, }; -use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; +use crate::bindings::_helpers::planner_phases::full_refresh::{ + check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, +}; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -17,9 +20,16 @@ use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_ use crate::bindings::_helpers::planner_resolution::orchestration::{ resolve_model_hook_entries, resolve_model_sql, }; +use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ + build_cursor_input_relations, reject_unexposed_cursor_input_columns, +}; +use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; -use crate::bindings::_helpers::planner_strategy::strategy::resolve_model_plan_action; +use crate::bindings::_helpers::planner_strategy::strategy::{ + materialization_of, resolve_model_plan_action, +}; +use crate::bindings::_helpers::planner_strategy::values::text; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -28,12 +38,41 @@ use pyo3::{pyfunction, wrap_pyfunction}; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; +/// The `ModelPlanContext` fields every model's planning reads, read once per plan. +struct ContextReads<'py> { + context: Bound<'py, PyAny>, + models_by_name: Bound<'py, PyAny>, + functions_by_name: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, + source_warehouse_columns: Bound<'py, PyAny>, + model_locations: Bound<'py, PyAny>, + seed_locations: Bound<'py, PyAny>, + runtime_cursor_producer_names: Bound<'py, PyAny>, + invocation_time: Bound<'py, PyAny>, +} + +impl<'py> ContextReads<'py> { + fn of(context: Bound<'py, PyAny>) -> PyResult { + Ok(Self { + models_by_name: context.getattr("models_by_name")?, + functions_by_name: context.getattr("functions_by_name")?, + source_map: context.getattr("source_map")?, + source_warehouse_columns: context.getattr("source_warehouse_columns")?, + model_locations: context.getattr("model_locations")?, + seed_locations: context.getattr("seed_locations")?, + runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + invocation_time: context.getattr("invocation_time")?, + context, + }) + } +} + /// One model's planning request after change detection. -struct ModelPlanRequest<'py> { +struct ModelPlanRequest<'a, 'py> { model: Bound<'py, PyAny>, snapshot: Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, - context: Bound<'py, PyAny>, + context: &'a ContextReads<'py>, sql_analysis_enabled: Bound<'py, PyAny>, full_refresh: bool, overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), @@ -44,10 +83,11 @@ struct ModelPlanRequest<'py> { /// Build a model plan entry and its warnings from a resolved change result. fn plan_model_from_change<'py>( - request: &ModelPlanRequest<'py>, + request: &ModelPlanRequest<'_, 'py>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let model = &request.model; - let context = &request.context; + let reads = request.context; + let context = &reads.context; let py = model.py(); let full_refresh = boolean(py, request.full_refresh)?; let backfill = if request.full_refresh { @@ -63,60 +103,35 @@ fn plan_model_from_change<'py>( request.change_result.getattr("backfill")? }; let cursor_column = config_str(model, "cursor")?; - native( + reject_unexposed_cursor_input_columns(Some(&keywords( py, - NativeAttr::ValidateSourceCursorInputColumns, &[ ("model", model.clone()), ("cursor_column", cursor_column.clone()), - ("models_by_name", context.getattr("models_by_name")?), - ("functions_by_name", context.getattr("functions_by_name")?), - ("source_map", context.getattr("source_map")?), + ("models_by_name", reads.models_by_name.clone()), + ("functions_by_name", reads.functions_by_name.clone()), + ("source_map", reads.source_map.clone()), ( "source_warehouse_columns", - context.getattr("source_warehouse_columns")?, + reads.source_warehouse_columns.clone(), ), ], - )?; + )?))?; let resolved = resolve_model_plan_action(model, &request.change_result, request.full_refresh)?; let (action, reason) = (resolved.get_item(0)?, resolved.get_item(1)?); - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; - let replaces_relation = native( - py, - NativeAttr::ReplacesIncrementalRelation, - &[ - ("materialization_type", materialization_type.clone()), - ("action", action.clone()), - ], - )? - .is_truthy()?; + let materialization_type = materialization_of(model)?; + let replaces_relation = + replaces_incremental_relation(&text(&materialization_type)?, &text(&action)?); let full_rebuild_cause = if replaces_relation { - native( - py, - NativeAttr::DescribeFullRebuildCause, - &[ - ("change_result", request.change_result.clone()), - ("full_refresh", full_refresh.clone()), - ], - )? + let cause: String = + describe_full_rebuild_cause(&request.change_result, request.full_refresh)?; + if request.protect_full_refresh && !request.full_refresh { + check_full_rebuild_allowed(model, &request.change_result, &cause)?; + } + string(py, &cause) } else { none(py) }; - if !full_rebuild_cause.is_none() && request.protect_full_refresh && !request.full_refresh { - native( - py, - NativeAttr::CheckFullRebuildAllowed, - &[ - ("model", model.clone()), - ("change_result", request.change_result.clone()), - ("full_rebuild_cause", full_rebuild_cause.clone()), - ], - )?; - } let pair = native( py, RecordId::CursorOverridePair, @@ -153,33 +168,29 @@ fn plan_model_from_change<'py>( request.external_resolver.clone(), )?; let (future_config, start_config) = cursor_safety_configs(model, context)?; - native( - py, - NativeAttr::ValidateCappedWatermarkInputs, - &[ - ("model", model.clone()), - ("models_by_name", context.getattr("models_by_name")?), - ("functions_by_name", context.getattr("functions_by_name")?), - ], + reject_capped_watermark_inputs( + model, + reads.models_by_name.clone(), + Some(reads.functions_by_name.clone()), )?; - let cursor_input_relations = native( + let cursor_input_relations = build_cursor_input_relations(Some(&keywords( py, - NativeAttr::BuildCursorInputRelations, &[ ("model", model.clone()), ("adapter", request.adapter.clone()), - ("model_locations", context.getattr("model_locations")?), - ("models_by_name", context.getattr("models_by_name")?), - ("functions_by_name", context.getattr("functions_by_name")?), - ("seed_locations", context.getattr("seed_locations")?), - ("source_map", context.getattr("source_map")?), + ("model_locations", reads.model_locations.clone()), + ("models_by_name", reads.models_by_name.clone()), + ("functions_by_name", reads.functions_by_name.clone()), + ("seed_locations", reads.seed_locations.clone()), + ("source_map", reads.source_map.clone()), ("cursor_column", cursor_column.clone()), ( "runtime_cursor_producer_names", - context.getattr("runtime_cursor_producer_names")?, + reads.runtime_cursor_producer_names.clone(), ), ], - )?; + )?))? + .into_any(); let mut any_runtime_owned = false; for relation in cursor_input_relations.try_iter()? { if relation?.getattr("is_runtime_owned")?.is_truthy()? { @@ -195,7 +206,7 @@ fn plan_model_from_change<'py>( overrides: request.overrides.clone(), runtime_owned, policies: (future_config.clone(), start_config.clone()), - invocation_time: context.getattr("invocation_time")?, + invocation_time: reads.invocation_time.clone(), full_refresh: request.full_refresh, replaces_relation, cursor_input_relations: cursor_input_relations.clone(), @@ -232,7 +243,7 @@ fn plan_model_from_change<'py>( ("end_cursor_override", request.overrides.1.clone()), ("future_cursor_config", future_config), ("start_cursor_config", start_config), - ("invocation_time", context.getattr("invocation_time")?), + ("invocation_time", reads.invocation_time.clone()), ], )?))?; let bounds = if range.is_truthy()? { @@ -483,6 +494,9 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult ("invocation_time", inputs.getattr("invocation_time")?), ], )?; + let reads = ContextReads::of(context)?; + let sql_analysis_enabled = settings.getattr("sql_analysis")?; + let external_resolver = project.getattr("external_sql_reference_resolver")?; let skip = planner_type(py, RecordId::PlanAction, "SKIP")?; let microbatch = registry::class(py, RecordId::IncrementalMode)?.getattr("MICROBATCH")?; let mut entries: Vec> = Vec::new(); @@ -508,12 +522,12 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult model: model.clone(), snapshot: snapshot.clone(), adapter: adapter.clone(), - context: context.clone(), - sql_analysis_enabled: settings.getattr("sql_analysis")?, + context: &reads, + sql_analysis_enabled: sql_analysis_enabled.clone(), full_refresh: effective_full_refresh, overrides, change_result: resolved.getattr("change")?, - external_resolver: project.getattr("external_sql_reference_resolver")?, + external_resolver: external_resolver.clone(), protect_full_refresh: !freshness_blocked.contains(&name)? && !external_blocked.contains(&name)?, })?; @@ -650,7 +664,7 @@ fn plan_model<'py>( model, snapshot, adapter: args.value("adapter")?, - context: args.value("context")?, + context: &ContextReads::of(args.value("context")?)?, sql_analysis_enabled: args.value("sql_analysis_enabled")?, full_refresh: args.flag("full_refresh")?, overrides: ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 6c14b8c730..c790f9de30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -135,7 +135,7 @@ pub(crate) fn resolve_lineage_reference<'py>( /// Reject capped producers feeding a microbatch model's watermark inputs. #[pyfunction(name = "validate_capped_watermark_inputs")] #[pyo3(signature = (*, model, models_by_name, functions_by_name = None))] -fn reject_capped_watermark_inputs<'py>( +pub(crate) fn reject_capped_watermark_inputs<'py>( model: &Bound<'py, PyAny>, models_by_name: Bound<'py, PyAny>, functions_by_name: Option>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index 0a2682167c..f829aa3104 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -62,7 +62,7 @@ fn schema_policy(on_schema_change: &Bound<'_, PyAny>) -> PyResult /// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. #[pyfunction(name = "get_materialization_type")] #[pyo3(signature = (model))] -fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { planner_enum( model.py(), RecordId::MaterializationType, From ce6828001006a0b8e2aa55c493f4e0af86d7fb30 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:16:41 +0100 Subject: [PATCH 4/7] refactor(planner): resolve model SQL and assemble entries through direct native calls Reference resolution passes the SQL between the source, ref, dbt ref, UDF and table-function resolvers as Rust strings and reads the plan context's relations once per resolution. Cursor bounds, cursor input roles, cursor intrinsics, the unresolved-marker check, schema actions, model warnings and the cursor type check are called directly instead of through the native module, and function SQL resolution no longer builds a plan context record to read it back. --- .../_helpers/attachments/reference_markers.rs | 2 +- .../planner_resolution/cursor_model.rs | 2 +- .../planner_resolution/orchestration.rs | 259 +++++++++--------- .../_helpers/planner_resolution/refs.rs | 11 +- .../_helpers/planner_resolution/sources.rs | 2 +- .../_helpers/planner_strategy/cursor_types.rs | 2 +- .../planner_strategy/model_entries.rs | 69 ++--- .../_helpers/planner_strategy/strategy.rs | 4 +- 8 files changed, 173 insertions(+), 178 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs index f259791419..827bd3bc77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs @@ -32,7 +32,7 @@ fn coded_value_error(py: Python<'_>, message: String, code: &str) -> PyResult, sql: &Bound<'_, PyString>, context: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index ea106be136..9626282229 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -67,7 +67,7 @@ fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> #[pyfunction] #[pyo3(signature = (**kwargs))] -fn compute_model_cursor_bounds<'py>( +pub(crate) fn compute_model_cursor_bounds<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index c9498d0cc1..f4f0192106 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -1,97 +1,103 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_intrinsics; +use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, + KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; -use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; +use crate::bindings::_helpers::planner_resolution::refs::{ + resolve_dbt_ref_references, resolve_ref_references, resolve_table_function_references, + resolve_udf_references, +}; +use crate::bindings::_helpers::planner_resolution::sources::resolve_source_references; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +/// What resolving one model's, hook's or function's references reads. struct Resolution<'py> { adapter: Bound<'py, PyAny>, - context: Bound<'py, PyAny>, external: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, + source_warehouse_columns: Bound<'py, PyAny>, + star_exclude_keyword: Bound<'py, PyAny>, + model_locations: Bound<'py, PyAny>, + seed_locations: Bound<'py, PyAny>, + function_locations: Bound<'py, PyAny>, } impl<'py> Resolution<'py> { + /// Resolution against a `ModelPlanContext`'s relations. + fn of_context( + adapter: Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external: Bound<'py, PyAny>, + ) -> PyResult { + Ok(Self { + adapter, + external, + source_map: context.getattr("source_map")?, + source_warehouse_columns: context.getattr("source_warehouse_columns")?, + star_exclude_keyword: context.getattr("star_exclude_keyword")?, + model_locations: context.getattr("model_locations")?, + seed_locations: context.getattr("seed_locations")?, + function_locations: context.getattr("function_locations")?, + }) + } + fn resolve( &self, sql: Bound<'py, PyAny>, cursor: &[(&str, Bound<'py, PyAny>)], - ) -> PyResult> { + ) -> PyResult { let py = self.adapter.py(); - let mut source = vec![("query_sql", sql), ("adapter", self.adapter.clone())]; - for key in [ - "source_map", - "source_warehouse_columns", - "star_exclude_keyword", - ] { - source.push((key, self.context.getattr(key)?)); - } + let mut source = vec![ + ("query_sql", sql), + ("adapter", self.adapter.clone()), + ("source_map", self.source_map.clone()), + ( + "source_warehouse_columns", + self.source_warehouse_columns.clone(), + ), + ("star_exclude_keyword", self.star_exclude_keyword.clone()), + ]; source.extend_from_slice(cursor); - let mut resolved = call_with( - ®istry::native(py, NativeAttr::ResolveSourceReferences)?, - &source, - )?; - let mut refs = vec![("query_sql", resolved), ("adapter", self.adapter.clone())]; - for key in ["model_locations", "seed_locations"] { - refs.push((key, self.context.getattr(key)?)); - } + let resolved: String = resolve_source_references(py, Some(keywords(py, &source)?))?; + let mut refs = vec![ + ("query_sql", PyString::new(py, &resolved).into_any()), + ("adapter", self.adapter.clone()), + ("model_locations", self.model_locations.clone()), + ("seed_locations", self.seed_locations.clone()), + ]; refs.extend_from_slice(cursor); - resolved = call_with( - ®istry::native(py, NativeAttr::ResolveRefReferences)?, - &refs, - )?; - resolved = call_with( - ®istry::native(py, NativeAttr::ResolveDbtRefReferences)?, - &[ - ("query_sql", resolved), - ("external_sql_reference_resolver", self.external.clone()), - ], - )?; - let locations = self.context.getattr("function_locations")?; - let locations = if locations.is_truthy()? { - locations + let resolved: String = resolve_ref_references(py, Some(keywords(py, &refs)?))?; + let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; + let locations = if self.function_locations.is_truthy()? { + self.function_locations.clone() } else { PyDict::new(py).into_any() }; - for name in [ - NativeAttr::ResolveUdfReferences, - NativeAttr::ResolveTableFunctionReferences, - ] { - resolved = call_with( - ®istry::native(py, name)?, - &[ - ("query_sql", resolved), - ("function_locations", locations.clone()), - ("adapter", self.adapter.clone()), - ], - )?; - } - Ok(resolved) + let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; + resolve_table_function_references(&resolved, &locations, &self.adapter) } - fn assert_resolved(&self, sql: &Bound<'py, PyAny>, label: &str) -> PyResult<()> { - call_with( - ®istry::native(self.adapter.py(), NativeAttr::AssertNoUnresolvedSqlMarkers)?, - &[ - ("sql", sql.clone()), - ( - "context", - string(self.adapter.py(), &format!("{label} planned SQL")), - ), - ], - )?; - Ok(()) + fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { + let py = self.adapter.py(); + assert_no_unresolved_sql_markers( + py, + &PyString::new(py, sql), + &format!("{label} planned SQL"), + ) } - fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult> { + fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { let py = self.adapter.py(); let resolved = self.resolve( sql, @@ -163,47 +169,46 @@ pub(crate) fn resolve_model_sql<'py>( ), )?; let model = args.value("model")?; - let resolution = Resolution { - adapter: args.value("adapter")?, - context: args.value("context")?, - external: args.value("external_sql_reference_resolver")?, - }; + let context = args.value("context")?; + let resolution = Resolution::of_context( + args.value("adapter")?, + &context, + args.value("external_sql_reference_resolver")?, + )?; let cursor_type = authored_config_str(&model.getattr("config")?.getattr("values")?, "cursor_type")? .unwrap_or_else(|| py.None().into_bound(py)); let overrides = args.value("cursor_overrides")?; - let bounds = call_with( - ®istry::native(py, NativeAttr::ComputeModelCursorBounds)?, - &[ - ("model", model.clone()), - ("snapshot", args.value("snapshot")?), - ("backfill", args.value("backfill")?), - ("full_refresh", args.value("full_refresh")?), - ( - "start_cursor_override", - overrides.getattr("start_cursor_override")?, - ), - ( - "end_cursor_override", - overrides.getattr("end_cursor_override")?, - ), - ( - "runtime_cursor_producer_names", - resolution - .context - .getattr("runtime_cursor_producer_names")?, - ), - ("context", resolution.context.clone()), - ( - "replaces_relation", - args.value_or_false("replaces_relation")?, - ), - ], - )?; - let roles = call_with( - ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, - &[("model", model.clone())], + let bounds = compute_model_cursor_bounds( + py, + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", args.value("snapshot")?), + ("backfill", args.value("backfill")?), + ("full_refresh", args.value("full_refresh")?), + ( + "start_cursor_override", + overrides.getattr("start_cursor_override")?, + ), + ( + "end_cursor_override", + overrides.getattr("end_cursor_override")?, + ), + ( + "runtime_cursor_producer_names", + context.getattr("runtime_cursor_producer_names")?, + ), + ("context", context.clone()), + ( + "replaces_relation", + args.value_or_false("replaces_relation")?, + ), + ], + )?), )?; + let roles = resolve_cursor_input_roles(&model)?; let inclusive = append_cursor_inclusive(&model)?; let mut sql = resolution.resolve( model_reads::query_sql(&model)?, @@ -214,12 +219,9 @@ pub(crate) fn resolve_model_sql<'py>( ("lower_bound_inclusive", inclusive), ], )?; - let intrinsics = call_with( - ®istry::native(py, NativeAttr::ResolveCursorIntrinsics)?, - &[("sql", sql.clone())], - )?; + let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; let name: String = model_reads::model_name(&model)?; - if intrinsics.get_item(1)?.is_truthy()? { + if uses_intrinsics { if bounds.is_none() { let cause = args.value("full_rebuild_cause")?; let cause = if cause.is_truthy()? { @@ -234,7 +236,7 @@ pub(crate) fn resolve_model_sql<'py>( sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; - Ok(sql) + Ok(PyString::new(py, &sql).into_any()) } /// Effective append cursor lower-bound inclusivity. @@ -256,13 +258,13 @@ fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult( - sql: &Bound<'py, PyAny>, + sql: &str, bounds: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, -) -> PyResult> { - let py = sql.py(); - let literal = |field: &str| -> PyResult> { +) -> PyResult { + let py = adapter.py(); + let literal = |field: &str| -> PyResult { call_with( &adapter.getattr("render_cursor_bound_literal")?, &[ @@ -272,19 +274,12 @@ fn intrinsic_bounds<'py>( ), ("cursor_type", cursor_type.clone()), ], - ) + )? + .extract() }; let start_sql = literal("start")?; let end_sql = literal("end")?; - call_with( - ®istry::native(py, NativeAttr::ResolveCursorIntrinsics)?, - &[ - ("sql", sql.clone()), - ("start_sql", start_sql), - ("end_sql", end_sql), - ], - )? - .get_item(0) + Ok(resolve_cursor_intrinsics(py, sql, Some(&start_sql), Some(&end_sql))?.0) } fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> PyResult { @@ -325,25 +320,21 @@ fn resolve_function_sql<'py>( &[], ), )?; - let mut context = vec![("models_by_name", PyDict::new(py).into_any())]; - for key in [ - "model_locations", - "seed_locations", - "function_locations", - "source_map", - "source_warehouse_columns", - "star_exclude_keyword", - ] { - context.push((key, args.value(key)?)); - } let resolution = Resolution { adapter: args.value("adapter")?, - context: call_with(®istry::class(py, RecordId::ModelPlanContext)?, &context)?, external: py.None().into_bound(py), + source_map: args.value("source_map")?, + source_warehouse_columns: args.value("source_warehouse_columns")?, + star_exclude_keyword: args.value("star_exclude_keyword")?, + model_locations: args.value("model_locations")?, + seed_locations: args.value("seed_locations")?, + function_locations: args.value("function_locations")?, }; let function = args.value("function")?; let name: String = function.getattr("name")?.extract()?; - resolution.unbounded(function.getattr("body_sql")?, &format!("Function '{name}'")) + let sql: String = + resolution.unbounded(function.getattr("body_sql")?, &format!("Function '{name}'"))?; + Ok(PyString::new(py, &sql).into_any()) } #[pyfunction] @@ -354,11 +345,7 @@ pub(crate) fn resolve_model_hook_entries<'py>( context: Bound<'py, PyAny>, external_sql_reference_resolver: Bound<'py, PyAny>, ) -> PyResult> { - let resolution = Resolution { - adapter, - context, - external: external_sql_reference_resolver, - }; + let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; let values = model.getattr("config")?.getattr("values")?; let name: String = model_reads::model_name(model)?; let mut hooks: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index e212bbf96f..63475c1b12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -14,7 +14,10 @@ use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans:: #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { +pub(crate) fn resolve_ref_references( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { let args = KeywordArguments::read( py, "resolve_ref_references", @@ -72,7 +75,7 @@ fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> #[pyfunction] #[pyo3(signature = (*, query_sql, external_sql_reference_resolver=None))] -fn resolve_dbt_ref_references( +pub(crate) fn resolve_dbt_ref_references( query_sql: &str, external_sql_reference_resolver: Option<&Bound<'_, PyAny>>, ) -> PyResult { @@ -195,7 +198,7 @@ fn resolve_function_calls( #[pyfunction] #[pyo3(signature = (*, query_sql, function_locations, adapter))] -fn resolve_udf_references( +pub(crate) fn resolve_udf_references( query_sql: &str, function_locations: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, @@ -205,7 +208,7 @@ fn resolve_udf_references( #[pyfunction] #[pyo3(signature = (*, query_sql, function_locations, adapter))] -fn resolve_table_function_references( +pub(crate) fn resolve_table_function_references( query_sql: &str, function_locations: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 8c09e4e2b5..d6d44f8596 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -279,7 +279,7 @@ fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyRes #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_source_references( +pub(crate) fn resolve_source_references( py: Python<'_>, kwargs: Option>, ) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index b20c8c11e1..29fcf254be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -69,7 +69,7 @@ fn classify_cursor_sql_type<'py>( /// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. #[pyfunction] #[pyo3(signature = (**options))] -fn check_cursor_type_consistency<'py>( +pub(crate) fn check_cursor_type_consistency<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 05a255c4fc..9143134c54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -11,7 +11,7 @@ use sqlbuild_analysis::planner_strategy::models::LookbackConfig; use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::{ config_bool, config_cursor_bound, config_int, config_str, config_string_tuple, @@ -23,10 +23,14 @@ use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; +use crate::bindings::_helpers::planner_strategy::cursor_types::check_cursor_type_consistency; use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; +use crate::bindings::_helpers::planner_strategy::strategy::{ + build_model_warnings, resolve_schema_actions, +}; use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; /// Model config keys copied to the plan entry as optional strings, with their entry fields. @@ -334,33 +338,39 @@ fn model_warnings<'py>( let py: Python<'py> = request.py(); let model: Bound<'py, PyAny> = request.get("model")?; let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; - let mut warnings: Vec> = call_with( - ®istry::native(py, NativeAttr::BuildModelWarnings)?, - &[ - ("model_name", name.clone()), - ("materialization_type", request.get("materialization_type")?), - ("change_result", request.get("change_result")?), - ("schema_actions", schema_actions.clone()), - ("on_schema_change", policy.clone()), - ("type_enforcement", enforcement.clone()), - ], + let mut warnings: Vec> = build_model_warnings( + py, + Some(keywords( + py, + &[ + ("model_name", name.clone()), + ("materialization_type", request.get("materialization_type")?), + ("change_result", request.get("change_result")?), + ("schema_actions", schema_actions.clone()), + ("on_schema_change", policy.clone()), + ("type_enforcement", enforcement.clone()), + ], + )?), )? .try_iter()? .collect::>()?; - let cursor_warning: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CheckCursorTypeConsistency)?, - &[ - ("model_name", name), - ("cursor_column", request.get("cursor_column")?), - ( - "cursor_type", - or_none(py, config_str(&model_values(&model)?, "cursor_type")?), - ), - ("warehouse_columns", warehouse_columns.clone()), - ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), - ], + let cursor_warning: Option> = check_cursor_type_consistency( + py, + Some(keywords( + py, + &[ + ("model_name", name), + ("cursor_column", request.get("cursor_column")?), + ( + "cursor_type", + or_none(py, config_str(&model_values(&model)?, "cursor_type")?), + ), + ("warehouse_columns", warehouse_columns.clone()), + ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), + ], + )?), )?; - if !cursor_warning.is_none() { + if let Some(cursor_warning) = cursor_warning.filter(|warning| !warning.is_none()) { warnings.push(cursor_warning); } PyTuple::new(py, warnings) @@ -493,13 +503,8 @@ pub(crate) fn assemble_model_plan_entry<'py>( let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let policy: Bound<'py, PyAny> = on_schema_change(&model)?; - let schema_actions: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveSchemaActions)?, - &[ - ("schema_findings", change.getattr("schema_findings")?), - ("on_schema_change", policy.clone()), - ], - )?; + let schema_actions: Bound<'py, PyAny> = + resolve_schema_actions(&change.getattr("schema_findings")?, &policy)?.into_any(); let enforcement: Bound<'py, PyAny> = type_enforcement(&model)?; let warehouse_columns: Bound<'py, PyAny> = snapshot .getattr("existing_columns")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index f829aa3104..ad4e0aa5eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -109,7 +109,7 @@ pub(crate) fn resolve_model_plan_action<'py>( /// Concrete schema change actions from findings and `on_schema_change`. #[pyfunction] #[pyo3(signature = (*, schema_findings, on_schema_change))] -fn resolve_schema_actions<'py>( +pub(crate) fn resolve_schema_actions<'py>( schema_findings: &Bound<'py, PyAny>, on_schema_change: &Bound<'py, PyAny>, ) -> PyResult> { @@ -141,7 +141,7 @@ fn resolve_schema_actions<'py>( /// Warnings for one model from its change detection and plan decisions. #[pyfunction] #[pyo3(signature = (**options))] -fn build_model_warnings<'py>( +pub(crate) fn build_model_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { From cdce1b1f4e4bf5e6059b5b857a2bbb312a068be1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:28:46 +0100 Subject: [PATCH 5/7] refactor(planner): run plan phases through direct native calls The execution plan calls scope resolution, cursor override typing, change detection, warehouse state, identities, buildability, model actions, source freshness, pruning, reconciliation, entry building and plan output assembly directly, and warehouse state calls model, column and old-name migration planning and the relations context directly, instead of looking each phase up on the native module. The SQL-test planning exit test now fails the plan with a missing source table instead of replacing a phase. --- .../_helpers/planner_changes/actions.rs | 2 +- .../_helpers/planner_changes/detect.rs | 2 +- .../planner_entries/execution_plan.rs | 309 ++++++++---------- .../bindings/_helpers/planner_entries/mod.rs | 2 +- .../_helpers/planner_entries/relations.rs | 2 +- .../planner_migrations/column_planning.rs | 2 +- .../_helpers/planner_migrations/mod.rs | 6 +- .../planner_migrations/model_planning.rs | 2 +- .../planner_migrations/old_name_planning.rs | 2 +- .../planner_phases/warehouse_state.rs | 114 ++++--- .../planner_resolution/cursor_overrides.rs | 2 +- .../_helpers/planner_resolution/mod.rs | 2 +- .../classes/test_sql_test_planning_exit.py | 18 +- 13 files changed, 224 insertions(+), 241 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index fee4441401..e6fbde744d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -15,7 +15,7 @@ const CREATE_TABLE_ACTION: &str = "create_table"; /// Each selected model's change with the backfill its own `replay_on_change` decided. #[pyfunction] #[pyo3(signature = (*, scope, changes))] -fn resolve_model_actions<'py>( +pub(crate) fn resolve_model_actions<'py>( scope: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index e9baf1cdaf..db69992068 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -62,7 +62,7 @@ pub(crate) struct ModelPass<'py> { /// Selected model and function changes. #[pyfunction] #[pyo3(signature = (**options))] -fn detect_changes<'py>( +pub(crate) fn detect_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 6fa68d01ba..a380d3b868 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -1,15 +1,31 @@ //! Top-level planner orchestration producing an execution plan. use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, within, + KeywordArguments, call_with, imported, keywords, within, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_changes::actions::resolve_model_actions; +use crate::bindings::_helpers::planner_changes::detect::detect_changes; use crate::bindings::_helpers::planner_entries::test_planning::{ PlanningRequest, SqlTestPlanning, start_planning, }; use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; +use crate::bindings::_helpers::planner_phases::execution_changes::{ + build_planner_entry_results, prune_planner_execution_scope, reconcile_execution_changes, +}; +use crate::bindings::_helpers::planner_phases::identities::{ + build_planner_identity_context, detect_stale_warning_changes, +}; +use crate::bindings::_helpers::planner_phases::output_assembly::{ + assemble_base_plan_output, with_plan_metadata, with_plan_warnings, with_storage_policies, +}; +use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; +use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; +use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -22,14 +38,10 @@ fn scopes_and_cursor_overrides<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let project = args.value("project")?; let py = project.py(); - let scopes = native( - py, - NativeAttr::ResolvePlannerScopes, - &[ - ("project", project.clone()), - ("selection", args.value("selection")?), - ("policies", args.value("policies")?), - ], + let scopes = resolve_planner_scopes( + &project, + &args.value("selection")?, + &args.value("policies")?, )?; let selected = scopes.getattr("selected_scope")?; let models_by_name = selected.getattr("models_by_name")?; @@ -43,13 +55,15 @@ fn scopes_and_cursor_overrides<'py>( } } let overrides = args.value("overrides")?; - let typed = native( + let typed = type_cursor_overrides( py, - NativeAttr::TypeCursorOverrides, - &[ - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ("selected_models", PyTuple::new(py, models)?.into_any()), - ], + Some(keywords( + py, + &[ + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ("selected_models", PyTuple::new(py, models)?.into_any()), + ], + )?), )?; Ok(( scopes, @@ -66,23 +80,25 @@ fn change_results<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let versions = identities.getattr("version_identities")?; - let changes = native( + let changes = detect_changes( py, - NativeAttr::DetectChanges, - &[ - ("project", project.clone()), - ("scope", scopes.getattr("inspection_scope")?), - ("snapshot", snapshot.clone()), - ("full_refresh", overrides.getattr("full_refresh")?), - ( - "expected_version_hashes", - versions.getattr("model_version_hashes")?, - ), - ( - "expected_metadata_jsons", - versions.getattr("model_metadata_jsons")?, - ), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("scope", scopes.getattr("inspection_scope")?), + ("snapshot", snapshot.clone()), + ("full_refresh", overrides.getattr("full_refresh")?), + ( + "expected_version_hashes", + versions.getattr("model_version_hashes")?, + ), + ( + "expected_metadata_jsons", + versions.getattr("model_metadata_jsons")?, + ), + ], + )?), )?; if !policies.getattr("selection_diagnostics")?.is_truthy()? { let empty = native( @@ -95,16 +111,18 @@ fn change_results<'py>( )?; return Ok((changes, empty)); } - let stale = native( + let stale = detect_stale_warning_changes( py, - NativeAttr::DetectStaleWarningChanges, - &[ - ("project", project.clone()), - ("scopes", scopes.clone()), - ("snapshot", snapshot.clone()), - ("identities", identities.clone()), - ("execution_changes", changes.clone()), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("scopes", scopes.clone()), + ("snapshot", snapshot.clone()), + ("identities", identities.clone()), + ("execution_changes", changes.clone()), + ], + )?), )?; Ok((changes, stale)) } @@ -120,39 +138,14 @@ fn planned_output<'py>( let py = project.py(); let deferral = args.value("deferral")?; let policies = args.value("policies")?; - let warehouse = native( - py, - NativeAttr::GatherPlannerWarehouseState, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ], - )?; + let warehouse = gather_planner_warehouse_state(runtime, scopes, overrides, &deferral)?; let started = Instant::now(); - let identities = native( - py, - NativeAttr::BuildPlannerIdentityContext, - &[ - ("project", project.clone()), - ("scopes", scopes.clone()), - ( - "include_stale_warning_identities", - policies.getattr("selection_diagnostics")?, - ), - ], - )?; - native( - py, - NativeAttr::CheckSelectedScopeBuildability, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("warehouse", warehouse.clone()), - ("deferral", deferral.clone()), - ], + let identities = build_planner_identity_context( + &project, + scopes, + policies.getattr("selection_diagnostics")?.extract()?, )?; + check_selected_scope_buildability(runtime, scopes, &warehouse, &deferral)?; let snapshot = warehouse.getattr("snapshot")?; let (changes, stale_warning_changes) = change_results( &project, @@ -161,114 +154,92 @@ fn planned_output<'py>( &policies, )?; let inspection_scope = scopes.getattr("inspection_scope")?; - let resolved_actions = native( + let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; + let source_freshness = build_planner_source_freshness_result( py, - NativeAttr::ResolveModelActions, - &[ - ("scope", inspection_scope.clone()), - ("changes", changes.clone()), - ], - )?; - let source_freshness = native( - py, - NativeAttr::BuildPlannerSourceFreshnessResult, - &[ - ("project", project.clone()), - ("adapter", args.value("adapter")?), - ("connection", args.value("connection")?), - ("scope", inspection_scope), - ("relations", warehouse.getattr("inspection_relations")?), - ( - "freshness_state_schemas", - snapshot.getattr("source_freshness_state_schemas")?, - ), - ], - )?; - let pruning = native( - py, - NativeAttr::PrunePlannerExecutionScope, - &[ - ("scopes", scopes.clone()), - ("resolved_actions", resolved_actions), - ], - )?; - let reconciliation = native( - py, - NativeAttr::ReconcileExecutionChanges, - &[ - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("pruning", pruning.clone()), - ("changes", changes), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", args.value("adapter")?), + ("connection", args.value("connection")?), + ("scope", inspection_scope), + ("relations", warehouse.getattr("inspection_relations")?), + ( + "freshness_state_schemas", + snapshot.getattr("source_freshness_state_schemas")?, + ), + ], + )?), )?; - let entries = native( + let pruning = prune_planner_execution_scope(scopes, &resolved_actions)?; + let reconciliation = reconcile_execution_changes(&warehouse, &identities, &pruning, &changes)?; + let entries = build_planner_entry_results( py, - NativeAttr::BuildPlannerEntryResults, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("policies", policies.clone()), - ("deferral", deferral), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation.clone()), - ("source_freshness", source_freshness.clone()), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("policies", policies.clone()), + ("deferral", deferral), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation.clone()), + ("source_freshness", source_freshness.clone()), + ], + )?), )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); - let mut plan_output = native( - py, - NativeAttr::AssembleBasePlanOutput, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation), - ("entries", entries), - ("source_freshness", source_freshness.clone()), - ("planned_sql_tests", planned_sql_tests), - ], - )?; - plan_output = native( + let mut plan_output = assemble_base_plan_output( py, - NativeAttr::WithStoragePolicies, - &[ - ("plan_output", plan_output), - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scopes", scopes.clone()), - ("policies", policies.clone()), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation), + ("entries", entries), + ("source_freshness", source_freshness.clone()), + ("planned_sql_tests", planned_sql_tests), + ], + )?), )?; - plan_output = native( + plan_output = with_storage_policies( py, - NativeAttr::WithPlanWarnings, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("warehouse", warehouse), - ("identities", identities), - ("stale_warning_changes", stale_warning_changes), - ("pruning", pruning.clone()), - ("source_freshness", source_freshness.clone()), - ("plan_output", plan_output), - ("policies", policies.clone()), - ], + Some(keywords( + py, + &[ + ("plan_output", plan_output), + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("scopes", scopes.clone()), + ("policies", policies.clone()), + ], + )?), )?; - plan_output = native( + plan_output = with_plan_warnings( py, - NativeAttr::WithPlanMetadata, - &[ - ("plan_output", plan_output), - ("pruning", pruning), - ("source_freshness", source_freshness), - ("policies", policies), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("warehouse", warehouse), + ("identities", identities), + ("stale_warning_changes", stale_warning_changes), + ("pruning", pruning.clone()), + ("source_freshness", source_freshness.clone()), + ("plan_output", plan_output), + ("policies", policies.clone()), + ], + )?), )?; + plan_output = with_plan_metadata(&plan_output, &pruning, &source_freshness, &policies)?; let on_progress = args.value("on_progress")?; if !on_progress.is_none() { on_progress.call1((format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 31de37adfb..4ef09d1d75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -7,6 +7,6 @@ mod execution_plan; mod model_entries; mod plan_output; pub(crate) mod registration; -mod relations; +pub(crate) mod relations; mod test_planning; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 723a14b74c..07db82f0c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -211,7 +211,7 @@ pub(crate) fn planner_relations_context<'py>( /// Resolve relation locations and source metadata for plan entry construction. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_planner_relations_context<'py>( +pub(crate) fn build_planner_relations_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 45dca7498f..33c059bba3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -14,7 +14,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; /// Decide column renames and return the existing planning result model. #[pyfunction] #[pyo3(signature = (**options))] -fn plan_column_migrations<'py>( +pub(crate) fn plan_column_migrations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index a6a080315f..44516c9d52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -2,7 +2,7 @@ mod column_candidates; mod column_entries; mod column_objects; mod column_overlay; -mod column_planning; +pub(crate) mod column_planning; mod column_warnings; pub(crate) mod columns; mod compatibility; @@ -18,12 +18,12 @@ mod matching; mod model_entries; mod model_handover; mod model_overlay; -mod model_planning; +pub(crate) mod model_planning; mod model_warnings; mod models; mod old_name_entries; mod old_name_objects; -mod old_name_planning; +pub(crate) mod old_name_planning; mod old_name_references; mod old_names; mod planning_boundary; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 687f6d4239..f16dd1a153 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -102,7 +102,7 @@ fn decide_model_migration<'py>( #[pyfunction] #[pyo3(signature = (**options))] -fn plan_model_migrations<'py>( +pub(crate) fn plan_model_migrations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 3fe90cdd55..2b506c7163 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -69,7 +69,7 @@ fn schemas<'py>( #[pyfunction] #[pyo3(signature = (**options))] -fn plan_old_name_views<'py>( +pub(crate) fn plan_old_name_views<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index 1ac5f15e7b..e06e19c04c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,12 +1,15 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_entries::relations::build_planner_relations_context; +use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; +use crate::bindings::_helpers::planner_migrations::model_planning::plan_model_migrations; +use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -85,30 +88,35 @@ pub(crate) fn gather_planner_warehouse_state<'py>( deferred_locations: deferral.getattr("deferred_locations")?, cursor_scope, })?; - let migrations = native( + let migrations = plan_model_migrations( py, - NativeAttr::PlanModelMigrations, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", snapshot.clone()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ("fingerprints", fingerprints.clone()), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope.clone()), + ("snapshot", snapshot.clone()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ("fingerprints", fingerprints.clone()), + ], + )?), )?; fingerprints.call_method0("persist")?; - let inspection_relations = call_with( - ®istry::native(py, NativeAttr::BuildPlannerRelationsContext)?, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", runtime.getattr("connection")?), - ("scope", scopes.getattr("inspection_scope")?), - ("deferral", deferral.clone()), - ("project_config", config), - ("local_config", runtime.getattr("local_config")?), - ], + let inspection_relations = build_planner_relations_context( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", runtime.getattr("connection")?), + ("scope", scopes.getattr("inspection_scope")?), + ("deferral", deferral.clone()), + ("project_config", config), + ("local_config", runtime.getattr("local_config")?), + ], + )?), )?; let physical = PyDict::new(py); for entry in migrations.getattr("entries")?.try_iter()? { @@ -130,36 +138,40 @@ pub(crate) fn gather_planner_warehouse_state<'py>( )?; } } - let columns = native( + let columns = plan_column_migrations( py, - NativeAttr::PlanColumnMigrations, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", migrations.getattr("snapshot")?), - ( - "full_refresh_model_names", - effectively_full_refreshed_model_names(&project, full_refresh)?, - ), - ("physical_relations", physical.into_any()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ( - "source_columns", - inspection_relations.getattr("source_warehouse_columns")?, - ), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope.clone()), + ("snapshot", migrations.getattr("snapshot")?), + ( + "full_refresh_model_names", + effectively_full_refreshed_model_names(&project, full_refresh)?, + ), + ("physical_relations", physical.into_any()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ( + "source_columns", + inspection_relations.getattr("source_warehouse_columns")?, + ), + ], + )?), )?; - let old_names = native( + let old_names = plan_old_name_migration_views( py, - NativeAttr::PlanOldNameViews, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope), - ("snapshot", snapshot), - ("migration_entries", migrations.getattr("entries")?), - ("column_entries", columns.getattr("entries")?), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope), + ("snapshot", snapshot), + ("migration_entries", migrations.getattr("entries")?), + ("column_entries", columns.getattr("entries")?), + ], + )?), )?; if !runtime.getattr("on_progress")?.is_none() { progress( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 403c5728ba..8c069d8dd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -15,7 +15,7 @@ use sqlbuild_cursor::planner::main::override_kind::override_kind; #[pyfunction] #[pyo3(signature = (**kwargs))] -fn type_cursor_overrides<'py>( +pub(crate) fn type_cursor_overrides<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index df9dfd5f7e..d5f276315b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -3,7 +3,7 @@ mod constants; mod cursor_bounds; mod cursor_model; -mod cursor_overrides; +pub(crate) mod cursor_overrides; pub(crate) mod cursor_policies; pub(crate) mod cursor_ranges; mod cursor_safety; diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index fc73db0258..463fa7cd55 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -39,14 +39,7 @@ def slow_plan_selected_sql_tests(**arguments): _FAILED_BUILD: str = dedent( """ from sqlbuild.cli.entry.main.entry import main - from sqlbuild._native import PlannerInputError - - def failing_reconciliation(**arguments): - raise PlannerInputError("forced planning failure") - - - native.reconcile_execution_changes = failing_reconciliation sys.exit(main(["--project-dir", sys.argv[1], "--no-color", "build"])) """ ) @@ -73,7 +66,7 @@ def failing_reconciliation(**arguments): "plan fails while SQL tests are still planning", _SLOW_TEST_PLANNING + _FAILED_BUILD, 1, - "error[S000]: forced planning failure", + "error[S405]: cannot build selected scope", ), PlanningExitTestCase( "SQL test planning never joined", @@ -95,8 +88,15 @@ def test_given_sql_test_planning_running_when_interpreter_finalizes_then_process '[connection]\ndatabase = "orders_project.duckdb"\n', encoding="utf-8", ) + (project_dir / "sources").mkdir() + (project_dir / "sources" / "raw.yml").write_text( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: raw\n table: orders\n", + encoding="utf-8", + ) (project_dir / "models" / "orders.sql").write_text( - "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized table);\n\n" + 'SELECT * FROM __source("raw_orders")\n', encoding="utf-8", ) script: Path = tmp_path / "planning_exit.py" From b8c098a5cb4a081d128799143351155341ea1a9a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:33:14 +0100 Subject: [PATCH 6/7] refactor(planner): keep fixture planning context as Rust data and build the plan output once The fixture planning context is a native class holding relation columns, authoritative names and expected types as Rust maps, with each model's lineage reads and type projections derived once per context instead of once per SQL test. Fixture completion, selection, empty-fixture rendering, typed-null completion and expected-output checks read it directly, and the completion's expected types are the chain's propagated types over the context's instead of a ChainMap. The Python FixtureColumnMetadata and FixtureRelationMetadata records and the per-step fixture pyfunctions are gone; SQL-test planning and scenario fixtures call the context, chain resolution, test planning and completion directly. A native counter reports context builds and chain resolutions for tests. The plan output is built once: base fields, storage policies, warnings and direct-mode metadata are gathered as fields and the PlanOutput is constructed at the end instead of being rebuilt by each phase. --- .../src/planner_fixtures/models.rs | 4 +- .../analysis_session/fixture_inferences.rs | 2 +- .../planner_entries/execution_plan.rs | 59 +--- .../bindings/_helpers/planner_entries/mod.rs | 2 +- .../_helpers/planner_entries/plan_output.rs | 172 +++++------ .../_helpers/planner_phases/constants.rs | 11 + .../bindings/_helpers/planner_phases/mod.rs | 1 + .../_helpers/planner_phases/models.rs | 10 + .../planner_phases/output_assembly.rs | 284 ++++++++--------- .../_helpers/planner_phases/registration.rs | 16 - .../_helpers/planner_records/constants.rs | 34 --- .../_helpers/planner_scenario/execution.rs | 37 +-- .../_helpers/planner_strategy/constants.rs | 4 + .../planner_strategy/fixture_completion.rs | 285 ++++++++++-------- .../planner_strategy/fixture_context.rs | 281 ++++++++++++----- .../planner_strategy/fixture_metadata.rs | 33 +- .../planner_strategy/fixture_preparation.rs | 17 +- .../planner_strategy/fixture_reads.rs | 91 +++--- .../planner_strategy/fixture_selection.rs | 80 +++-- .../_helpers/planner_strategy/fixture_sql.rs | 43 ++- .../planner_strategy/fixture_types.rs | 83 ++--- .../planner_strategy/fixture_values.rs | 27 +- .../bindings/_helpers/planner_strategy/mod.rs | 1 + .../_helpers/planner_strategy/models.rs | 51 +++- .../_helpers/planner_strategy/registration.rs | 6 +- .../_helpers/planner_strategy/types.rs | 7 + .../src/bindings/_helpers/records/ids.rs | 8 - .../src/bindings/_helpers/records/models.rs | 10 - .../sql_test_glue/fixture_validation.rs | 80 ++--- .../bindings/_helpers/sql_test_glue/mod.rs | 4 +- .../_helpers/sql_test_glue/plan_assembly.rs | 72 +++-- .../_helpers/sql_test_glue/plan_driver.rs | 2 + .../_helpers/sql_test_glue/plan_entries.rs | 29 +- .../_helpers/sql_test_glue/plan_fixtures.rs | 27 +- src/sqlbuild/_native.pyi | 129 +------- .../planner/_helpers/fixtures/helpers.py | 11 + .../_helpers/fixtures/test_type_scope.py | 42 +-- .../planner/_helpers/test_fixture_planning.py | 4 +- .../_helpers/test_fixture_planning_work.py | 26 +- 39 files changed, 1011 insertions(+), 1074 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs create mode 100644 tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index e6995f179a..703a4defa1 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -58,14 +58,14 @@ pub struct FixtureStarRelation { } /// One upstream typed column identified by its resource and folded name. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct FixtureTypeRead { pub key: (String, String), pub column: String, } /// Direct lineage facts used to propagate fixture types. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct FixtureTypeProjection { pub output_column: String, pub upstream: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs index ff98e042bf..41b25aaa3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs @@ -17,7 +17,7 @@ pub(crate) struct FixtureColumnInferences { #[pymethods] impl FixtureColumnInferences { #[new] - fn new() -> Self { + pub(crate) fn new() -> Self { Self { by_profile: Mutex::new(Vec::new()), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index a380d3b868..a47e075b37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::planner_phases::identities::{ build_planner_identity_context, detect_stale_warning_changes, }; use crate::bindings::_helpers::planner_phases::output_assembly::{ - assemble_base_plan_output, with_plan_metadata, with_plan_warnings, with_storage_policies, + add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; @@ -192,54 +192,19 @@ fn planned_output<'py>( )?), )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); - let mut plan_output = assemble_base_plan_output( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation), - ("entries", entries), - ("source_freshness", source_freshness.clone()), - ("planned_sql_tests", planned_sql_tests), - ], - )?), - )?; - plan_output = with_storage_policies( - py, - Some(keywords( - py, - &[ - ("plan_output", plan_output), - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scopes", scopes.clone()), - ("policies", policies.clone()), - ], - )?), + let output = base_plan_output( + (runtime, &warehouse, &identities, overrides), + (&pruning, &reconciliation, &entries, &source_freshness), + &planned_sql_tests, )?; - plan_output = with_plan_warnings( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("warehouse", warehouse), - ("identities", identities), - ("stale_warning_changes", stale_warning_changes), - ("pruning", pruning.clone()), - ("source_freshness", source_freshness.clone()), - ("plan_output", plan_output), - ("policies", policies.clone()), - ], - )?), + let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; + let output = add_plan_warnings( + output, + (scopes, &warehouse, &identities, &stale_warning_changes), + (&pruning, &source_freshness, &policies), )?; - plan_output = with_plan_metadata(&plan_output, &pruning, &source_freshness, &policies)?; + let plan_output = + add_plan_metadata(output, &pruning, &source_freshness, &policies)?.build(py)?; let on_progress = args.value("on_progress")?; if !on_progress.is_none() { on_progress.call1((format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 4ef09d1d75..8ae3efa1f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -5,7 +5,7 @@ mod command_plans; mod display_plan; mod execution_plan; mod model_entries; -mod plan_output; +pub(crate) mod plan_output; pub(crate) mod registration; pub(crate) mod relations; mod test_planning; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 7e21078603..87d8d2f91b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,12 +1,15 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -474,28 +477,26 @@ pub(super) fn selected_test_entries<'py>( } let analysis = project.getattr("settings")?.getattr("sql_analysis")?; let fixture_context = if analysis.is_truthy()? { - native( - py, - NativeAttr::BuildRelationFixtureContext, - &[("project", project.clone())], - )? + Bound::new(py, planning_context(project, None)?)?.into_any() } else { none(py) }; - let results = native( + let results = assemble_sql_test_plans( py, - NativeAttr::PlanSqlTests, - &[ - ("tests", tuple_of(py, selected)?), - ("project", project.clone()), - ("adapter", adapter.clone()), - ("sql_analysis_enabled", analysis), - ( - "validate_fixtures", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("fixture_planning_context", fixture_context), - ], + Some(keywords( + py, + &[ + ("tests", tuple_of(py, selected)?), + ("project", project.clone()), + ("adapter", adapter.clone()), + ("sql_analysis_enabled", analysis), + ( + "validate_fixtures", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("fixture_planning_context", fixture_context), + ], + )?), )?; let mut diagnostics: Vec = Vec::new(); for result in results.try_iter()? { @@ -544,7 +545,10 @@ fn planned_sql_tests<'py>( ) } -fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> { +/// The plan output's fields from planned entries and resolved relation inputs. +pub(crate) fn plan_output_fields<'py>( + args: &KeywordArguments<'py>, +) -> PyResult)>> { let project = args.value("project")?; let py = project.py(); let adapter = args.value("adapter")?; @@ -627,65 +631,59 @@ fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> for warning in planned.getattr("warnings")?.try_iter()? { warnings.push(warning?); } - planner_model( - py, - RecordId::PlanOutput, - &[ - ("execution_order", tuple_of(py, scoped_order)?), - ("model_entries", model_entries.clone()), - ("seed_entries", tuple_of(py, seeds)?), - ( - "source_load_entries", - PyTuple::new( - py, - source_load_entries - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ("function_entries", tuple_of(py, functions)?), - ( - "audit_entries", - PyTuple::new(py, audit_entries.try_iter()?.collect::>>()?)? - .into_any(), - ), - ( - "test_entries", - PyTuple::new(py, test_entries.try_iter()?.collect::>>()?)? - .into_any(), - ), - ("selected_keys", selected_keys), - ("warnings", tuple_of(py, warnings)?), - ("upstream_deps", scope.getattr("upstream_deps")?), - ("downstream_deps", scope.getattr("downstream_deps")?), - ("model_locations", relations.getattr("model_locations")?), - ("seed_locations", relations.getattr("seed_locations")?), - ( - "function_locations", - relations.getattr("function_locations")?, - ), - ("source_map", source_map), - ("source_read_map", relations.getattr("source_read_map")?), - ( - "python_source_read_map", - relations.getattr("python_source_read_map")?, - ), - ("hook_functions", project.getattr("hook_functions")?), - ( - "enforce_explicit_references", - project.getattr("enforce_explicit_references")?, - ), - ( - "provider_usages", - plan_provider_usages(&project, &model_entries, &source_load_entries)?, - ), - ( - "python_identity_fingerprints", - snapshot.getattr("fingerprints")?.getattr("python_nodes")?, - ), - ], - ) + Ok(vec![ + ("execution_order", tuple_of(py, scoped_order)?), + ("model_entries", model_entries.clone()), + ("seed_entries", tuple_of(py, seeds)?), + ( + "source_load_entries", + PyTuple::new( + py, + source_load_entries + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ("function_entries", tuple_of(py, functions)?), + ( + "audit_entries", + PyTuple::new(py, audit_entries.try_iter()?.collect::>>()?)?.into_any(), + ), + ( + "test_entries", + PyTuple::new(py, test_entries.try_iter()?.collect::>>()?)?.into_any(), + ), + ("selected_keys", selected_keys), + ("warnings", tuple_of(py, warnings)?), + ("upstream_deps", scope.getattr("upstream_deps")?), + ("downstream_deps", scope.getattr("downstream_deps")?), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", source_map), + ("source_read_map", relations.getattr("source_read_map")?), + ( + "python_source_read_map", + relations.getattr("python_source_read_map")?, + ), + ("hook_functions", project.getattr("hook_functions")?), + ( + "enforce_explicit_references", + project.getattr("enforce_explicit_references")?, + ), + ( + "provider_usages", + plan_provider_usages(&project, &model_entries, &source_load_entries)?, + ), + ( + "python_identity_fingerprints", + snapshot.getattr("fingerprints")?.getattr("python_nodes")?, + ), + ]) } /// Assemble the plan output from planned entries and resolved relation inputs. @@ -699,21 +697,9 @@ fn build_plan_output<'py>( py, "build_plan_output", kwargs, - ( - &[ - "project", - "adapter", - "scope", - "snapshot", - "relations", - "changes", - "model_entry_results", - "reload_sources", - ], - &["extras"], - ), + (PLAN_OUTPUT_INPUTS, &["extras"]), )?; - plan_output(&args) + planner_model(py, RecordId::PlanOutput, &plan_output_fields(&args)?) } /// Build one canonical function entry from resolved relation inputs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index 5c45d15791..ac9c86d04a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -12,3 +12,14 @@ pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; pub(crate) const RETENTION_DECREASE: &str = "decrease"; pub(crate) const RETENTION_INCREASE: &str = "increase"; pub(crate) const RETENTION_MATCH: &str = "match"; +/// The required inputs of plan output assembly. +pub(crate) const PLAN_OUTPUT_INPUTS: &[&str] = &[ + "project", + "adapter", + "scope", + "snapshot", + "relations", + "changes", + "model_entry_results", + "reload_sources", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index 32482259eb..e750afe4fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub(crate) mod execution_changes; pub(crate) mod full_refresh; pub(crate) mod identities; +pub(crate) mod models; pub(crate) mod objects; pub(crate) mod output_assembly; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs new file mode 100644 index 0000000000..0fb08904de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -0,0 +1,10 @@ +//! Plan output fields gathered across the plan phases before the output is built once. + +use pyo3::prelude::{Bound, PyAny}; + +/// The plan output's fields and metadata entries, each set before the output is built. +#[derive(Debug)] +pub(crate) struct PlanOutputFields<'py> { + pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, + pub(crate) metadata: Vec<(&'static str, Bound<'py, PyAny>)>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index a2a54769be..3bf915e1a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,8 +1,12 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; +use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; +use crate::bindings::_helpers::planner_phases::models::PlanOutputFields; use crate::bindings::_helpers::planner_phases::objects::{ model_key_names, native, replace, sorted_names, strings, text, tuple, }; @@ -11,7 +15,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use std::collections::BTreeSet; @@ -25,57 +28,51 @@ fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult( - plan_output: &Bound<'py, PyAny>, - items: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - let py = plan_output.py(); - let metadata = PyDict::new(py); - metadata.update( - plan_output - .getattr("metadata")? - .downcast::()?, - )?; - for (name, value) in items { - metadata.set_item(*name, value)?; +impl<'py> PlanOutputFields<'py> { + fn field(&self, name: &str) -> PyResult> { + self.fields + .iter() + .find(|(field, _)| *field == name) + .map(|(_, value)| value.clone()) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(name.to_owned())) } - replace(plan_output, &[("metadata", metadata.into_any())]) -} -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn assemble_base_plan_output<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "assemble_base_plan_output", - options, - ( - &[ - "runtime", - "warehouse", - "identities", - "overrides", - "pruning", - "reconciliation", - "entries", - "source_freshness", - ], - &["planned_sql_tests"], - ), - )?; - let runtime = &args.value("runtime")?; - let warehouse = &args.value("warehouse")?; - let identities = &args.value("identities")?; - let overrides = &args.value("overrides")?; - let pruning = &args.value("pruning")?; - let reconciliation = &args.value("reconciliation")?; - let entries = &args.value("entries")?; - let source_freshness = &args.value("source_freshness")?; - let planned_sql_tests = &args.value("planned_sql_tests")?; + fn set(&mut self, name: &'static str, value: Bound<'py, PyAny>) { + match self.fields.iter_mut().find(|(field, _)| *field == name) { + Some(existing) => existing.1 = value, + None => self.fields.push((name, value)), + } + } + + /// The `PlanOutput` hand-off, built once from every field the phases set. + pub(crate) fn build(mut self, py: Python<'py>) -> PyResult> { + if !self.metadata.is_empty() { + let metadata = PyDict::new(py); + for (name, value) in &self.metadata { + metadata.set_item(*name, value)?; + } + self.set("metadata", metadata.into_any()); + } + call_with(®istry::class(py, RecordId::PlanOutput)?, &self.fields) + } +} +/// The plan output's base fields from the planned entries, relations and source freshness. +pub(crate) fn base_plan_output<'py>( + (runtime, warehouse, identities, overrides): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + (pruning, reconciliation, entries, source_freshness): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + planned_sql_tests: &Bound<'py, PyAny>, +) -> PyResult> { let py = runtime.py(); let version = identities.getattr("version_identities")?; let extras = native( @@ -93,8 +90,8 @@ pub(crate) fn assemble_base_plan_output<'py>( ("planned_sql_tests", planned_sql_tests.clone()), ], )?; - let plan_output = call_with( - ®istry::native(py, NativeAttr::BuildPlanOutput)?, + let inputs = keywords( + py, &[ ("project", runtime.getattr("project")?), ("adapter", runtime.getattr("adapter")?), @@ -110,18 +107,22 @@ pub(crate) fn assemble_base_plan_output<'py>( ("extras", extras), ], )?; - let plan_output = replace( - &plan_output, - &[ - ("source_freshness", source_freshness.clone()), - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ], - )?; + let mut output = PlanOutputFields { + fields: plan_output_fields(&KeywordArguments::read( + py, + "build_plan_output", + Some(inputs), + (PLAN_OUTPUT_INPUTS, &["extras"]), + )?)?, + metadata: Vec::new(), + }; + output.set("source_freshness", source_freshness.clone()); + output.set("cursor_overrides", overrides.getattr("cursor_overrides")?); let pruned = pruning.getattr("pruned_direct_model_names")?; if pruned.is_truthy()? { - return with_metadata(&plan_output, &[("direct_pruned_model_names", pruned)]); + output.metadata.push(("direct_pruned_model_names", pruned)); } - Ok(plan_output) + Ok(output) } /// Attach near-match column rename hints to the model entries they describe. @@ -159,29 +160,18 @@ pub(crate) fn with_column_rename_hints<'py>( tuple(py, attached) } -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn with_storage_policies<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "with_storage_policies", - options, - ( - &["plan_output", "runtime", "warehouse", "scopes", "policies"], - &[], - ), - )?; - let plan_output = &args.value("plan_output")?; - let runtime = &args.value("runtime")?; - let warehouse = &args.value("warehouse")?; - let scopes = &args.value("scopes")?; - let policies = &args.value("policies")?; - +/// Add migration fingerprints, rename hints, migration, table-type and retention entries. +pub(crate) fn add_storage_policies<'py>( + mut output: PlanOutputFields<'py>, + (runtime, warehouse, scopes, policies): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { let py = runtime.py(); - let mut entries = plan_output.getattr("model_entries")?; + let mut entries = output.field("model_entries")?; if policies .getattr("record_migration_fingerprints")? .is_truthy()? @@ -216,7 +206,7 @@ pub(crate) fn with_storage_policies<'py>( py, &[ warehouse.getattr("migration_warnings")?, - plan_output.getattr("warnings")?, + output.field("warnings")?, ], )?; let selected = scopes.getattr("selected_scope")?; @@ -227,65 +217,40 @@ pub(crate) fn with_storage_policies<'py>( ]; let table_types = native(py, NativeAttr::PlanTableTypes, &storage)?; let retention = native(py, NativeAttr::PlanRetention, &storage)?; - replace( - plan_output, - &[ - ("model_entries", entries), - ("migration_entries", migration_entries), - ("column_migration_entries", column_entries), - ("old_name_view_entries", old_name_entries), - ("old_name_views", old_name_views), - ("warnings", warnings), - ("table_type_entries", table_types), - ("retention_entries", retention), - ], - ) + output.set("model_entries", entries); + output.set("migration_entries", migration_entries); + output.set("column_migration_entries", column_entries); + output.set("old_name_view_entries", old_name_entries); + output.set("old_name_views", old_name_views); + output.set("warnings", warnings); + output.set("table_type_entries", table_types); + output.set("retention_entries", retention); + Ok(output) } -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn with_plan_warnings<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "with_plan_warnings", - options, - ( - &[ - "runtime", - "scopes", - "warehouse", - "identities", - "stale_warning_changes", - "pruning", - "source_freshness", - "plan_output", - "policies", - ], - &[], - ), - )?; - let runtime = &args.value("runtime")?; - let scopes = &args.value("scopes")?; - let warehouse = &args.value("warehouse")?; - let identities = &args.value("identities")?; - let stale_warning_changes = &args.value("stale_warning_changes")?; - let pruning = &args.value("pruning")?; - let source_freshness = &args.value("source_freshness")?; - let plan_output = &args.value("plan_output")?; - let policies = &args.value("policies")?; - - let py = runtime.py(); - let mut plan_output = plan_output.clone(); +/// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. +pub(crate) fn add_plan_warnings<'py>( + mut output: PlanOutputFields<'py>, + (scopes, warehouse, identities, stale_warning_changes): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + (pruning, source_freshness, policies): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py = scopes.py(); let freshness = source_freshness_unknown_warnings(source_freshness)?; if freshness.is_truthy()? { - let warnings = concat(py, &[plan_output.getattr("warnings")?, freshness])?; - plan_output = replace(&plan_output, &[("warnings", warnings)])?; + let warnings = concat(py, &[output.field("warnings")?, freshness])?; + output.set("warnings", warnings); } if !policies.getattr("selection_diagnostics")?.is_truthy()? { - return Ok(plan_output); + return Ok(output); } let stale = native( py, @@ -311,10 +276,10 @@ pub(crate) fn with_plan_warnings<'py>( ], )?; if stale.is_truthy()? { - let warnings = concat(py, &[plan_output.getattr("warnings")?, stale])?; - plan_output = replace(&plan_output, &[("warnings", warnings)])?; + let warnings = concat(py, &[output.field("warnings")?, stale])?; + output.set("warnings", warnings); } - Ok(plan_output) + Ok(output) } fn source_names<'py>(items: &Bound<'py, PyAny>) -> PyResult> { @@ -387,15 +352,14 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( - plan_output: &Bound<'py, PyAny>, +/// Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. +pub(crate) fn add_plan_metadata<'py>( + mut output: PlanOutputFields<'py>, pruning: &Bound<'py, PyAny>, source_freshness: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, -) -> PyResult> { - let py = plan_output.py(); +) -> PyResult> { + let py = pruning.py(); let selected: BTreeSet = model_key_names( &pruning .getattr("inspection_scope")? @@ -412,18 +376,16 @@ pub(crate) fn with_plan_metadata<'py>( let diagnostics = PyDict::new(py); diagnostics.set_item("mode", text(py, "direct"))?; diagnostics.set_item("enabled", policies.getattr("selection_diagnostics")?)?; - with_metadata( - plan_output, - &[ - ( - "direct_source_freshness", - freshness_metadata(source_freshness)?, - ), - ( - "direct_remaining_stale_model_names", - sorted_names(py, remaining)?, - ), - ("selection_diagnostics", diagnostics.into_any()), - ], - ) + output.metadata.push(( + "direct_source_freshness", + freshness_metadata(source_freshness)?, + )); + output.metadata.push(( + "direct_remaining_stale_model_names", + sorted_names(py, remaining)?, + )); + output + .metadata + .push(("selection_diagnostics", diagnostics.into_any())); + Ok(output) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index fe50f5e71e..66fc997a07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -61,26 +61,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(super::retention::plan_retention, module)?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::assemble_base_plan_output, - module - )?)?; module.add_function(wrap_pyfunction!( super::output_assembly::with_column_rename_hints, module )?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::with_storage_policies, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::with_plan_warnings, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::with_plan_metadata, - module - )?)?; module.add_function(wrap_pyfunction!( super::warehouse_state::gather_planner_warehouse_state, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 6bd7cb4f40..a77e1e89d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1013,37 +1013,6 @@ const R_RELATION_FIXTURE_COMPLETION: RecordSpec = RecordSpec { FieldSpec::required("diagnostics"), ], }; -const R_FIXTURE_COLUMN_METADATA: RecordSpec = RecordSpec { - name: "FixtureColumnMetadata", - doc: "Known type and nullability for one relation fixture column.", - fields: &[ - FieldSpec::required("name"), - FieldSpec::required("type"), - FieldSpec::required("nullable"), - ], -}; -const R_FIXTURE_RELATION_METADATA: RecordSpec = RecordSpec { - name: "FixtureRelationMetadata", - doc: "Known columns and whether they form an authoritative relation shape.", - fields: &[ - FieldSpec::required("columns"), - FieldSpec::required("authoritative_names"), - ], -}; -const R_RELATION_FIXTURE_PLANNING_CONTEXT: RecordSpec = RecordSpec { - name: "RelationFixturePlanningContext", - doc: "Project-wide fixture metadata shared across test and scenario planning.", - fields: &[ - FieldSpec::required("models_by_name"), - FieldSpec::required("relations"), - FieldSpec::required("authoritative_columns"), - FieldSpec::required("expected_types"), - FieldSpec::uncompared_shown( - "fixture_inferences", - FieldDefault::NativeClassCall(NativeAttr::FixtureColumnInferences), - ), - ], -}; const R_SELECTOR_EXPANSION: RecordSpec = RecordSpec { name: "SelectorExpansion", doc: "One selector split into graph expansion flags and core text.", @@ -1579,9 +1548,6 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_NATIVE_SQL_TEST_ARTIFACT, R_RELATION_FIXTURE_DIAGNOSTIC, R_RELATION_FIXTURE_COMPLETION, - R_FIXTURE_COLUMN_METADATA, - R_FIXTURE_RELATION_METADATA, - R_RELATION_FIXTURE_PLANNING_CONTEXT, R_SELECTOR_EXPANSION, R_PARSED_SCENARIO_ARTIFACT_NAME, R_MISSING_UPSTREAM, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index e6c452e83f..1303f5e260 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -13,6 +13,8 @@ use crate::bindings::_helpers::planner_scenario::{ sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, values::{error, string}, }; +use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures as complete_relation_fixtures; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; @@ -208,31 +210,22 @@ fn complete_fixtures<'py>( .iter() .map(|key| key.getattr("name")) .collect::>>()?; - let completion = call_with( - ®istry::native(py, NativeAttr::BuildRelationFixtureCompletion)?, - &[ - ("project", project.clone()), - ("adapter", context.adapter.clone()), - ( - "request", - PyTuple::new( - py, - [ - PyTuple::new(py, names)?.into_any(), - PyTuple::new(py, groups)?.into_any(), - ], - )? - .into_any(), - ), - ], + let names = names + .iter() + .map(|name| name.extract()) + .collect::>>()?; + let planning = planning_context(project, None)?; + let completion = complete_relation_fixtures( + &planning, + &context.adapter, + (&names, PyTuple::new(py, groups)?.as_any()), )?; - let diagnostics = completion.getattr("diagnostics")?; - if diagnostics.is_truthy()? { + if !completion.diagnostics.is_empty() { let mut details: Vec = Vec::new(); - for diagnostic in diagnostics.try_iter()? { + for diagnostic in &completion.diagnostics { details.push(format!( "- {}", - diagnostic?.getattr("message")?.extract::()? + diagnostic.getattr("message")?.extract::()? )); } let name: String = scenario.getattr("name")?.extract()?; @@ -247,7 +240,7 @@ fn complete_fixtures<'py>( None, )?); } - completion.getattr("fixture_sql_by_key") + Ok(completion.sql_by_key.into_any()) } fn model_context<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index aa811c5350..a2bc231951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -17,3 +17,7 @@ pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; pub(crate) const HIGH_CONFIDENCE: &str = "high"; /// The lineage transform kind that passes an input column's type through. pub(crate) const DIRECT_TRANSFORM: &str = "direct"; +/// Resource type values keying fixture relations. +pub(crate) const MODEL_REFERENCE_KIND: &str = "model"; +pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; +pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs index f614f02137..7d034d3faf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -1,12 +1,14 @@ -//! Native-hosted relation fixture completion over Python boundary models. +//! Relation fixture completion over the Rust fixture planning context. + +use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; -use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, RequiredColumn}; +use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, FixtureColumn}; -use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context as project_planning_context; use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; use crate::bindings::_helpers::planner_strategy::fixture_reads::required_fixture_columns; use crate::bindings::_helpers::planner_strategy::fixture_selection::{ @@ -15,190 +17,217 @@ use crate::bindings::_helpers::planner_strategy::fixture_selection::{ use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; use crate::bindings::_helpers::planner_strategy::fixture_values::{ - columns, diagnostic, folded, label, record, strings, + columns, diagnostic, label, record, +}; +use crate::bindings::_helpers::planner_strategy::models::{ + ExpectedTypes, RelationFixturePlanningContext, }; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::records::ids::RecordId; -#[pyfunction] -#[pyo3(signature = (*, project, adapter, request, planning_context = None))] -pub(crate) fn build_relation_fixture_completion<'py>( - project: &Bound<'py, PyAny>, +/// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. +pub(crate) struct FixtureCompletion<'py, 'a> { + pub(crate) sql_by_key: Bound<'py, PyDict>, + pub(crate) inferred_by_fixture: Bound<'py, PyDict>, + pub(crate) expected_types: ExpectedTypes<'a>, + pub(crate) diagnostics: Vec>, +} + +impl ExpectedTypes<'_> { + /// The relation's expected column types: the chain's propagated types, else the context's. + pub(crate) fn get(&self, key: &FixtureKey) -> Option<&[(String, String)]> { + self.overlay + .iter() + .find(|(known, _)| known == key) + .map(|(_, types)| types.as_slice()) + .or_else(|| self.base.get(key).map(Vec::as_slice)) + } +} + +/// Complete the fixture groups of one chain against the planning context. +pub(crate) fn complete_fixtures<'py, 'a>( + context: &'a RelationFixturePlanningContext, adapter: &Bound<'py, PyAny>, - request: (Bound<'py, PyAny>, Bound<'py, PyAny>), - planning_context: Option>, -) -> PyResult> { - let (ordered_model_names, fixture_groups) = request; - let context = match planning_context { - Some(context) if context.is_truthy()? => context, - _ => build_relation_fixture_context(project, None)?, - }; - let relations = context.getattr("relations")?.cast_into::()?; - let prepared = prepare( - &fixture_groups, - &relations, - adapter, - &context.getattr("fixture_inferences")?, - )?; - let fixture_keys = fixture_keys_requiring_analysis( - &prepared.inferred, - &context - .getattr("authoritative_columns")? - .cast_into::()?, - )?; - let (required, stars) = required_fixture_columns( - ( - ordered_model_names.extract()?, - context.getattr("models_by_name")?.cast_into::()?, - ), - ( - fixture_keys.into_any(), - relations.clone(), - context - .getattr("authoritative_columns")? - .cast_into::()?, - ), - adapter, - )?; - let expected = expected_fixture_types( - &context.getattr("expected_types")?.cast_into::()?, - ordered_model_names.extract()?, - &context.getattr("models_by_name")?.cast_into::()?, - )?; - complete_prepared( - prepared, - (relations, required, stars.into_any()), - expected.into_any(), - adapter, - ) + (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), +) -> PyResult> { + let py = adapter.py(); + let prepared = prepare(fixture_groups, context, adapter)?; + let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; + let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; + let expected = expected_fixture_types(context, py, ordered_model_names)?; + complete_prepared(prepared, (context, &required), expected, adapter) } -fn complete_prepared<'py>( +fn complete_prepared<'py, 'a>( mut prepared: PreparedFixtures<'py>, - plan: (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyAny>), - expected: Bound<'py, PyAny>, + (context, required): ( + &'a RelationFixturePlanningContext, + &sqlbuild_analysis::planner_fixtures::models::FixtureReadRequirements, + ), + expected: ExpectedTypes<'a>, adapter: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py = adapter.py(); - let (relations, required, stars) = plan; + let python_keys: Vec<(FixtureKey, Bound<'py, PyAny>)> = prepared.keys.clone(); + let python_key = |key: &FixtureKey| -> Option<&Bound<'py, PyAny>> { + python_keys + .iter() + .find(|(known, _)| known == key) + .map(|(_, python)| python) + }; let typed_nulls = PyDict::new(py); - for (key, names) in prepared.null_names.iter() { - let types = expected.call_method1("get", (&key, PyDict::new(py)))?; + for (key, original) in &python_keys { + let Some(names) = prepared.null_names.get_item(original)? else { + continue; + }; + let types: &[(String, String)] = expected.get(key).unwrap_or_default(); let selected = PyDict::new(py); for name in names.try_iter()? { let name = name?; - let value = types.call_method1("get", (&name,))?; - if !value.is_none() { - selected.set_item(name, value)?; + let folded: String = name.extract()?; + if let Some((_, sql_type)) = types.iter().find(|(column, _)| *column == folded) { + selected.set_item(name, sql_type)?; } } if !selected.is_empty() { - typed_nulls.set_item(key, selected)?; + typed_nulls.set_item(original, selected)?; } } - for value in unknown_fixture_column_diagnostics(&prepared.inferred, &relations)?.try_iter()? { - prepared.diagnostics.push(value?); + for value in unknown_fixture_column_diagnostics(&prepared, context)? { + prepared.diagnostics.push(value); } - let completed_keys = PySet::empty(py)?; - for (key, required_columns) in required.iter() { - let Some(inferred) = prepared.inferred.get_item(&key)? else { + let mut completed_keys: HashSet = HashSet::new(); + for (key, requirements) in &required.columns { + let Some(original) = python_key(key) else { + continue; + }; + let Some(inferred) = prepared.inferred.get_item(original)? else { continue; }; let typed = typed_nulls - .get_item(&key)? + .get_item(original)? .map(|value| value.cast_into::()) .transpose()? .unwrap_or_else(|| PyDict::new(py)); - let metadata = match relations.get_item(&key)? { - Some(relation) => relation.getattr("columns")?, - None => PyTuple::empty(py).into_any(), - }; - let requirements = required_columns - .cast_into::()? - .iter() - .map(|(name, readers)| { - Ok(RequiredColumn { - name: name.extract()?, - folded_name: folded(&name)?, - readers: strings(&readers)?, - }) - }) - .collect::>>()?; + let metadata: &[FixtureColumn] = context + .relations + .get(key) + .map(|relation| relation.columns.as_slice()) + .unwrap_or_default(); let alignment = align_columns( - &label(&key)?, + &label(original)?, AlignmentInput { supplied: &columns(&inferred, true)?, - metadata: &columns(&metadata, false)?, - required: &requirements, - star: stars.contains(&key)?, + metadata, + required: requirements, + star: required.stars.contains(key), typed_nulls: !typed.is_empty(), }, ); for message in alignment.diagnostics { - prepared.diagnostics.push(diagnostic(&key, &message)?); + prepared.diagnostics.push(diagnostic(original, &message)?); } let Some(completed) = alignment.completed else { continue; }; - let completed = completed + let completed: Vec = completed .into_iter() - .map(|index| metadata.get_item(index)) - .collect::>>()?; - let sql: String = prepared - .sql - .get_item(&key)? - .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? - .extract()?; - let quoted = prepared - .quoted_names - .get_item(&key)? - .unwrap_or(PyFrozenSet::empty(py)?.into_any()); + .map(|index| metadata[index].clone()) + .collect(); let sql = render_completed( CompletionSql { - sql: &sql, + sql: &fixture_sql(&prepared, original)?, inferred, typed_nulls: typed, - quoted_names: quoted, - completed: PyTuple::new(py, completed)?.into_any(), + quoted_names: quoted_names(&prepared, original)?, + completed, }, adapter, )?; - prepared.sql.set_item(&key, sql)?; - completed_keys.add(key)?; + prepared.sql.set_item(original, sql)?; + completed_keys.insert(key.clone()); } - for (key, typed) in typed_nulls.iter() { - if completed_keys.contains(&key)? { + for (key, original) in &python_keys { + let Some(typed) = typed_nulls.get_item(original)? else { + continue; + }; + if completed_keys.contains(key) { continue; } let inferred = prepared .inferred - .get_item(&key)? + .get_item(original)? .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture inference"))?; - let sql: String = prepared - .sql - .get_item(&key)? - .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? - .extract()?; - let quoted = prepared - .quoted_names - .get_item(&key)? - .unwrap_or(PyFrozenSet::empty(py)?.into_any()); let sql = render_completed( CompletionSql { - sql: &sql, + sql: &fixture_sql(&prepared, original)?, inferred, typed_nulls: typed.cast_into::()?, - quoted_names: quoted, - completed: PyTuple::empty(py).into_any(), + quoted_names: quoted_names(&prepared, original)?, + completed: Vec::new(), }, adapter, )?; - prepared.sql.set_item(key, sql)?; + prepared.sql.set_item(original, sql)?; + } + Ok(FixtureCompletion { + sql_by_key: prepared.sql, + inferred_by_fixture: prepared.inferred, + expected_types: expected, + diagnostics: prepared.diagnostics, + }) +} + +fn fixture_sql(prepared: &PreparedFixtures<'_>, key: &Bound<'_, PyAny>) -> PyResult { + prepared + .sql + .get_item(key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? + .extract() +} + +fn quoted_names<'py>( + prepared: &PreparedFixtures<'py>, + key: &Bound<'py, PyAny>, +) -> PyResult> { + Ok(prepared + .quoted_names + .get_item(key)? + .unwrap_or(PyFrozenSet::empty(key.py())?.into_any())) +} + +/// Complete one chain's fixture groups, building the planning context when none is given. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, request, planning_context = None))] +fn build_relation_fixture_completion<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + request: (Vec, Bound<'py, PyAny>), + planning_context: Option>, +) -> PyResult> { + let py = project.py(); + let built: RelationFixturePlanningContext; + let context: &RelationFixturePlanningContext = match &planning_context { + Some(context) => context.get(), + None => { + built = project_planning_context(project, None)?; + &built + } + }; + let completion = complete_fixtures(context, adapter, (&request.0, &request.1))?; + let expected = PyDict::new(py); + for (key, types) in &completion.expected_types.overlay { + let columns = PyDict::new(py); + for (name, sql_type) in types { + columns.set_item(name, sql_type)?; + } + expected.set_item(key, columns)?; } let kwargs = PyDict::new(py); - kwargs.set_item("fixture_sql_by_key", prepared.sql)?; - kwargs.set_item("inferred_by_fixture", prepared.inferred)?; + kwargs.set_item("fixture_sql_by_key", completion.sql_by_key)?; + kwargs.set_item("inferred_by_fixture", completion.inferred_by_fixture)?; kwargs.set_item("expected_types", expected)?; - kwargs.set_item("diagnostics", PyTuple::new(py, prepared.diagnostics)?)?; + kwargs.set_item("diagnostics", PyTuple::new(py, completion.diagnostics)?)?; record(py, RecordId::RelationFixtureCompletion, &kwargs) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 33f67645b7..e259839244 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -1,22 +1,48 @@ -//! Project fixture context materialization and schema conversions. +//! The project-wide fixture planning context: relation metadata and per-model facts as Rust data. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; +use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_metadata; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata, record}; -use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; +use crate::bindings::_helpers::planner_strategy::constants::{ + MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, +}; +use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_relation; +use crate::bindings::_helpers::planner_strategy::fixture_reads::lineage_reads; +use crate::bindings::_helpers::planner_strategy::fixture_types::projections; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::{ + FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, +}; +use crate::bindings::_helpers::planner_strategy::types::{FixtureKey, FixtureRelationInput}; +use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::records::registry; -#[pyfunction] -#[pyo3(signature = (*, project, models_by_name = None))] -pub(crate) fn build_relation_fixture_context<'py>( - project: &Bound<'py, PyAny>, - models_by_name: Option>, -) -> PyResult> { +/// Count one batch resolution of SQL-test model chains. +pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { + work_counts(py)?.borrow_mut().chain_resolutions += 1; + Ok(()) +} + +fn work_counts(py: Python<'_>) -> PyResult> { + Ok(registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?) +} + +/// The planning context of a project, over its models or the given name-to-model map. +pub(crate) fn planning_context( + project: &Bound<'_, PyAny>, + models_by_name: Option>, +) -> PyResult { let py = project.py(); + work_counts(py)?.borrow_mut().context_builds += 1; let model_map = match models_by_name { Some(models) => models, None => { @@ -28,92 +54,199 @@ pub(crate) fn build_relation_fixture_context<'py>( models } }; - let relations = relation_metadata(project, &model_map)?; - let kwargs = PyDict::new(py); - kwargs.set_item("models_by_name", model_map)?; - kwargs.set_item( - "authoritative_columns", - authoritative_fixture_columns(&relations)?, - )?; - kwargs.set_item("expected_types", base_expected_fixture_types(&relations)?)?; - kwargs.set_item("relations", relations)?; - record(py, RecordId::RelationFixturePlanningContext, &kwargs) -} - -fn relation_metadata<'py>( - project: &Bound<'py, PyAny>, - models: &Bound<'py, PyDict>, -) -> PyResult> { - let py = project.py(); - let kinds = py - .import("sqlbuild.compiler.compile.types")? - .getattr("CompiledResourceType")?; - let relations = PyDict::new(py); - for (name, model) in models.iter() { - relations.set_item( - (kinds.getattr("MODEL")?, name), - model_fixture_metadata(&model)?, - )?; + let mut relations: HashMap = HashMap::new(); + for (name, model) in model_map.iter() { + relations.insert( + (MODEL_REFERENCE_KIND.to_owned(), name.extract()?), + model_fixture_relation(&model)?, + ); } for (collection, entry, kind, source) in [ - ("sources", "source_entry", "SOURCE", true), - ("seeds", "schema_entry", "SEED", false), + ("sources", "source_entry", SOURCE_REFERENCE_KIND, true), + ("seeds", "schema_entry", SEED_REFERENCE_KIND, false), ] { for resource in project.getattr(collection)?.try_iter()? { let resource = resource?; let entry = resource.getattr(entry)?; - let columns = columns(&entry.getattr("columns")?, false)?; + let columns: Vec = columns(&entry.getattr("columns")?, false)?; let authoritative = if source { entry.getattr("contract")?.eq("enforced")? } else { !columns.is_empty() }; - relations.set_item( - (kinds.getattr(kind)?, resource.getattr("name")?), - metadata(py, &columns, authoritative)?, - )?; + relations.insert( + (kind.to_owned(), resource.getattr("name")?.extract()?), + FixtureRelation { + columns, + authoritative, + }, + ); } } - Ok(relations) + Ok(context_of( + model_map.unbind(), + relations, + Py::new(py, FixtureColumnInferences::new())?.into_any(), + )) } -#[pyfunction] -#[pyo3(signature = (*, relations))] -pub(crate) fn authoritative_fixture_columns<'py>( - relations: &Bound<'py, PyDict>, -) -> PyResult> { - let result = PyDict::new(relations.py()); - for (key, relation) in relations.iter() { - let schema = fixture_schema( - &columns(&relation.getattr("columns")?, false)?, - relation.getattr("authoritative_names")?.extract()?, +fn context_of( + models_by_name: Py, + relations: HashMap, + fixture_inferences: Py, +) -> RelationFixturePlanningContext { + let mut authoritative_columns: HashMap> = HashMap::new(); + let mut expected_types: HashMap> = HashMap::new(); + for (key, relation) in &relations { + if let Some(names) = + fixture_schema(&relation.columns, relation.authoritative).authoritative_names + { + authoritative_columns.insert(key.clone(), names); + } + expected_types.insert( + key.clone(), + fixture_schema(&relation.columns, false).expected_types, ); - if let Some(names) = schema.authoritative_names { - result.set_item(key, PyFrozenSet::new(relations.py(), names)?)?; + } + RelationFixturePlanningContext { + models_by_name, + relations, + authoritative_columns, + expected_types, + fixture_inferences, + model_facts: Mutex::new(HashMap::new()), + } +} + +impl RelationFixturePlanningContext { + /// One model's lineage reads and type projections, derived once per context. + pub(crate) fn model_facts( + &self, + py: Python<'_>, + name: &str, + ) -> PyResult>> { + if let Some(found) = self.facts_cache()?.get(name) { + return Ok(Some(Arc::clone(found))); + } + let Some(model) = self.models_by_name.bind(py).get_item(name)? else { + return Ok(None); + }; + let facts = Arc::new(ModelFixtureFacts { + reads: lineage_reads(&model)?, + projections: projections(&model)?, + }); + self.facts_cache()? + .insert(name.to_owned(), Arc::clone(&facts)); + Ok(Some(facts)) + } + + fn facts_cache( + &self, + ) -> PyResult>>> { + self.model_facts + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture model facts")) + } +} + +#[pymethods] +impl RelationFixturePlanningContext { + /// A context over explicit relation metadata: `{key: ((name, type, nullable), ...), authoritative}`. + #[new] + #[pyo3(signature = (*, models_by_name, relations))] + fn new( + py: Python<'_>, + models_by_name: Bound<'_, PyDict>, + relations: HashMap, + ) -> PyResult { + let mut known: HashMap = HashMap::new(); + for (key, (columns, authoritative)) in relations { + let mut fixture_columns: Vec = Vec::with_capacity(columns.len()); + for (name, sql_type, nullable) in columns { + fixture_columns.push(FixtureColumn { + folded_name: python_casefold(&name), + name, + sql_type, + nullable, + }); + } + known.insert( + key, + FixtureRelation { + columns: fixture_columns, + authoritative, + }, + ); + } + Ok(context_of( + models_by_name.unbind(), + known, + Py::new(py, FixtureColumnInferences::new())?.into_any(), + )) + } + + /// Whether the relation's columns form an authoritative shape, `None` for unknown relations. + fn authoritative_names(&self, key: FixtureKey) -> Option { + self.relations + .get(&key) + .map(|relation| relation.authoritative) + } + + /// The relation's known column types by name, `None` for unknown relations. + fn expected_types<'py>( + &self, + py: Python<'py>, + key: FixtureKey, + ) -> PyResult>> { + let Some(types) = self.expected_types.get(&key) else { + return Ok(None); + }; + let result = PyDict::new(py); + for (name, sql_type) in types { + result.set_item(name, sql_type)?; } + Ok(Some(result)) + } + + /// The relation's authoritative column names, `None` when the shape is not authoritative. + fn authoritative_columns<'py>( + &self, + py: Python<'py>, + key: FixtureKey, + ) -> PyResult>> { + self.authoritative_columns + .get(&key) + .map(|names| PyFrozenSet::new(py, names)) + .transpose() } - Ok(result) } +/// Build the project-wide fixture planning context once for many SQL-test plans. #[pyfunction] -#[pyo3(signature = (*, relations))] -pub(crate) fn base_expected_fixture_types<'py>( - relations: &Bound<'py, PyDict>, -) -> PyResult> { - let result = PyDict::new(relations.py()); - for (key, relation) in relations.iter() { - let schema = fixture_schema(&columns(&relation.getattr("columns")?, false)?, false); - let types = PyDict::new(relations.py()); - for (name, sql_type) in schema.expected_types { - types.set_item(name, sql_type)?; - } - result.set_item(key, types)?; +#[pyo3(signature = (*, project, models_by_name = None))] +fn build_relation_fixture_context( + project: &Bound<'_, PyAny>, + models_by_name: Option>, +) -> PyResult { + planning_context(project, models_by_name) +} + +/// Planning contexts built and SQL-test chain batches resolved, optionally resetting both. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64)> { + let host = work_counts(py)?; + let mut host = host.borrow_mut(); + let counts = (host.context_builds, host.chain_resolutions); + if reset { + host.context_builds = 0; + host.chain_resolutions = 0; } - Ok(result) + Ok(counts) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; - module.add_function(wrap_pyfunction!(authoritative_fixture_columns, module)?)?; - module.add_function(wrap_pyfunction!(base_expected_fixture_types, module)?) + module.add_function(wrap_pyfunction!(fixture_planning_work_info, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index c9aeb5bcf1..ac20594665 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -1,28 +1,23 @@ -//! Model fixture schema materialization at the Python boundary. +//! A model's fixture relation metadata from its inferred and declared columns. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::PyTuple; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata}; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; -#[pyfunction] -#[pyo3(signature = (*, model))] -pub(crate) fn model_fixture_metadata<'py>( - model: &Bound<'py, PyAny>, -) -> PyResult> { - let py = model.py(); +/// The model's merged inferred and declared columns, authoritative unless its analysis saw a star. +pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult { let inferred: Option> = inferred_rows(model)?; let schema = model.getattr("schema_entry")?; - let declared = if schema.is_none() { - PyTuple::empty(py).into_any() + let declared: Vec = if schema.is_none() { + Vec::new() } else { - schema.getattr("columns")? + columns(&schema.getattr("columns")?, false)? }; let mut inferred_columns: Vec = Vec::new(); for (name, sql_type, _) in inferred.iter().flatten() { @@ -33,7 +28,6 @@ pub(crate) fn model_fixture_metadata<'py>( nullable: None, }); } - let merged = merge_metadata(inferred_columns, columns(&declared, false)?); let authoritative = !star_flags(model)?.0 && (model .getattr("config")? @@ -41,9 +35,8 @@ pub(crate) fn model_fixture_metadata<'py>( .call_method1("get", ("contract",))? .eq("enforced")? || inferred.is_some()); - metadata(py, &merged, authoritative) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_fixture_metadata, module)?) + Ok(FixtureRelation { + columns: merge_metadata(inferred_columns, declared), + authoritative, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs index b0b1446345..ff108f28d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs @@ -7,8 +7,12 @@ use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixtu use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::fixture_sql::complete_empty_fixture_sql; use crate::bindings::_helpers::planner_strategy::fixture_values::{diagnostic, label}; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; pub(crate) struct PreparedFixtures<'py> { + /// Each fixture's key with its Python key, in request order. + pub(crate) keys: Vec<(FixtureKey, Bound<'py, PyAny>)>, pub(crate) sql: Bound<'py, PyDict>, pub(crate) inferred: Bound<'py, PyDict>, pub(crate) null_names: Bound<'py, PyDict>, @@ -18,12 +22,13 @@ pub(crate) struct PreparedFixtures<'py> { pub(crate) fn prepare<'py>( groups: &Bound<'py, PyAny>, - relations: &Bound<'py, PyDict>, + context: &RelationFixturePlanningContext, adapter: &Bound<'py, PyAny>, - inferences: &Bound<'py, PyAny>, ) -> PyResult> { let py = groups.py(); + let inferences = context.fixture_inferences.bind(py); let mut prepared = PreparedFixtures { + keys: Vec::new(), sql: PyDict::new(py), inferred: PyDict::new(py), null_names: PyDict::new(py), @@ -36,12 +41,12 @@ pub(crate) fn prepare<'py>( let fixtures = group.get_item(1)?.cast_into::()?; for (name, sql) in fixtures.iter() { let key = PyTuple::new(py, [&kind, &name])?.into_any(); + let fixture_key: FixtureKey = key.extract()?; + prepared.keys.push((fixture_key.clone(), key.clone())); let mut sql: String = sql.extract()?; if empty_fixture_marker(&sql).map_err(value_error)? { - let relation = relations - .get_item(&key)? - .unwrap_or_else(|| py.None().into_bound(py)); - let (empty_sql, error) = complete_empty_fixture_sql(&sql, &relation, adapter)?; + let (empty_sql, error) = + complete_empty_fixture_sql(&sql, context.relations.get(&fixture_key), adapter)?; if let Some(error) = error { prepared .diagnostics diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 4d1332cabd..21d982b07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -1,12 +1,13 @@ //! Boundary conversion for native fixture read analysis. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet}; +use pyo3::types::{PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ - FixtureReadKey, FixtureReadModel, FixtureReadSchemas, FixtureStarRelation, + FixtureReadKey, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, + FixtureStarRelation, }; use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; use sqlbuild_analysis::required_columns::models::ColumnRead; @@ -17,54 +18,48 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; -use crate::bindings::_helpers::planner_strategy::fixture_values::columns; -use crate::bindings::_helpers::planner_strategy::models::FixtureReadCacheHost; +use crate::bindings::_helpers::planner_strategy::models::{ + FixtureReadCacheHost, RelationFixturePlanningContext, +}; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use crate::bindings::types::CompilerDetach; -#[pyfunction] -#[pyo3(signature = (*, model_inputs, fixture_inputs, adapter))] -pub(crate) fn required_fixture_columns<'py>( - model_inputs: (Vec, Bound<'py, PyDict>), - fixture_inputs: (Bound<'py, PyAny>, Bound<'py, PyDict>, Bound<'py, PyDict>), - adapter: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyFrozenSet>)> { - let (ordered_model_names, model_map) = model_inputs; - let (fixture_keys, relations, authoritative_columns) = fixture_inputs; +/// The columns each fixture must supply for the chain's models, and the fixtures read through a star. +pub(crate) fn required_fixture_columns( + context: &RelationFixturePlanningContext, + ordered_model_names: &[String], + fixture_keys: &HashSet, + adapter: &Bound<'_, PyAny>, +) -> PyResult { let py = adapter.py(); - let mut keys: HashMap<(String, String), Bound<'py, PyAny>> = HashMap::new(); - for key in fixture_keys.try_iter()? { - let key = key?; - keys.insert(key.extract::<(String, String)>()?, key); - } let mut schema: Vec = Vec::new(); - let mut authoritative: HashMap<(String, String), HashSet> = HashMap::new(); - for (key, original) in &keys { - if let Some(value) = relations.get_item(original)? { - schema.push(FixtureStarRelation { - key: key.clone(), - columns: columns(&value.getattr("columns")?, false)?, - authoritative: value.getattr("authoritative_names")?.extract()?, - }); - } else { - schema.push(FixtureStarRelation { - key: key.clone(), - columns: Vec::new(), - authoritative: false, - }); - } - if let Some(value) = authoritative_columns.get_item(original)? { - authoritative.insert(key.clone(), value.extract()?); + let mut authoritative: HashMap> = HashMap::new(); + for key in fixture_keys { + let relation = context.relations.get(key); + schema.push(FixtureStarRelation { + key: key.clone(), + columns: relation + .map(|relation| relation.columns.clone()) + .unwrap_or_default(), + authoritative: relation.is_some_and(|relation| relation.authoritative), + }); + if let Some(names) = context.authoritative_columns.get(key) { + authoritative.insert(key.clone(), names.iter().cloned().collect()); } } let dialect = adapter .call_method0("expression_inference_profile")? .getattr("sql_analysis_dialect")? .extract::>()?; + let models_by_name = context.models_by_name.bind(py); let mut models: Vec = Vec::new(); for name in ordered_model_names { - let Some(model) = model_map.get_item(&name)? else { + let Some(model) = models_by_name.get_item(name)? else { + continue; + }; + let Some(facts) = context.model_facts(py, name)? else { continue; }; let mut references: Vec = Vec::new(); @@ -78,13 +73,13 @@ pub(crate) fn required_fixture_columns<'py>( )); } models.push(FixtureReadModel { - name, + name: name.clone(), key: FixtureReadKey { sql: model_reads::query_sql(&model)?.extract()?, references, dialect: dialect.clone(), }, - high_confidence: lineage_reads(&model)?, + high_confidence: facts.reads.clone(), stars: model_stars(&model, &schema, adapter)?, }); } @@ -92,13 +87,12 @@ pub(crate) fn required_fixture_columns<'py>( registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?; let mut owner = owner.borrow_mut(); let cache = std::mem::take(&mut owner.cache); - let fixture_keys: HashSet<(String, String)> = keys.keys().cloned().collect(); let (cache, result) = catch_compiler_panic(|| { Ok(required_columns( cache, &models, FixtureReadSchemas { - keys: &fixture_keys, + keys: fixture_keys, authoritative: &authoritative, relations: &schema, }, @@ -106,20 +100,10 @@ pub(crate) fn required_fixture_columns<'py>( }) .map_err(value_error)?; owner.cache = cache; - let result = result.map_err(|error| value_error(format!("{error:?}")))?; - let required = PyDict::new(py); - for (key, columns) in result.columns { - let values = PyDict::new(py); - for column in columns { - values.set_item(column.name, PySet::new(py, column.readers)?)?; - } - required.set_item(&keys[&key], values)?; - } - let stars = PyFrozenSet::new(py, result.stars.iter().map(|key| &keys[key]))?; - Ok((required, stars)) + result.map_err(|error| value_error(format!("{error:?}"))) } -fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let mut reads: Vec = Vec::new(); for (_, _, confidence, sources) in lineage_facts(model)?.unwrap_or_default() { if confidence != HIGH_CONFIDENCE { @@ -194,8 +178,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { "_fixture_read_cache", pyo3::Py::new(module.py(), FixtureReadCacheHost::default())?, )?; - module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?)?; - module.add_function(wrap_pyfunction!(required_fixture_columns, module)?) + module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) } /// Report resolver misses and retained entries, optionally clearing prior fixture reuse. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs index c21012b96e..3f25962c17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs @@ -1,66 +1,60 @@ -//! Fixture analysis selection and diagnostic boundary conversions. +//! Fixture analysis selection and unknown-column diagnostics over the planning context. -use std::collections::BTreeSet; +use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDictMethods; use sqlbuild_analysis::planner_fixtures::main::{ fixture_schema::fixture_schema, requires_analysis::requires_analysis, unknown_columns::unknown_columns, }; +use crate::bindings::_helpers::planner_strategy::fixture_preparation::PreparedFixtures; use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, diagnostic, label}; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -#[pyfunction] -#[pyo3(signature = (*, inferred_by_fixture, authoritative_columns))] -pub(crate) fn fixture_keys_requiring_analysis<'py>( - inferred_by_fixture: &Bound<'py, PyDict>, - authoritative_columns: &Bound<'py, PyDict>, -) -> PyResult> { - let mut selected: Vec> = Vec::new(); - for (key, inferred) in inferred_by_fixture.iter() { - let expected: Option> = authoritative_columns - .get_item(&key)? - .map(|names| names.extract()) - .transpose()?; - if requires_analysis(&columns(&inferred, true)?, expected.as_ref()) { - selected.push(key); +/// Fixtures whose inferred columns do not already match an authoritative relation shape. +pub(crate) fn fixture_keys_requiring_analysis( + prepared: &PreparedFixtures<'_>, + context: &RelationFixturePlanningContext, +) -> PyResult> { + let mut selected: HashSet = HashSet::new(); + for (key, python_key) in &prepared.keys { + let Some(inferred) = prepared.inferred.get_item(python_key)? else { + continue; + }; + if requires_analysis( + &columns(&inferred, true)?, + context.authoritative_columns.get(key), + ) { + selected.insert(key.clone()); } } - PyFrozenSet::new(inferred_by_fixture.py(), selected) + Ok(selected) } -#[pyfunction] -#[pyo3(signature = (*, inferred_by_fixture, relations))] +/// Diagnostics for fixtures that supply columns an authoritative relation does not have. pub(crate) fn unknown_fixture_column_diagnostics<'py>( - inferred_by_fixture: &Bound<'py, PyDict>, - relations: &Bound<'py, PyDict>, -) -> PyResult> { - let mut diagnostics = Vec::>::new(); - for (key, inferred) in inferred_by_fixture.iter() { - let Some(relation) = relations.get_item(&key)? else { + prepared: &PreparedFixtures<'py>, + context: &RelationFixturePlanningContext, +) -> PyResult>> { + let mut diagnostics: Vec> = Vec::new(); + for (key, python_key) in &prepared.keys { + let Some(inferred) = prepared.inferred.get_item(python_key)? else { continue; }; - let schema = fixture_schema( - &columns(&relation.getattr("columns")?, false)?, - relation.getattr("authoritative_names")?.extract()?, - ); + let Some(relation) = context.relations.get(key) else { + continue; + }; + let schema = fixture_schema(&relation.columns, relation.authoritative); if let Some(message) = unknown_columns( - &label(&key)?, + &label(python_key)?, &columns(&inferred, true)?, schema.authoritative_names.as_ref(), ) { - diagnostics.push(diagnostic(&key, &message)?); + diagnostics.push(diagnostic(python_key, &message)?); } } - PyTuple::new(inferred_by_fixture.py(), diagnostics) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_keys_requiring_analysis, module)?)?; - module.add_function(wrap_pyfunction!( - unknown_fixture_column_diagnostics, - module - )?) + Ok(diagnostics) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs index e5f4eb6767..7f8f327f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs @@ -7,38 +7,37 @@ use sqlbuild_analysis::planner_fixtures::main::{ empty_fixture_error::empty_fixture_error, empty_fixture_marker::empty_fixture_marker, render_completed::render_completed as native_render_completed, render_empty::render_empty, }; -use sqlbuild_analysis::planner_fixtures::models::{FixtureSqlInput, SuppliedProjection}; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureColumn, FixtureSqlInput, SuppliedProjection, +}; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; #[pyfunction(name = "is_empty_fixture_query")] pub(crate) fn fixture_marker(sql: &str) -> PyResult { empty_fixture_marker(sql).map_err(value_error) } -#[pyfunction] -#[pyo3(signature = (*, sql, relation, adapter))] +/// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. pub(crate) fn complete_empty_fixture_sql( sql: &str, - relation: &Bound<'_, PyAny>, + relation: Option<&FixtureRelation>, adapter: &Bound<'_, PyAny>, ) -> PyResult<(String, Option)> { if !fixture_marker(sql)? { return Ok((sql.into(), None)); } - let authoritative = - !relation.is_none() && relation.getattr("authoritative_names")?.is_truthy()?; - let metadata = if authoritative { - columns(&relation.getattr("columns")?, false)? - } else { - Vec::new() + let authoritative = relation.is_some_and(|relation| relation.authoritative); + let metadata: &[FixtureColumn] = match relation { + Some(relation) if authoritative => &relation.columns, + _ => &[], }; - if let Some(error) = empty_fixture_error(authoritative, &metadata) { + if let Some(error) = empty_fixture_error(authoritative, metadata) { return Ok((sql.into(), Some(error))); } Ok(( - render_empty(&metadata, |name| { + render_empty(metadata, |name| { adapter .call_method1("render_identifier", (name,))? .extract::() @@ -52,7 +51,7 @@ pub(crate) struct CompletionSql<'py, 'a> { pub(crate) inferred: Bound<'py, PyAny>, pub(crate) typed_nulls: Bound<'py, PyDict>, pub(crate) quoted_names: Bound<'py, PyAny>, - pub(crate) completed: Bound<'py, PyAny>, + pub(crate) completed: Vec, } pub(crate) fn render_completed( @@ -81,7 +80,7 @@ pub(crate) fn render_completed( sql: input.sql, supplied: &supplied, typed_nulls: !input.typed_nulls.is_empty(), - completed: &columns(&input.completed, false)?, + completed: &input.completed, }, |name, exact| { adapter @@ -98,19 +97,19 @@ pub(crate) fn render_completed( ) } -#[pyfunction] -#[pyo3(signature = (*, sql, inference, expected_types, adapter))] +/// Fixture SQL with its untyped null columns cast to the expected types. pub(crate) fn complete_typed_null_columns( sql: &str, inference: &Bound<'_, PyAny>, - expected_types: &Bound<'_, PyDict>, + expected_types: &[(String, String)], adapter: &Bound<'_, PyAny>, ) -> PyResult { let py = inference.py(); let typed_nulls = PyDict::new(py); for name in inference.getattr("null_literal_names")?.try_iter()? { let name = name?; - if let Some(expected) = expected_types.get_item(&name)? { + let folded: String = name.extract()?; + if let Some((_, expected)) = expected_types.iter().find(|(column, _)| *column == folded) { typed_nulls.set_item(name, expected)?; } } @@ -123,14 +122,12 @@ pub(crate) fn complete_typed_null_columns( inferred: inference.getattr("columns")?, typed_nulls, quoted_names: inference.getattr("quoted_names")?, - completed: pyo3::types::PyTuple::empty(py).into_any(), + completed: Vec::new(), }, adapter, ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_marker, module)?)?; - module.add_function(wrap_pyfunction!(complete_empty_fixture_sql, module)?)?; - module.add_function(wrap_pyfunction!(complete_typed_null_columns, module)?) + module.add_function(wrap_pyfunction!(fixture_marker, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index 3715f6bfb7..3a6a1dd758 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -1,9 +1,7 @@ //! Expected fixture type propagation boundary conversions. use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_analysis::planner_fixtures::main::propagate_types::propagate_types; use sqlbuild_analysis::planner_fixtures::models::{ FixtureTypeModel, FixtureTypeProjection, FixtureTypeRead, FixtureTypeState, @@ -12,33 +10,40 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis::model_reads::lineage_facts; -use crate::bindings::_helpers::planner_strategy::constants::DIRECT_TRANSFORM; +use crate::bindings::_helpers::planner_strategy::constants::{ + DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, +}; +use crate::bindings::_helpers::planner_strategy::models::{ + ExpectedTypes, RelationFixturePlanningContext, +}; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -#[pyfunction] -#[pyo3(signature = (*, base_expected_types, ordered_model_names, model_map))] -pub(crate) fn expected_fixture_types<'py>( - base_expected_types: &Bound<'py, PyDict>, - ordered_model_names: Vec, - model_map: &Bound<'py, PyDict>, -) -> PyResult> { +/// Each chain model's known column types with direct lineage types propagated from its inputs. +pub(crate) fn expected_fixture_types<'a>( + context: &'a RelationFixturePlanningContext, + py: Python<'_>, + ordered_model_names: &[String], +) -> PyResult> { + let base = &context.expected_types; let mut state = FixtureTypeState::default(); - let result = PyDict::new(base_expected_types.py()); + let mut overlay: Vec<(FixtureKey, Vec<(String, String)>)> = Vec::new(); let mut models: Vec = Vec::new(); for name in ordered_model_names { - let Some(model) = model_map.get_item(&name)? else { + let Some(facts) = context.model_facts(py, name)? else { continue; }; - let key = ("model".to_owned(), name); - let types = base_expected_types - .get_item(&key)? + let key: FixtureKey = (MODEL_REFERENCE_KIND.to_owned(), name.clone()); + let types = base + .get(&key) .ok_or_else(|| PyKeyError::new_err(key.clone()))? - .cast_into::()? - .copy()?; - state.known.insert(key.clone(), types.extract()?); - result.set_item(&key, types)?; + .clone(); + state + .known + .insert(key.clone(), types.iter().cloned().collect()); + overlay.push((key.clone(), types)); models.push(FixtureTypeModel { key, - projections: projections(&model)?, + projections: facts.projections.clone(), }); } for model in &models { @@ -51,33 +56,37 @@ pub(crate) fn expected_fixture_types<'py>( { continue; } - if let Some(types) = base_expected_types.get_item(&read.key)? - && let Some(value) = types.cast_into::()?.get_item(&read.column)? - { + if let Some(sql_type) = base.get(&read.key).and_then(|types| { + types + .iter() + .find(|(name, _)| *name == read.column) + .map(|(_, sql_type)| sql_type.clone()) + }) { state .known .entry(read.key.clone()) .or_default() - .insert(read.column.clone(), value.extract()?); + .insert(read.column.clone(), sql_type); } } } } let state = propagate_types(state, &models); for (read, sql_type) in state.additions { - result - .get_item(read.key)? - .ok_or_else(|| PyKeyError::new_err("fixture types"))? - .set_item(read.column, sql_type)?; + let types = overlay + .iter_mut() + .find(|(key, _)| *key == read.key) + .map(|(_, types)| types) + .ok_or_else(|| PyKeyError::new_err("fixture types"))?; + match types.iter_mut().find(|(name, _)| *name == read.column) { + Some(existing) => existing.1 = sql_type, + None => types.push((read.column, sql_type)), + } } - base_expected_types - .py() - .import("collections")? - .getattr("ChainMap")? - .call1((result, base_expected_types)) + Ok(ExpectedTypes { overlay, base }) } -fn projections(model: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn projections(model: &Bound<'_, PyAny>) -> PyResult> { let mut projections: Vec = Vec::new(); for (output_column, transform, _, sources) in lineage_facts(model)?.unwrap_or_default() { if transform != DIRECT_TRANSFORM { @@ -97,7 +106,3 @@ fn projections(model: &Bound<'_, PyAny>) -> PyResult> } Ok(projections) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expected_fixture_types, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs index 2177f21723..5d621a37c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs @@ -3,17 +3,13 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; pub(crate) fn folded(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("casefold")?.extract() } -pub(crate) fn strings(values: &Bound<'_, PyAny>) -> PyResult> { - values.try_iter()?.map(|value| value?.extract()).collect() -} - pub(crate) fn columns(values: &Bound<'_, PyAny>, inferred: bool) -> PyResult> { values .try_iter()? @@ -42,27 +38,6 @@ pub(crate) fn record<'py>( registry::class(py, name)?.call((), Some(kwargs)) } -pub(crate) fn metadata<'py>( - py: Python<'py>, - columns: &[FixtureColumn], - authoritative: bool, -) -> PyResult> { - let values = columns - .iter() - .map(|column| { - let kwargs = PyDict::new(py); - kwargs.set_item("name", &column.name)?; - kwargs.set_item("type", &column.sql_type)?; - kwargs.set_item("nullable", column.nullable)?; - record(py, RecordId::FixtureColumnMetadata, &kwargs) - }) - .collect::>>()?; - let kwargs = PyDict::new(py); - kwargs.set_item("columns", PyTuple::new(py, values)?)?; - kwargs.set_item("authoritative_names", authoritative)?; - record(py, RecordId::FixtureRelationMetadata, &kwargs) -} - pub(crate) fn label(key: &Bound<'_, PyAny>) -> PyResult { Ok(format!( "mock {} '{}'", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index ca7401805f..b29e2442b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -19,4 +19,5 @@ pub(crate) mod model_entries; pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; +pub(crate) mod types; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs index f3abaa0b26..6c998ddebe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -1,11 +1,58 @@ -//! Extension-owned fixture analysis reuse, with pure state passed into analysis. +//! Extension-owned fixture analysis reuse and the project-wide fixture planning context. +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +use pyo3::prelude::Py; use pyo3::pyclass; -use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; +use pyo3::types::{PyAny, PyDict}; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureColumn, FixtureReadCache, FixtureTypeProjection, +}; +use sqlbuild_analysis::required_columns::models::ColumnRead; + +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; /// Per-extension cache replacing the Python fixture resolver's LRU. #[pyclass] #[derive(Debug, Default)] pub(crate) struct FixtureReadCacheHost { pub(crate) cache: FixtureReadCache, + /// Fixture planning contexts built since the last reset. + pub(crate) context_builds: u64, + /// SQL-test model chain batches resolved since the last reset. + pub(crate) chain_resolutions: u64, +} + +/// One relation's known fixture columns and whether they form an authoritative shape. +#[derive(Clone, Debug)] +pub(crate) struct FixtureRelation { + pub(crate) columns: Vec, + pub(crate) authoritative: bool, +} + +/// One model's high-confidence lineage reads and direct type projections. +#[derive(Debug)] +pub(crate) struct ModelFixtureFacts { + pub(crate) reads: Vec, + pub(crate) projections: Vec, +} + +/// Project-wide fixture metadata shared across test and scenario planning. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct RelationFixturePlanningContext { + pub(crate) models_by_name: Py, + pub(crate) relations: HashMap, + pub(crate) authoritative_columns: HashMap>, + pub(crate) expected_types: HashMap>, + pub(crate) fixture_inferences: Py, + pub(crate) model_facts: Mutex>>, +} + +/// The expected column types of one completion: its chain's propagated types over the context's. +#[derive(Debug)] +pub(crate) struct ExpectedTypes<'a> { + pub(crate) overlay: Vec<(FixtureKey, Vec<(String, String)>)>, + pub(crate) base: &'a HashMap>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index c5bdbebbc5..89538ed245 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -4,8 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::planner_strategy::{ - audits, cursor_types, fixture_completion, fixture_metadata, fixture_sql, lineage, microbatches, - strategy, + audits, cursor_types, fixture_completion, fixture_sql, lineage, microbatches, strategy, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -53,12 +52,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audits::register(module)?; cursor_types::register(module)?; fixture_completion::register(module)?; - fixture_metadata::register(module)?; fixture_sql::register(module)?; super::fixture_context::register(module)?; super::fixture_reads::register(module)?; - super::fixture_selection::register(module)?; - super::fixture_types::register(module)?; lineage::register(module)?; microbatches::register(module)?; strategy::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs new file mode 100644 index 0000000000..2ae899fbc1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs @@ -0,0 +1,7 @@ +//! Fixture planning key and input shapes. + +/// `(resource type value, name)` of one fixture relation. +pub(crate) type FixtureKey = (String, String); + +/// One relation's `((name, type, nullable), ...)` columns and whether they are authoritative. +pub(crate) type FixtureRelationInput = (Vec<(String, Option, Option)>, bool); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 78b2df5d8d..e23a03ebc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -250,8 +250,6 @@ native_ids! { FingerprintSet => "FingerprintSet", FixVerdict => "FixVerdict", FixtureColumnInference => "FixtureColumnInference", - FixtureColumnMetadata => "FixtureColumnMetadata", - FixtureRelationMetadata => "FixtureRelationMetadata", FreshnessCommandRequest => "FreshnessCommandRequest", FunctionArgument => "FunctionArgument", FunctionChangeResult => "FunctionChangeResult", @@ -446,7 +444,6 @@ native_ids! { RelatedLocation => "RelatedLocation", RelationFixtureCompletion => "RelationFixtureCompletion", RelationFixtureDiagnostic => "RelationFixtureDiagnostic", - RelationFixturePlanningContext => "RelationFixturePlanningContext", ResolvedDbtConfig => "ResolvedDbtConfig", ResolvedDbtProfileOutput => "ResolvedDbtProfileOutput", ResolvedModelAction => "ResolvedModelAction", @@ -698,7 +695,6 @@ native_ids! { ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleBasePlanOutput => "assemble_base_plan_output", AssembleModelPlanEntry => "assemble_model_plan_entry", AssembleNativeProjectResources => "assemble_native_project_resources", AssembleNativeSqlTests => "assemble_native_sql_tests", @@ -752,7 +748,6 @@ native_ids! { BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", BuildPlanEntries => "build_plan_entries", - BuildPlanOutput => "build_plan_output", BuildPlannerEntryResults => "build_planner_entry_results", BuildPlannerIdentityContext => "build_planner_identity_context", BuildPlannerRelationsContext => "build_planner_relations_context", @@ -1249,9 +1244,6 @@ native_ids! { WithDuplicateLiteralHints => "with_duplicate_literal_hints", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", - WithPlanMetadata => "with_plan_metadata", - WithPlanWarnings => "with_plan_warnings", - WithStoragePolicies => "with_storage_policies", WithTypeRecovery => "with_type_recovery", WithoutDeclaredColumnsHash => "without_declared_columns_hash", WithoutDestinationCursor => "without_destination_cursor", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 19c80e379a..327da1b23d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -312,14 +312,4 @@ impl FieldSpec { shown: false, } } - - /// A field left out of equality and hashing but shown in repr. - pub(crate) const fn uncompared_shown(name: &'static str, default: FieldDefault) -> Self { - Self { - name, - default, - compared: false, - shown: true, - } - } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs index 6915959b3b..fce670a9e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -10,11 +10,13 @@ use sqlbuild_analysis::planner_fixtures::main::{ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_strategy::fixture_completion::build_relation_fixture_completion; -use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; +use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures; use crate::bindings::_helpers::planner_strategy::fixture_sql::{ complete_empty_fixture_sql, complete_typed_null_columns, }; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, FIXTURE_PREFIXES}; /// Report every statically provable fixture defect or return the completed fixture SQL groups. @@ -22,11 +24,11 @@ pub(super) fn validated_fixtures<'py>( test: &Bound<'py, PyAny>, request: (&Bound<'py, PyAny>, [Bound<'py, PyDict>; 4]), inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, + context: &RelationFixturePlanningContext, ) -> PyResult; 4], Vec>> { let py = test.py(); let (chain, [refs, sources, seeds, expected]) = request; - let (project, adapter) = inputs; + let adapter = inputs.1; let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; let model = resource_types.getattr("MODEL")?; let kinds = [ @@ -42,20 +44,10 @@ pub(super) fn validated_fixtures<'py>( PyTuple::new(py, [kinds[2].clone(), seeds.clone().into_any()])?, ], )?; - let context = if context.is_truthy()? { - context.clone() - } else { - build_relation_fixture_context(project, None)? - }; - let completion = build_relation_fixture_completion( - project, - adapter, - (chain.clone(), groups.into_any()), - Some(context.clone()), - )?; + let chain: Vec = chain.extract()?; + let completion = complete_fixtures(context, adapter, (&chain, groups.as_any()))?; let mut errors: BTreeSet = BTreeSet::new(); - for diagnostic in completion.getattr("diagnostics")?.try_iter()? { - let diagnostic = diagnostic?; + for diagnostic in &completion.diagnostics { let key = diagnostic.getattr("key")?; errors.insert(format!( "{}: {}", @@ -67,8 +59,7 @@ pub(super) fn validated_fixtures<'py>( diagnostic.getattr("message")?.extract::()? )); } - let expected_types = completion.getattr("expected_types")?; - let inferred_by_fixture = completion.getattr("inferred_by_fixture")?; + let inferred_by_fixture = &completion.inferred_by_fixture; let completed_expected = expected.copy()?; let sql_test = resource_types.getattr("SQL_TEST")?; let type_groups = [ @@ -83,12 +74,12 @@ pub(super) fn validated_fixtures<'py>( let name: String = key_name.extract()?; let location = || fixture_location(test, kind, &name); let inferred = if expected_output { - let model_key = PyTuple::new(py, [&model, &key_name])?; - let relation = context - .getattr("relations")? - .call_method1("get", (&model_key,))?; - let (completed_sql, empty_error) = - complete_empty_fixture_sql(&sql.extract::()?, &relation, adapter)?; + let model_key: FixtureKey = (MODEL_REFERENCE_KIND.to_owned(), name.clone()); + let (completed_sql, empty_error) = complete_empty_fixture_sql( + &sql.extract::()?, + context.relations.get(&model_key), + adapter, + )?; if let Some(error) = empty_error { errors.insert(format!("{}: expected output '{name}' {error}", location()?)); } @@ -99,20 +90,21 @@ pub(super) fn validated_fixtures<'py>( "inference_profile", adapter.call_method0("expression_inference_profile")?, )?; - let inference = context.getattr("fixture_inferences")?.call_method( - "infer", - (), - Some(&kwargs), - )?; + let inference = + context + .fixture_inferences + .bind(py) + .call_method("infer", (), Some(&kwargs))?; if inference.is_none() { None } else { - let types = expected_types - .call_method1("get", (&model_key, PyDict::new(py)))? - .cast_into::()?; + let types: &[(String, String)] = completion + .expected_types + .get(&model_key) + .unwrap_or_default(); completed_expected.set_item( &name, - complete_typed_null_columns(&completed_sql, &inference, &types, adapter)?, + complete_typed_null_columns(&completed_sql, &inference, types, adapter)?, )?; Some(inference.getattr("columns")?) } @@ -124,15 +116,23 @@ pub(super) fn validated_fixtures<'py>( let Some(columns) = inferred else { continue; }; - let type_kind = if expected_output { &model } else { kind }; - let type_key = PyTuple::new(py, [type_kind, &key_name])?; - let types = expected_types.call_method1("get", (type_key, PyDict::new(py)))?; + let type_kind: String = if expected_output { + MODEL_REFERENCE_KIND.to_owned() + } else { + kind.extract()? + }; + let types: &[(String, String)] = completion + .expected_types + .get(&(type_kind, name.clone())) + .unwrap_or_default(); for column in columns.try_iter()? { let column = column?; let column_name = column.getattr("name")?; + let folded_name: String = column_name.call_method0("casefold")?.extract()?; let expected_type: Option = types - .call_method1("get", (column_name.call_method0("casefold")?,))? - .extract()?; + .iter() + .find(|(column, _)| *column == folded_name) + .map(|(_, sql_type)| sql_type.clone()); let inferred_type: Option = column.getattr("type")?.extract()?; if !collection_mismatch(expected_type.as_deref(), inferred_type.as_deref()) { continue; @@ -159,7 +159,7 @@ pub(super) fn validated_fixtures<'py>( .map(|error| format!("SQL test '{test_name}': {error}")) .collect())); } - let fixture_sql = completion.getattr("fixture_sql_by_key")?; + let fixture_sql = &completion.sql_by_key; let completed = |kind: &Bound<'py, PyAny>, fixtures: &Bound<'py, PyDict>| -> PyResult> { let result = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 17e0dd54a1..b067230958 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -4,8 +4,8 @@ mod compiled_tests; pub(crate) mod constants; mod cursor_window; mod fixture_validation; -mod plan_assembly; -mod plan_driver; +pub(crate) mod plan_assembly; +pub(crate) mod plan_driver; mod plan_entries; mod plan_errors; mod plan_fixtures; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index 926142499f..dca5cd13fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -4,9 +4,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::plan_driver::plan_sql_tests_natively; #[pyfunction(name = "plan_sql_tests")] #[pyo3(signature = (**kwargs))] @@ -30,40 +35,45 @@ pub(crate) fn assemble_sql_test_plans<'py>( let tests: Vec> = args.value("tests")?.extract()?; let project = args.value("project")?; let adapter = args.value("adapter")?; - let mut context = py.None().into_bound(py); - let (planned, diagnostics) = - if args.flag("sql_analysis_enabled")? && args.flag("validate_fixtures")? { - context = args.value("fixture_planning_context")?; - if !context.is_truthy()? { - context = call_with( - ®istry::native(py, NativeAttr::BuildRelationFixtureContext)?, - &[("project", project.clone())], - )?; - } - super::plan_fixtures::validate(&tests, (&project, &adapter), &context)? + let mut context: Option> = None; + let (planned, diagnostics) = if args.flag("sql_analysis_enabled")? + && args.flag("validate_fixtures")? + { + let given = args.value("fixture_planning_context")?; + let shared = if given.is_none() { + Bound::new(py, planning_context(&project, None)?)? } else { - (tests.clone(), Default::default()) + given.cast_into::()? }; + let validated = super::plan_fixtures::validate(&tests, (&project, &adapter), shared.get())?; + context = Some(shared); + validated + } else { + (tests.clone(), Default::default()) + }; let plannable: Vec<_> = planned .into_iter() .enumerate() .filter_map(|(index, test)| (!diagnostics.contains_key(&index)).then_some(test)) .collect(); - let plans = call_with( - ®istry::native(py, NativeAttr::PlanSqlTestsNatively)?, - &[ - ("project", project.clone()), - ("tests", PyTuple::new(py, plannable)?.into_any()), - ("adapter", adapter), - ( - "sql_analysis_enabled", - args.value_or_false("sql_analysis_enabled")?, - ), - ( - "render_sql", - pyo3::types::PyBool::new(py, false).to_owned().into_any(), - ), - ], + let plans = plan_sql_tests_natively( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("tests", PyTuple::new(py, plannable)?.into_any()), + ("adapter", adapter), + ( + "sql_analysis_enabled", + args.value_or_false("sql_analysis_enabled")?, + ), + ( + "render_sql", + pyo3::types::PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?), )?; let plans: Vec> = plans.extract()?; if plans.len() != tests.len() - diagnostics.len() { @@ -97,7 +107,9 @@ pub(crate) fn assemble_sql_test_plans<'py>( }; let mut errors: Vec = plan.getattr("error_messages")?.extract()?; errors.extend(super::plan_entries::expected_diagnostics( - test, &plan, &context, + test, + &plan, + context.as_ref().map(Bound::get), )?); if !errors.is_empty() { results.push(call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index a9095ed179..4e5d1a0190 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::compile_timings::add_compile_timi use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; @@ -58,6 +59,7 @@ pub(crate) fn resolve_sql_test_model_chains<'py>( kwargs, (&["project", "tests"], &[]), )?; + count_chain_resolution(py)?; chains(&args.value("project")?, &args.value("tests")?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 153ce6dedc..7d60c531ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -3,11 +3,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use crate::bindings::_helpers::analysis::sql_test_comparisons::describe_missing_expected_columns; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_TYPES; use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; +use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -132,29 +134,32 @@ pub(super) fn entry<'py>( pub(super) fn expected_diagnostics( test: &Bound<'_, PyAny>, plan: &Bound<'_, PyAny>, - context: &Bound<'_, PyAny>, + context: Option<&RelationFixturePlanningContext>, ) -> PyResult> { let mut result: Vec = Vec::new(); - if context.is_none() { + let Some(context) = context else { return Ok(result); - } + }; let py = test.py(); let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; for step in plan.getattr("chain")?.try_iter()? { let step = step?; let name = step.getattr("model_name")?; + let model_name: String = name.extract()?; let available = context - .getattr("authoritative_columns")? - .call_method1("get", ((resource_types.getattr("MODEL")?, &name),))?; + .authoritative_columns + .get(&(MODEL_REFERENCE_KIND.to_owned(), model_name)); let expected = step.getattr("expected_columns")?; - if expected.is_none() || available.is_none() { + let Some(available) = available else { + continue; + }; + if expected.is_none() { continue; } - let Some(message) = describe_missing_expected_columns( - &name.extract::()?, - expected.extract()?, - available, - )? + let available: Vec = available.iter().cloned().collect(); + let expected: Vec = expected.extract()?; + let Some(message) = + missing_expected_columns(&name.extract::()?, &expected, &available) else { continue; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs index 78a9c510a2..4e8d3f69c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -3,18 +3,18 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; +use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; pub(super) fn validated_test<'py>( test: &Bound<'py, PyAny>, chain: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, + context: &RelationFixturePlanningContext, ) -> PyResult, Vec>> { let py = test.py(); let payload = test.getattr("payload")?; @@ -78,9 +78,9 @@ pub(super) fn validated_test<'py>( pub(super) fn validate<'py>( tests: &[Bound<'py, PyAny>], inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, + context: &RelationFixturePlanningContext, ) -> PyResult> { - let py = context.py(); + let py = inputs.0.py(); let mut model_tests: Vec> = Vec::new(); let mut indexes: Vec = Vec::new(); for (index, test) in tests.iter().enumerate() { @@ -92,12 +92,15 @@ pub(super) fn validate<'py>( if indexes.is_empty() { return Ok((tests.to_vec(), Default::default())); } - let chains = call_with( - ®istry::native(py, NativeAttr::ResolveSqlTestModelChains)?, - &[ - ("project", inputs.0.clone()), - ("tests", PyTuple::new(py, model_tests)?.into_any()), - ], + let chains = resolve_sql_test_model_chains( + py, + Some(keywords( + py, + &[ + ("project", inputs.0.clone()), + ("tests", PyTuple::new(py, model_tests)?.into_any()), + ], + )?), )?; let chains: Vec> = chains.extract()?; if chains.len() != indexes.len() { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 54f6ce2037..bbe387de6a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -768,42 +768,23 @@ class RelationFixtureCompletion(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> RelationFixtureCompletion: ... -class FixtureColumnMetadata(NativeRecord): - """Known type and nullability for one relation fixture column.""" - - name: str - type: str | None - nullable: bool | None - def __init__(self, name: str, type: str | None, nullable: bool | None) -> None: ... - def __replace__(self, **changes: Any) -> FixtureColumnMetadata: ... - -class FixtureRelationMetadata(NativeRecord): - """Known columns and whether they form an authoritative relation shape.""" - - columns: tuple[FixtureColumnMetadata, ...] - authoritative_names: bool - def __init__( - self, columns: tuple[FixtureColumnMetadata, ...], authoritative_names: bool - ) -> None: ... - def __replace__(self, **changes: Any) -> FixtureRelationMetadata: ... - -class RelationFixturePlanningContext(NativeRecord): +class RelationFixturePlanningContext: """Project-wide fixture metadata shared across test and scenario planning.""" - models_by_name: dict[str, CompiledModelView] - relations: dict[FixtureKey, FixtureRelationMetadata] - authoritative_columns: dict[FixtureKey, frozenset[str]] - expected_types: dict[FixtureKey, dict[str, str]] - fixture_inferences: FixtureColumnInferences def __init__( self, + *, models_by_name: dict[str, CompiledModelView], - relations: dict[FixtureKey, FixtureRelationMetadata], - authoritative_columns: dict[FixtureKey, frozenset[str]], - expected_types: dict[FixtureKey, dict[str, str]], - fixture_inferences: FixtureColumnInferences = ..., + relations: dict[ + tuple[CompiledResourceType, str], + tuple[tuple[tuple[str, str | None, bool | None], ...], bool], + ], ) -> None: ... - def __replace__(self, **changes: Any) -> RelationFixturePlanningContext: ... + def authoritative_names(self, key: tuple[CompiledResourceType, str]) -> bool | None: ... + def expected_types(self, key: tuple[CompiledResourceType, str]) -> dict[str, str] | None: ... + def authoritative_columns( + self, key: tuple[CompiledResourceType, str] + ) -> frozenset[str] | None: ... class SelectorExpansion(NativeRecord): """One selector split into graph expansion flags and core text.""" @@ -4179,58 +4160,13 @@ def lint_interpolation_text_at(body: str, start: int, dialect: str) -> str | Non def scan_fixture_typed_null_candidates( project_dir: Path, selected_paths: frozenset[Path] | None = None ) -> tuple[frozenset[Path], frozenset[str]]: ... -def simplify_fixture_null_projections(*, sql: str, relation: FixtureRelationMetadata) -> str: ... +def simplify_fixture_null_projections(*, sql: str, relation: Any) -> str: ... def is_empty_fixture_query(sql: str) -> bool: ... -def complete_empty_fixture_sql( - *, sql: str, relation: FixtureRelationMetadata | None, adapter: BaseAdapter -) -> tuple[str, str | None]: ... -def complete_typed_null_columns( - *, - sql: str, - inference: FixtureColumnInference, - expected_types: dict[str, str], - adapter: BaseAdapter, -) -> str: ... -def model_fixture_metadata(*, model: CompiledModelView) -> FixtureRelationMetadata: ... -def expected_fixture_types( - *, - base_expected_types: dict[tuple[CompiledResourceType, str], dict[str, str]], - ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModelView], -) -> Mapping[tuple[CompiledResourceType, str], dict[str, str]]: ... def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... -def required_fixture_columns( - *, - model_inputs: tuple[tuple[str, ...], dict[str, CompiledModelView]], - fixture_inputs: tuple[ - frozenset[tuple[CompiledResourceType, str]], - dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], - dict[tuple[CompiledResourceType, str], frozenset[str]], - ], - adapter: BaseAdapter, -) -> tuple[ - dict[tuple[CompiledResourceType, str], dict[str, set[str]]], - frozenset[tuple[CompiledResourceType, str]], -]: ... +def fixture_planning_work_info(*, reset: bool = False) -> tuple[int, int]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... -def authoritative_fixture_columns( - *, relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata] -) -> dict[tuple[CompiledResourceType, str], frozenset[str]]: ... -def base_expected_fixture_types( - *, relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata] -) -> dict[tuple[CompiledResourceType, str], dict[str, str]]: ... -def fixture_keys_requiring_analysis( - *, - inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]], - authoritative_columns: dict[tuple[CompiledResourceType, str], frozenset[str]], -) -> frozenset[tuple[CompiledResourceType, str]]: ... -def unknown_fixture_column_diagnostics( - *, - inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]], - relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], -) -> tuple[RelationFixtureDiagnostic, ...]: ... def build_relation_fixture_completion( *, project: CompiledProjectView, @@ -7940,48 +7876,9 @@ def plan_table_types( def plan_retention( *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, scope: PlannerScope ) -> tuple[RetentionPlanEntry, ...]: ... -def assemble_base_plan_output( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - pruning: PlannerScopePruningResult, - reconciliation: PlannerChangeReconciliation, - entries: PlannerEntryResults, - source_freshness: DirectSourceFreshnessPlanningResult, - planned_sql_tests: PlannedSqlTests | None = None, -) -> PlanOutput: ... def with_column_rename_hints( *, entries: tuple[ModelPlanEntry, ...], hints: tuple[ColumnRenameHint, ...] ) -> tuple[ModelPlanEntry, ...]: ... -def with_storage_policies( - *, - plan_output: PlanOutput, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - scopes: PlannerScopeResolution, - policies: PlannerPolicies, -) -> PlanOutput: ... -def with_plan_warnings( - *, - runtime: PlannerRuntime, - scopes: PlannerScopeResolution, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - stale_warning_changes: PlannerChangeResults, - pruning: PlannerScopePruningResult, - source_freshness: DirectSourceFreshnessPlanningResult, - plan_output: PlanOutput, - policies: PlannerPolicies, -) -> PlanOutput: ... -def with_plan_metadata( - *, - plan_output: PlanOutput, - pruning: PlannerScopePruningResult, - source_freshness: DirectSourceFreshnessPlanningResult, - policies: PlannerPolicies, -) -> PlanOutput: ... def gather_planner_warehouse_state( *, runtime: PlannerRuntime, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py new file mode 100644 index 0000000000..d70c2abf67 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py @@ -0,0 +1,11 @@ +"""Helpers for fixture type-scope integration tests.""" + + +def unrelated_source_entries(*, count: int) -> str: + """Source YAML entries for relations no fixture chain reads.""" + return "".join( + f" - name: unrelated_orders_{index}\n description: Unrelated input.\n" + " expression: SELECT 1 AS order_id\n" + " columns:\n - name: order_id\n type: INTEGER\n" + for index in range(count) + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py index 601f40a9c9..bb8df9c6ab 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -1,6 +1,4 @@ -from collections import ChainMap -from collections.abc import Callable, Mapping -from copy import deepcopy +from collections.abc import Callable from pathlib import Path import pytest @@ -18,14 +16,16 @@ from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures._test_types import ( FixtureTypeScopeTestCase, ) +from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures.helpers import ( + unrelated_source_entries, +) @pytest.mark.parametrize( "test_case", ( FixtureTypeScopeTestCase(description="small metadata map", unrelated_relations=10), - FixtureTypeScopeTestCase(description="large metadata map", unrelated_relations=1000), - FixtureTypeScopeTestCase(description="very large metadata map", unrelated_relations=10000), + FixtureTypeScopeTestCase(description="large metadata map", unrelated_relations=200), ), ids=lambda case: case.description, ) @@ -48,6 +48,7 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i " contract: enforced\n columns:\n" " - name: order_id\n type: INTEGER\n nullable: false\n" " - name: status\n type: VARCHAR\n nullable: true\n" + + unrelated_source_entries(count=test_case.unrelated_relations) ), }, ) @@ -56,14 +57,9 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) context: RelationFixturePlanningContext = build_relation_fixture_context(project=project) - # Model the shared project metadata supplied to every individual SQL-test plan. - for index in range(test_case.unrelated_relations): - context.expected_types[(CompiledResourceType.SOURCE, f"unrelated_orders_{index}")] = { - "order_id": "INTEGER", - "status": "VARCHAR", - } - before: dict[tuple[CompiledResourceType, str], dict[str, str]] = deepcopy( - context.expected_types + unrelated_key: tuple[CompiledResourceType, str] = ( + CompiledResourceType.SOURCE, + "unrelated_orders_0", ) fixture_key: tuple[CompiledResourceType, str] = (CompiledResourceType.SOURCE, "raw_orders") completed: RelationFixtureCompletion = build_relation_fixture_completion( @@ -77,20 +73,12 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i ) assert completed.diagnostics == () assert 'CAST(NULL AS VARCHAR) AS "status"' in completed.fixture_sql_by_key[fixture_key] - assert dict(completed.expected_types) == before - assert list(completed.expected_types) == list(before) - assert context.expected_types == before - # Count eagerly materialized relation/type entries, independent of machine timing. - eager: Mapping[tuple[CompiledResourceType, str], dict[str, str]] = getattr( - completed.expected_types, "maps", (completed.expected_types,) - )[0] - assert len(eager) == test_case.expected_eager_relations - assert sum(len(columns) for columns in eager.values()) == test_case.expected_eager_columns - assert isinstance(completed.expected_types, ChainMap) - assert completed.expected_types.maps[-1] is context.expected_types - assert completed.expected_types[fixture_key] is context.expected_types[fixture_key] - model_key: tuple[CompiledResourceType, str] = (CompiledResourceType.MODEL, "orders") - assert completed.expected_types[model_key] is not context.expected_types[model_key] + assert context.expected_types(unrelated_key) == {"order_id": "INTEGER"} + assert len(completed.expected_types) == test_case.expected_eager_relations + assert ( + sum(len(columns) for columns in completed.expected_types.values()) + == test_case.expected_eager_columns + ) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 6cf18838c5..98f64d28f0 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -226,8 +226,8 @@ def test_given_source_contract_when_completing_fixture_then_respects_authoritati ), planning_context=context, ) - assert context.relations[key].authoritative_names is test_case.expected_authoritative - assert context.expected_types[key] == {"order_id": "INTEGER"} + assert context.authoritative_names(key) is test_case.expected_authoritative + assert context.expected_types(key) == {"order_id": "INTEGER"} assert ( tuple(diagnostic.message for diagnostic in completed.diagnostics) == test_case.expected_diagnostics diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 5da157a363..826bc5e16d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -11,8 +11,6 @@ CompiledObjectKey, CompiledSqlTest, CompileSqlReference, - FixtureColumnMetadata, - FixtureRelationMetadata, PlanWarning, RelationFixtureCompletion, RelationFixturePlanningContext, @@ -55,7 +53,6 @@ ) def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is_built_once( test_case: FixturePlanningWorkTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: compiled_test, project = build_test_and_project( PlanTestChainTestCase( @@ -94,11 +91,8 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is sql_tests=tests, settings=SettingsConfig(sql_analysis=True), ) - metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) - monkeypatch.setattr(fixture_completion, "build_relation_fixture_context", metadata_builder) fixture_completion.fixture_read_cache_info(reset=True) - topology_builder: Mock = Mock(wraps=sql_test_assembly.resolve_sql_test_model_chains) - monkeypatch.setattr(sql_test_assembly, "resolve_sql_test_model_chains", topology_builder) + fixture_completion.fixture_planning_work_info(reset=True) entries: list[SqlTestPlanEntry] warnings: list[PlanWarning] @@ -111,9 +105,12 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is assert len(entries) == test_case.test_count assert warnings == [] - assert metadata_builder.call_count == test_case.expected_metadata_builds + metadata_builds: int + topology_builds: int + metadata_builds, topology_builds = fixture_completion.fixture_planning_work_info() + assert metadata_builds == test_case.expected_metadata_builds assert fixture_completion.fixture_read_cache_info()[0] == test_case.expected_analysis_calls - assert topology_builder.call_count == test_case.expected_topology_builds + assert topology_builds == test_case.expected_topology_builds @pytest.mark.parametrize( @@ -216,15 +213,8 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ context: RelationFixturePlanningContext = RelationFixturePlanningContext( models_by_name={model.name: model}, relations={ - source_key: FixtureRelationMetadata( - columns=(FixtureColumnMetadata(name="order_id", type="INTEGER", nullable=False),), - authoritative_names=True, - ) - }, - authoritative_columns={source_key: frozenset(("order_id",))}, - expected_types={ - source_key: {"order_id": "INTEGER"}, - (CompiledResourceType.MODEL, model.name): {}, + source_key: ((("order_id", "INTEGER", False),), True), + (CompiledResourceType.MODEL, model.name): ((), False), }, ) fixture_completion.fixture_read_cache_info(reset=True) From a310e028a9430177fa96d8b0458734e4513346d0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:00:07 +0100 Subject: [PATCH 7/7] fix: keep the last declared type for case-variant duplicate fixture columns --- .../planner_fixtures/main/fixture_schema.rs | 8 ++- .../src/planner_fixtures/tests/test_schema.rs | 53 ++++++++++++++++++- .../src/planner_fixtures/tests/test_types.rs | 7 +++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs index d5e1de8904..3aff0a8181 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs @@ -7,7 +7,13 @@ pub fn fixture_schema(columns: &[FixtureColumn], authoritative: bool) -> Fixture let mut expected_types: Vec<(String, String)> = Vec::new(); for column in columns { if let Some(sql_type) = &column.sql_type { - expected_types.push((column.folded_name.clone(), sql_type.clone())); + match expected_types + .iter_mut() + .find(|(name, _)| *name == column.folded_name) + { + Some((_, existing)) => existing.clone_from(sql_type), + None => expected_types.push((column.folded_name.clone(), sql_type.clone())), + } } } FixtureSchema { diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs index bce18ee592..86c585a89d 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs @@ -2,7 +2,11 @@ use crate::planner_fixtures::main::{ fixture_schema::fixture_schema, requires_analysis::requires_analysis, unknown_columns::unknown_columns, }; -use crate::planner_fixtures::tests::{helpers::column, test_types::SchemaTestCase}; +use crate::planner_fixtures::models::FixtureColumn; +use crate::planner_fixtures::tests::{ + helpers::column, + test_types::{ExpectedTypesTestCase, SchemaTestCase}, +}; #[test] fn given_closed_and_open_schemas_when_selecting_then_names_and_diagnostics_agree() { @@ -86,3 +90,50 @@ fn given_closed_and_open_schemas_when_selecting_then_names_and_diagnostics_agree ); } } + +#[test] +fn given_duplicate_folded_columns_when_collecting_types_then_last_declared_type_wins() { + let test_cases = [ + ExpectedTypesTestCase { + description: "distinct columns keep declaration order", + columns: &[ + ("order_id", "order_id", "INTEGER"), + ("status", "status", "VARCHAR"), + ], + expected_types: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + }, + ExpectedTypesTestCase { + description: "case variants keep the first position and the last type", + columns: &[ + ("order_id", "order_id", "INTEGER"), + ("status", "status", "INTEGER"), + ("STATUS", "status", "VARCHAR"), + ], + expected_types: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + }, + ]; + for test_case in test_cases { + let columns: Vec = test_case + .columns + .iter() + .map(|(name, folded_name, sql_type)| FixtureColumn { + name: (*name).into(), + folded_name: (*folded_name).into(), + sql_type: Some((*sql_type).into()), + nullable: Some(true), + }) + .collect(); + let schema = fixture_schema(&columns, false); + let collected: Vec<(&str, &str)> = schema + .expected_types + .iter() + .map(|(name, sql_type)| (name.as_str(), sql_type.as_str())) + .collect(); + assert_eq!( + collected.as_slice(), + test_case.expected_types, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs index feade452bc..acbc523c4b 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -7,6 +7,13 @@ pub(crate) struct PropagationTestCase { pub(crate) expected_type: Option<&'static str>, } +#[derive(Debug)] +pub(crate) struct ExpectedTypesTestCase { + pub(crate) description: &'static str, + pub(crate) columns: &'static [(&'static str, &'static str, &'static str)], + pub(crate) expected_types: &'static [(&'static str, &'static str)], +} + #[derive(Debug)] pub(crate) struct SchemaTestCase { pub(crate) description: &'static str,