diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs index d5e1de8904..3aff0a8181 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/fixture_schema.rs @@ -7,7 +7,13 @@ pub fn fixture_schema(columns: &[FixtureColumn], authoritative: bool) -> Fixture let mut expected_types: Vec<(String, String)> = Vec::new(); for column in columns { if let Some(sql_type) = &column.sql_type { - expected_types.push((column.folded_name.clone(), sql_type.clone())); + match expected_types + .iter_mut() + .find(|(name, _)| *name == column.folded_name) + { + Some((_, existing)) => existing.clone_from(sql_type), + None => expected_types.push((column.folded_name.clone(), sql_type.clone())), + } } } FixtureSchema { diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index e6995f179a..703a4defa1 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -58,14 +58,14 @@ pub struct FixtureStarRelation { } /// One upstream typed column identified by its resource and folded name. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct FixtureTypeRead { pub key: (String, String), pub column: String, } /// Direct lineage facts used to propagate fixture types. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct FixtureTypeProjection { pub output_column: String, pub upstream: Vec, diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs index bce18ee592..86c585a89d 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_schema.rs @@ -2,7 +2,11 @@ use crate::planner_fixtures::main::{ fixture_schema::fixture_schema, requires_analysis::requires_analysis, unknown_columns::unknown_columns, }; -use crate::planner_fixtures::tests::{helpers::column, test_types::SchemaTestCase}; +use crate::planner_fixtures::models::FixtureColumn; +use crate::planner_fixtures::tests::{ + helpers::column, + test_types::{ExpectedTypesTestCase, SchemaTestCase}, +}; #[test] fn given_closed_and_open_schemas_when_selecting_then_names_and_diagnostics_agree() { @@ -86,3 +90,50 @@ fn given_closed_and_open_schemas_when_selecting_then_names_and_diagnostics_agree ); } } + +#[test] +fn given_duplicate_folded_columns_when_collecting_types_then_last_declared_type_wins() { + let test_cases = [ + ExpectedTypesTestCase { + description: "distinct columns keep declaration order", + columns: &[ + ("order_id", "order_id", "INTEGER"), + ("status", "status", "VARCHAR"), + ], + expected_types: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + }, + ExpectedTypesTestCase { + description: "case variants keep the first position and the last type", + columns: &[ + ("order_id", "order_id", "INTEGER"), + ("status", "status", "INTEGER"), + ("STATUS", "status", "VARCHAR"), + ], + expected_types: &[("order_id", "INTEGER"), ("status", "VARCHAR")], + }, + ]; + for test_case in test_cases { + let columns: Vec = test_case + .columns + .iter() + .map(|(name, folded_name, sql_type)| FixtureColumn { + name: (*name).into(), + folded_name: (*folded_name).into(), + sql_type: Some((*sql_type).into()), + nullable: Some(true), + }) + .collect(); + let schema = fixture_schema(&columns, false); + let collected: Vec<(&str, &str)> = schema + .expected_types + .iter() + .map(|(name, sql_type)| (name.as_str(), sql_type.as_str())) + .collect(); + assert_eq!( + collected.as_slice(), + test_case.expected_types, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs index feade452bc..acbc523c4b 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_types.rs @@ -7,6 +7,13 @@ pub(crate) struct PropagationTestCase { pub(crate) expected_type: Option<&'static str>, } +#[derive(Debug)] +pub(crate) struct ExpectedTypesTestCase { + pub(crate) description: &'static str, + pub(crate) columns: &'static [(&'static str, &'static str, &'static str)], + pub(crate) expected_types: &'static [(&'static str, &'static str)], +} + #[derive(Debug)] pub(crate) struct SchemaTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index a15ec21969..747d81f21a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -6,6 +6,19 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ DeclaredRow, InferredRow, LineageInput, NativeCompiledModel, }; +/// `(output, transform kind, confidence, [(resource type, resource, column)])` of one lineage output. +pub(crate) type LineageFactRow = (String, String, String, Vec<(String, String, String)>); + +const TRANSFORM_VALUES: [&str; 6] = [ + "direct", + "cast", + "expression", + "aggregation", + "star", + "constant", +]; +const CONFIDENCE_VALUES: [&str; 3] = ["unknown", "high", "medium"]; + /// Each inferred output column as `(name, type, nullability)`, `None` when none were inferred. pub(crate) fn inferred_rows(model: &Bound<'_, PyAny>) -> PyResult>> { if let Ok(view) = model.downcast::() { @@ -117,3 +130,110 @@ pub(crate) fn output_proven(model: &Bound<'_, PyAny>) -> PyResult { let proof: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; Ok(!proof.is_none() && proof.getattr("output_proven")?.is_truthy()?) } + +/// Each lineage output with its transform kind and confidence values, `None` without lineage. +pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Ok(view) = model.downcast::() { + let Some(rows) = view.get().lineage_rows(model.py())? else { + return Ok(None); + }; + let mut facts: Vec = Vec::with_capacity(rows.len()); + for row in rows { + facts.push(( + row.output_column, + coded(&TRANSFORM_VALUES, row.transform_code)?, + coded(&CONFIDENCE_VALUES, row.confidence_code)?, + row.sources, + )); + } + return Ok(Some(facts)); + } + let columns: Bound<'_, PyAny> = model.getattr("fast_lineage_columns")?; + if columns.is_none() { + return Ok(None); + } + let mut facts: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut sources: Vec<(String, String, String)> = Vec::new(); + for source in output.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + sources.push(( + source.getattr("resource_type")?.extract()?, + source.getattr("resource_name")?.extract()?, + source.getattr("column_name")?.extract()?, + )); + } + facts.push(( + output.getattr("output_column")?.extract()?, + output.getattr("transform_kind")?.str()?.extract()?, + output.getattr("confidence")?.str()?.extract()?, + sources, + )); + } + Ok(Some(facts)) +} + +fn coded(values: &[&str], code: u8) -> PyResult { + values + .get(usize::from(code)) + .map(|value| (*value).to_owned()) + .ok_or_else(|| pyo3::exceptions::PyIndexError::new_err("tuple index out of range")) +} + +/// The input relations of a dynamic pivot proof that proved the model's outputs, else `None`. +pub(crate) fn proven_inputs(model: &Bound<'_, PyAny>) -> PyResult>> { + if let Ok(view) = model.downcast::() { + return view.get().proven_inputs(model.py()); + } + let proof: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; + if proof.is_none() || !proof.getattr("output_proven")?.is_truthy()? { + return Ok(None); + } + proof.getattr("input_relations")?.extract().map(Some) +} + +/// The model's name. +pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult { + if let Ok(view) = model.downcast::() { + return view.get().name(model.py()); + } + model.getattr("name")?.extract() +} + +/// The model's compiled query SQL. +pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + return Ok(view + .get() + .query_sql(model.py())? + .into_bound(model.py()) + .into_any()); + } + model.getattr("query_sql") +} + +/// The model's `CompiledObjectKey`. +pub(crate) fn model_key<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + return Ok(view.get().key(model.py())?.into_bound(model.py())); + } + model.getattr("key") +} + +/// The model's destination `CompiledRelationLocation`. +pub(crate) fn destination<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + return Ok(view.get().destination(model.py())?.into_bound(model.py())); + } + model.getattr("destination") +} + +/// The model's name as a Python string. +pub(crate) fn name_object<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + if let Ok(view) = model.downcast::() { + let name: String = view.get().name(model.py())?; + return Ok(pyo3::types::PyString::new(model.py(), &name).into_any()); + } + model.getattr("name") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs index ff98e042bf..41b25aaa3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs @@ -17,7 +17,7 @@ pub(crate) struct FixtureColumnInferences { #[pymethods] impl FixtureColumnInferences { #[new] - fn new() -> Self { + pub(crate) fn new() -> Self { Self { by_profile: Mutex::new(Vec::new()), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs index f259791419..827bd3bc77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs @@ -32,7 +32,7 @@ fn coded_value_error(py: Python<'_>, message: String, code: &str) -> PyResult, sql: &Bound<'_, PyString>, context: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index 19874d97ce..a23568749f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -12,6 +12,7 @@ use sqlbuild_analysis::diff_selection::errors::SelectorError; use sqlbuild_analysis::diff_selection::main::resolve_diff_selection::resolve_diff_selection; use sqlbuild_analysis::diff_selection::models::DiffGraph; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -165,13 +166,13 @@ fn diff_model_names<'py>( let mut names: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - let key: Bound<'py, PyAny> = model.getattr("key")?; + let key: Bound<'py, PyAny> = model_reads::model_key(&model)?; let chosen: bool = match keys.indexes.get_item(&key)? { Some(index) => selected.contains(&index.extract::()?), None => false, }; if chosen && key.getattr("resource_type")?.eq(&model_type)? { - names.push(model.getattr("name")?); + names.push(model_reads::name_object(&model)?); } } PyTuple::new(py, names) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index ac38fc0f72..511b6b8033 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -72,6 +72,8 @@ pub(crate) struct DestinationFact { /// What assembly and the semantic checks finished for one compiled model. pub(crate) struct CompiledRecord { + /// The model's name, its file stem. + name: String, /// `(resource type, name)` of each dependency. deps: Vec<(String, String)>, destination: DestinationFact, @@ -156,6 +158,7 @@ pub(crate) fn compiled_record( let config: &Bound<'_, PyAny> = input.config.bind(py); let values: Bound<'_, PyDict> = config.getattr(VALUES_ATTRIBUTE)?.downcast_into()?; Ok(CompiledRecord { + name: name.to_owned(), deps, destination: DestinationFact { database: text_value(&values, DATABASE_KEY)?, @@ -309,6 +312,25 @@ impl NativeCompiledModel { &self.overlay.poisoned } + /// The kept lineage rows with their transform and confidence codes, or `None`. + pub(crate) fn lineage_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled.analysis.lineage_rows().map(<[LineageRow]>::to_vec)) + }) + } + + /// The input relations of a dynamic pivot proof that proved its outputs, else `None`. + pub(crate) fn proven_inputs(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + Ok(compiled + .analysis + .proof + .as_ref() + .filter(|proof| proof.output_proven) + .map(|proof| proof.input_relations.clone())) + }) + } + fn model_file_attribute(&self, py: Python<'_>, name: &str) -> PyResult> { self.read(py, |record, py| { Ok(record.model_file.bind(py).getattr(name)?.unbind()) @@ -344,7 +366,7 @@ fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { #[pymethods] impl NativeCompiledModel { #[getter] - fn key(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn key(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| { let compiled: &CompiledRecord = compiled_of(record)?; compiled @@ -387,7 +409,7 @@ impl NativeCompiledModel { } #[getter] - fn query_sql(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn query_sql(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, _| Ok(record.query_sql.clone_ref(py))) } @@ -397,7 +419,7 @@ impl NativeCompiledModel { } #[getter] - fn destination(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn destination(&self, py: Python<'_>) -> PyResult> { self.compiled(py, |compiled, py| { self.projections .destination @@ -731,6 +753,9 @@ pub(crate) fn model_graph_row( } fn model_name(record: &ModelInputRecord, py: Python<'_>) -> PyResult { + if let Some(compiled) = &record.compiled { + return Ok(compiled.name.clone()); + } record .model_file .bind(py) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index 031744d937..f9944cedda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -7,6 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::janitor_planning::main::migration_origin_keys::migration_origin_keys; use sqlbuild_analysis::janitor_planning::main::old_name_request_matches::old_name_request_matches; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::janitor::janitor_facts::{ SchemaRelations, Warehouse, model_destinations, @@ -228,7 +229,7 @@ fn project_destinations<'py>(project: &Bound<'py, PyAny>) -> PyResult = project.py(); let destinations: Bound<'py, PyDict> = PyDict::new(py); for (key, model) in model_destinations(project)? { - destinations.set_item(key.to_python(py)?, model.getattr("name")?)?; + destinations.set_item(key.to_python(py)?, model_reads::name_object(&model)?)?; } Ok(destinations) } @@ -369,7 +370,7 @@ pub(crate) fn pending_migration_origins( if listed.contains(&destination.folded()) { continue; } - let model_name: String = model.getattr("name")?.extract()?; + let model_name: String = model_reads::model_name(&model)?; for origin in migration_origin_keys(&raw, destination.database.as_deref(), &schemas) { let folded: Folded = Key { database: origin.database, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index 052731e1ce..e6fbde744d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -15,7 +15,7 @@ const CREATE_TABLE_ACTION: &str = "create_table"; /// Each selected model's change with the backfill its own `replay_on_change` decided. #[pyfunction] #[pyo3(signature = (*, scope, changes))] -fn resolve_model_actions<'py>( +pub(crate) fn resolve_model_actions<'py>( scope: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, ) -> PyResult> { @@ -52,7 +52,7 @@ fn resolve_model_actions<'py>( /// Incremental replacements, whose bounds must ignore the destination maximum. #[pyfunction] #[pyo3(signature = (*, materialization_type, action))] -fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { +pub(crate) fn replaces_incremental_relation(materialization_type: &str, action: &str) -> bool { materialization_type == INCREMENTAL_MATERIALIZATION && action == CREATE_TABLE_ACTION } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 274e121ff8..db69992068 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -1,5 +1,6 @@ //! Detect how each selected model and function changed since its recorded build. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -22,9 +23,11 @@ use sqlbuild_analysis::planner_changes::models::{ SchemaFindingRow, }; +use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -59,7 +62,7 @@ pub(crate) struct ModelPass<'py> { /// Selected model and function changes. #[pyfunction] #[pyo3(signature = (**options))] -fn detect_changes<'py>( +pub(crate) fn detect_changes<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -273,7 +276,7 @@ fn scope_model_changes<'py>( if model.is_none() { continue; } - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let expected: (Option, Option) = ( lookup(&version_hashes, &name)?, lookup(&metadata_jsons, &name)?, @@ -297,7 +300,7 @@ pub(crate) fn model_change<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let snapshot: &Bound<'py, PyAny> = &pass.snapshot; - let model_name: Bound<'py, PyAny> = model.getattr("name")?; + let model_name: Bound<'py, PyAny> = model_reads::name_object(model)?; let columns_hash: String = model_declared_columns_hash(model)?; let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { Some(text) => text, @@ -337,7 +340,7 @@ pub(crate) fn model_change<'py>( py, &model_name.extract::()?, ( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(model)?.extract::()?, dialect.extract::>()?.as_deref(), ), )?) @@ -381,7 +384,7 @@ pub(crate) fn model_change<'py>( let renames: Bound<'py, PyAny> = snapshot.getattr("reference_renames")?; let origins: Option> = match (&compiled_query_hash, &recorded) { (Some(_), Some(recorded)) if definition_changed && renames.is_truthy()? => origin_names( - (model.getattr("query_sql")?.extract()?, &fingerprint), + (model_reads::query_sql(model)?.extract()?, &fingerprint), &renames, (&recorded.definition_hash, &dialect), )?, @@ -495,7 +498,7 @@ fn log_query_check( &log_sql, &[ ("logger", logger.clone()), - ("sql", model.getattr("query_sql")?), + ("sql", model_reads::query_sql(model)?), ("action", text("compiled_query")), ], )?; @@ -542,26 +545,11 @@ fn model_schema_findings<'py>( dynamic = Some(families); } } - let inferred: Option)>> = { - let columns: Bound<'py, PyAny> = model.getattr("inferred_columns")?; - if columns.is_none() { - None - } else { - let mut items: Vec<(String, Option)> = Vec::new(); - for column in columns.try_iter()? { - let column: Bound<'py, PyAny> = column?; - items.push(( - column.getattr("name")?.extract()?, - column.getattr("type")?.extract()?, - )); - } - Some(items) - } - }; + let inferred: Option)>> = inferred_rows(model)?.map(typed_names); if yml.is_empty() && inferred.as_ref().is_none_or(Vec::is_empty) { return Ok(Vec::new()); } - let inferred_schema_complete: bool = !model.getattr("fast_lineage_has_star")?.is_truthy()?; + let inferred_schema_complete: bool = !star_flags(model)?.0; schema_findings_of( SchemaColumns { yml, @@ -573,6 +561,14 @@ fn model_schema_findings<'py>( ) } +fn typed_names(rows: Vec) -> Vec<(String, Option)> { + let mut columns: Vec<(String, Option)> = Vec::with_capacity(rows.len()); + for (name, column_type, _) in rows { + columns.push((name, column_type)); + } + columns +} + fn column_pairs(columns: &Bound<'_, PyAny>) -> PyResult> { let mut pairs: Vec<(String, String)> = Vec::new(); for column in columns.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index dc1bbb01b7..47462cf513 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -10,6 +10,7 @@ use sqlbuild_analysis::planner_changes::main::origin_reference_names::origin_ref use sqlbuild_analysis::planner_changes::models::{NameSpan, RenameCandidate}; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, @@ -240,7 +241,7 @@ fn changed_referencing_query( let dialect: Option = snapshot.getattr("column_dialect")?.extract()?; for model in selected { let recorded: Bound<'_, PyAny> = - recorded_models.call_method1("get", (model.getattr("name")?,))?; + recorded_models.call_method1("get", (model_reads::name_object(&model)?,))?; if recorded.is_none() { continue; } @@ -254,9 +255,9 @@ fn changed_referencing_query( if references_models && model_query_hash( py, - &model.getattr("name")?.extract::()?, + &model_reads::model_name(&model)?, ( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(&model)?.extract::()?, dialect.as_deref(), ), )? != recorded.getattr("definition_hash")?.extract::()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index d8f8bdae34..d12fd6e241 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -1,5 +1,6 @@ //! Clone planning: destination and origin entries, and deferred-clone graph traversal. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, @@ -145,8 +146,8 @@ fn model_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model.getattr("key")?), - ("name", model.getattr("name")?), + ("key", model_reads::model_key(model)?), + ("name", model_reads::name_object(model)?), ("relative_path", model.getattr("relative_path")?), ("materialization_type", materialization_type), ("action", planner_type(py, RecordId::PlanAction, action)?), @@ -154,7 +155,7 @@ fn model_entry<'py>( "reason", planner_type(py, RecordId::PlanReason, "NO_CHANGE")?, ), - ("destination", model.getattr("destination")?), + ("destination", model_reads::destination(model)?), ("fingerprint_query_sql", string(py, "")), ("resolved_sql", resolved_sql), ("logical_ddl", string(py, "")), @@ -173,7 +174,7 @@ fn clone_model_entries<'py>( let entries = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - if !selected.contains(model.getattr("key")?)? || disabled_model(&model)? { + if !selected.contains(model_reads::model_key(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = native( @@ -237,7 +238,7 @@ fn clone_model_entries<'py>( string(py, "") }; entries.set_item( - model.getattr("key")?, + model_reads::model_key(&model)?, model_entry(&model, materialization_type, resolved_sql)?, )?; } @@ -415,7 +416,7 @@ fn origin_model_entries<'py>( let mut entries: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - if !selected_names.contains(model.getattr("name")?)? || disabled_model(&model)? { + if !selected_names.contains(model_reads::name_object(&model)?)? || disabled_model(&model)? { continue; } let materialization_type = native( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index d8d83ae37b..3edf007fad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -1,5 +1,6 @@ //! Static plans for commands that do not build models, and direct seed identity reads. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, @@ -155,7 +156,7 @@ fn build_audit_command_plan<'py>( for model in project.getattr("models")?.try_iter()? { let model = model?; materializations.set_item( - model.getattr("name")?, + model_reads::name_object(&model)?, native( py, NativeAttr::GetMaterializationType, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index a778e2173f..b56f6ed178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -1,5 +1,6 @@ //! Display-only plans for external plan surfaces when warehouse planning cannot resolve. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_entries::values::{ boolean, native, none, planner_model, planner_type, string, @@ -59,7 +60,7 @@ fn display_entry<'py>( ], )? .is_truthy()?; - let query_sql = model.getattr("query_sql")?; + let query_sql = model_reads::query_sql(model)?; let custom_name = if materialization_type.eq(CUSTOM)? { config(model, "materialized")? } else { @@ -69,8 +70,8 @@ fn display_entry<'py>( py, RecordId::ModelPlanEntry, &[ - ("key", model.getattr("key")?), - ("name", model.getattr("name")?), + ("key", model_reads::model_key(model)?), + ("name", model_reads::name_object(model)?), ("relative_path", model.getattr("relative_path")?), ( "action", @@ -93,7 +94,7 @@ fn display_entry<'py>( }, )?, ), - ("destination", model.getattr("destination")?), + ("destination", model_reads::destination(model)?), ("fingerprint_query_sql", query_sql.clone()), ("resolved_sql", query_sql), ("logical_ddl", string(py, "")), @@ -142,7 +143,7 @@ fn build_display_only_sqlbuild_plan<'py>( let mut entries: Vec> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - if selected.contains(model.getattr("name")?)? { + if selected.contains(model_reads::name_object(&model)?)? { entries.push(display_entry(&model, full_refresh)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 6fa68d01ba..a47e075b37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -1,15 +1,31 @@ //! Top-level planner orchestration producing an execution plan. use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, within, + KeywordArguments, call_with, imported, keywords, within, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::planner_changes::actions::resolve_model_actions; +use crate::bindings::_helpers::planner_changes::detect::detect_changes; use crate::bindings::_helpers::planner_entries::test_planning::{ PlanningRequest, SqlTestPlanning, start_planning, }; use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; +use crate::bindings::_helpers::planner_phases::execution_changes::{ + build_planner_entry_results, prune_planner_execution_scope, reconcile_execution_changes, +}; +use crate::bindings::_helpers::planner_phases::identities::{ + build_planner_identity_context, detect_stale_warning_changes, +}; +use crate::bindings::_helpers::planner_phases::output_assembly::{ + add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, +}; +use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; +use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; +use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -22,14 +38,10 @@ fn scopes_and_cursor_overrides<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let project = args.value("project")?; let py = project.py(); - let scopes = native( - py, - NativeAttr::ResolvePlannerScopes, - &[ - ("project", project.clone()), - ("selection", args.value("selection")?), - ("policies", args.value("policies")?), - ], + let scopes = resolve_planner_scopes( + &project, + &args.value("selection")?, + &args.value("policies")?, )?; let selected = scopes.getattr("selected_scope")?; let models_by_name = selected.getattr("models_by_name")?; @@ -43,13 +55,15 @@ fn scopes_and_cursor_overrides<'py>( } } let overrides = args.value("overrides")?; - let typed = native( + let typed = type_cursor_overrides( py, - NativeAttr::TypeCursorOverrides, - &[ - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ("selected_models", PyTuple::new(py, models)?.into_any()), - ], + Some(keywords( + py, + &[ + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ("selected_models", PyTuple::new(py, models)?.into_any()), + ], + )?), )?; Ok(( scopes, @@ -66,23 +80,25 @@ fn change_results<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py = project.py(); let versions = identities.getattr("version_identities")?; - let changes = native( + let changes = detect_changes( py, - NativeAttr::DetectChanges, - &[ - ("project", project.clone()), - ("scope", scopes.getattr("inspection_scope")?), - ("snapshot", snapshot.clone()), - ("full_refresh", overrides.getattr("full_refresh")?), - ( - "expected_version_hashes", - versions.getattr("model_version_hashes")?, - ), - ( - "expected_metadata_jsons", - versions.getattr("model_metadata_jsons")?, - ), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("scope", scopes.getattr("inspection_scope")?), + ("snapshot", snapshot.clone()), + ("full_refresh", overrides.getattr("full_refresh")?), + ( + "expected_version_hashes", + versions.getattr("model_version_hashes")?, + ), + ( + "expected_metadata_jsons", + versions.getattr("model_metadata_jsons")?, + ), + ], + )?), )?; if !policies.getattr("selection_diagnostics")?.is_truthy()? { let empty = native( @@ -95,16 +111,18 @@ fn change_results<'py>( )?; return Ok((changes, empty)); } - let stale = native( + let stale = detect_stale_warning_changes( py, - NativeAttr::DetectStaleWarningChanges, - &[ - ("project", project.clone()), - ("scopes", scopes.clone()), - ("snapshot", snapshot.clone()), - ("identities", identities.clone()), - ("execution_changes", changes.clone()), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("scopes", scopes.clone()), + ("snapshot", snapshot.clone()), + ("identities", identities.clone()), + ("execution_changes", changes.clone()), + ], + )?), )?; Ok((changes, stale)) } @@ -120,39 +138,14 @@ fn planned_output<'py>( let py = project.py(); let deferral = args.value("deferral")?; let policies = args.value("policies")?; - let warehouse = native( - py, - NativeAttr::GatherPlannerWarehouseState, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ], - )?; + let warehouse = gather_planner_warehouse_state(runtime, scopes, overrides, &deferral)?; let started = Instant::now(); - let identities = native( - py, - NativeAttr::BuildPlannerIdentityContext, - &[ - ("project", project.clone()), - ("scopes", scopes.clone()), - ( - "include_stale_warning_identities", - policies.getattr("selection_diagnostics")?, - ), - ], - )?; - native( - py, - NativeAttr::CheckSelectedScopeBuildability, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("warehouse", warehouse.clone()), - ("deferral", deferral.clone()), - ], + let identities = build_planner_identity_context( + &project, + scopes, + policies.getattr("selection_diagnostics")?.extract()?, )?; + check_selected_scope_buildability(runtime, scopes, &warehouse, &deferral)?; let snapshot = warehouse.getattr("snapshot")?; let (changes, stale_warning_changes) = change_results( &project, @@ -161,114 +154,57 @@ fn planned_output<'py>( &policies, )?; let inspection_scope = scopes.getattr("inspection_scope")?; - let resolved_actions = native( - py, - NativeAttr::ResolveModelActions, - &[ - ("scope", inspection_scope.clone()), - ("changes", changes.clone()), - ], - )?; - let source_freshness = native( - py, - NativeAttr::BuildPlannerSourceFreshnessResult, - &[ - ("project", project.clone()), - ("adapter", args.value("adapter")?), - ("connection", args.value("connection")?), - ("scope", inspection_scope), - ("relations", warehouse.getattr("inspection_relations")?), - ( - "freshness_state_schemas", - snapshot.getattr("source_freshness_state_schemas")?, - ), - ], - )?; - let pruning = native( + let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; + let source_freshness = build_planner_source_freshness_result( py, - NativeAttr::PrunePlannerExecutionScope, - &[ - ("scopes", scopes.clone()), - ("resolved_actions", resolved_actions), - ], - )?; - let reconciliation = native( - py, - NativeAttr::ReconcileExecutionChanges, - &[ - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("pruning", pruning.clone()), - ("changes", changes), - ], + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", args.value("adapter")?), + ("connection", args.value("connection")?), + ("scope", inspection_scope), + ("relations", warehouse.getattr("inspection_relations")?), + ( + "freshness_state_schemas", + snapshot.getattr("source_freshness_state_schemas")?, + ), + ], + )?), )?; - let entries = native( + let pruning = prune_planner_execution_scope(scopes, &resolved_actions)?; + let reconciliation = reconcile_execution_changes(&warehouse, &identities, &pruning, &changes)?; + let entries = build_planner_entry_results( py, - NativeAttr::BuildPlannerEntryResults, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("policies", policies.clone()), - ("deferral", deferral), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation.clone()), - ("source_freshness", source_freshness.clone()), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("policies", policies.clone()), + ("deferral", deferral), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation.clone()), + ("source_freshness", source_freshness.clone()), + ], + )?), )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); - let mut plan_output = native( - py, - NativeAttr::AssembleBasePlanOutput, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation), - ("entries", entries), - ("source_freshness", source_freshness.clone()), - ("planned_sql_tests", planned_sql_tests), - ], + let output = base_plan_output( + (runtime, &warehouse, &identities, overrides), + (&pruning, &reconciliation, &entries, &source_freshness), + &planned_sql_tests, )?; - plan_output = native( - py, - NativeAttr::WithStoragePolicies, - &[ - ("plan_output", plan_output), - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scopes", scopes.clone()), - ("policies", policies.clone()), - ], - )?; - plan_output = native( - py, - NativeAttr::WithPlanWarnings, - &[ - ("runtime", runtime.clone()), - ("scopes", scopes.clone()), - ("warehouse", warehouse), - ("identities", identities), - ("stale_warning_changes", stale_warning_changes), - ("pruning", pruning.clone()), - ("source_freshness", source_freshness.clone()), - ("plan_output", plan_output), - ("policies", policies.clone()), - ], - )?; - plan_output = native( - py, - NativeAttr::WithPlanMetadata, - &[ - ("plan_output", plan_output), - ("pruning", pruning), - ("source_freshness", source_freshness), - ("policies", policies), - ], + let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; + let output = add_plan_warnings( + output, + (scopes, &warehouse, &identities, &stale_warning_changes), + (&pruning, &source_freshness, &policies), )?; + let plan_output = + add_plan_metadata(output, &pruning, &source_freshness, &policies)?.build(py)?; let on_progress = args.value("on_progress")?; if !on_progress.is_none() { on_progress.call1((format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 31de37adfb..8ae3efa1f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -5,8 +5,8 @@ mod command_plans; mod display_plan; mod execution_plan; mod model_entries; -mod plan_output; +pub(crate) mod plan_output; pub(crate) mod registration; -mod relations; +pub(crate) mod relations; mod test_planning; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index efef439137..2b7b859fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,13 +1,17 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, imported, keywords, }; +use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_entries::values::{ COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, boolean, error_of, native, none, planner_type, replaced, string, }; -use crate::bindings::_helpers::planner_phases::full_refresh::resolve_model_full_refresh; +use crate::bindings::_helpers::planner_phases::full_refresh::{ + check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, +}; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -16,9 +20,16 @@ use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_ use crate::bindings::_helpers::planner_resolution::orchestration::{ resolve_model_hook_entries, resolve_model_sql, }; +use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ + build_cursor_input_relations, reject_unexposed_cursor_input_columns, +}; +use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; -use crate::bindings::_helpers::planner_strategy::strategy::resolve_model_plan_action; +use crate::bindings::_helpers::planner_strategy::strategy::{ + materialization_of, resolve_model_plan_action, +}; +use crate::bindings::_helpers::planner_strategy::values::text; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -27,12 +38,41 @@ use pyo3::{pyfunction, wrap_pyfunction}; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; +/// The `ModelPlanContext` fields every model's planning reads, read once per plan. +struct ContextReads<'py> { + context: Bound<'py, PyAny>, + models_by_name: Bound<'py, PyAny>, + functions_by_name: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, + source_warehouse_columns: Bound<'py, PyAny>, + model_locations: Bound<'py, PyAny>, + seed_locations: Bound<'py, PyAny>, + runtime_cursor_producer_names: Bound<'py, PyAny>, + invocation_time: Bound<'py, PyAny>, +} + +impl<'py> ContextReads<'py> { + fn of(context: Bound<'py, PyAny>) -> PyResult { + Ok(Self { + models_by_name: context.getattr("models_by_name")?, + functions_by_name: context.getattr("functions_by_name")?, + source_map: context.getattr("source_map")?, + source_warehouse_columns: context.getattr("source_warehouse_columns")?, + model_locations: context.getattr("model_locations")?, + seed_locations: context.getattr("seed_locations")?, + runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + invocation_time: context.getattr("invocation_time")?, + context, + }) + } +} + /// One model's planning request after change detection. -struct ModelPlanRequest<'py> { +struct ModelPlanRequest<'a, 'py> { model: Bound<'py, PyAny>, snapshot: Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, - context: Bound<'py, PyAny>, + context: &'a ContextReads<'py>, sql_analysis_enabled: Bound<'py, PyAny>, full_refresh: bool, overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), @@ -43,10 +83,11 @@ struct ModelPlanRequest<'py> { /// Build a model plan entry and its warnings from a resolved change result. fn plan_model_from_change<'py>( - request: &ModelPlanRequest<'py>, + request: &ModelPlanRequest<'_, 'py>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let model = &request.model; - let context = &request.context; + let reads = request.context; + let context = &reads.context; let py = model.py(); let full_refresh = boolean(py, request.full_refresh)?; let backfill = if request.full_refresh { @@ -62,60 +103,35 @@ fn plan_model_from_change<'py>( request.change_result.getattr("backfill")? }; let cursor_column = config_str(model, "cursor")?; - native( + reject_unexposed_cursor_input_columns(Some(&keywords( py, - NativeAttr::ValidateSourceCursorInputColumns, &[ ("model", model.clone()), ("cursor_column", cursor_column.clone()), - ("models_by_name", context.getattr("models_by_name")?), - ("functions_by_name", context.getattr("functions_by_name")?), - ("source_map", context.getattr("source_map")?), + ("models_by_name", reads.models_by_name.clone()), + ("functions_by_name", reads.functions_by_name.clone()), + ("source_map", reads.source_map.clone()), ( "source_warehouse_columns", - context.getattr("source_warehouse_columns")?, + reads.source_warehouse_columns.clone(), ), ], - )?; + )?))?; let resolved = resolve_model_plan_action(model, &request.change_result, request.full_refresh)?; let (action, reason) = (resolved.get_item(0)?, resolved.get_item(1)?); - let materialization_type = native( - py, - NativeAttr::GetMaterializationType, - &[("model", model.clone())], - )?; - let replaces_relation = native( - py, - NativeAttr::ReplacesIncrementalRelation, - &[ - ("materialization_type", materialization_type.clone()), - ("action", action.clone()), - ], - )? - .is_truthy()?; + let materialization_type = materialization_of(model)?; + let replaces_relation = + replaces_incremental_relation(&text(&materialization_type)?, &text(&action)?); let full_rebuild_cause = if replaces_relation { - native( - py, - NativeAttr::DescribeFullRebuildCause, - &[ - ("change_result", request.change_result.clone()), - ("full_refresh", full_refresh.clone()), - ], - )? + let cause: String = + describe_full_rebuild_cause(&request.change_result, request.full_refresh)?; + if request.protect_full_refresh && !request.full_refresh { + check_full_rebuild_allowed(model, &request.change_result, &cause)?; + } + string(py, &cause) } else { none(py) }; - if !full_rebuild_cause.is_none() && request.protect_full_refresh && !request.full_refresh { - native( - py, - NativeAttr::CheckFullRebuildAllowed, - &[ - ("model", model.clone()), - ("change_result", request.change_result.clone()), - ("full_rebuild_cause", full_rebuild_cause.clone()), - ], - )?; - } let pair = native( py, RecordId::CursorOverridePair, @@ -152,33 +168,29 @@ fn plan_model_from_change<'py>( request.external_resolver.clone(), )?; let (future_config, start_config) = cursor_safety_configs(model, context)?; - native( - py, - NativeAttr::ValidateCappedWatermarkInputs, - &[ - ("model", model.clone()), - ("models_by_name", context.getattr("models_by_name")?), - ("functions_by_name", context.getattr("functions_by_name")?), - ], + reject_capped_watermark_inputs( + model, + reads.models_by_name.clone(), + Some(reads.functions_by_name.clone()), )?; - let cursor_input_relations = native( + let cursor_input_relations = build_cursor_input_relations(Some(&keywords( py, - NativeAttr::BuildCursorInputRelations, &[ ("model", model.clone()), ("adapter", request.adapter.clone()), - ("model_locations", context.getattr("model_locations")?), - ("models_by_name", context.getattr("models_by_name")?), - ("functions_by_name", context.getattr("functions_by_name")?), - ("seed_locations", context.getattr("seed_locations")?), - ("source_map", context.getattr("source_map")?), + ("model_locations", reads.model_locations.clone()), + ("models_by_name", reads.models_by_name.clone()), + ("functions_by_name", reads.functions_by_name.clone()), + ("seed_locations", reads.seed_locations.clone()), + ("source_map", reads.source_map.clone()), ("cursor_column", cursor_column.clone()), ( "runtime_cursor_producer_names", - context.getattr("runtime_cursor_producer_names")?, + reads.runtime_cursor_producer_names.clone(), ), ], - )?; + )?))? + .into_any(); let mut any_runtime_owned = false; for relation in cursor_input_relations.try_iter()? { if relation?.getattr("is_runtime_owned")?.is_truthy()? { @@ -194,7 +206,7 @@ fn plan_model_from_change<'py>( overrides: request.overrides.clone(), runtime_owned, policies: (future_config.clone(), start_config.clone()), - invocation_time: context.getattr("invocation_time")?, + invocation_time: reads.invocation_time.clone(), full_refresh: request.full_refresh, replaces_relation, cursor_input_relations: cursor_input_relations.clone(), @@ -231,7 +243,7 @@ fn plan_model_from_change<'py>( ("end_cursor_override", request.overrides.1.clone()), ("future_cursor_config", future_config), ("start_cursor_config", start_config), - ("invocation_time", context.getattr("invocation_time")?), + ("invocation_time", reads.invocation_time.clone()), ], )?))?; let bounds = if range.is_truthy()? { @@ -239,8 +251,11 @@ fn plan_model_from_change<'py>( } else { cursor_bounds }; - let warnings = - with_cursor_safety_warnings(&assembled.get_item(1)?, &model.getattr("name")?, &bounds)?; + let warnings = with_cursor_safety_warnings( + &assembled.get_item(1)?, + &model_reads::name_object(model)?, + &bounds, + )?; Ok((assembled.get_item(0)?, warnings)) } @@ -479,6 +494,9 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult ("invocation_time", inputs.getattr("invocation_time")?), ], )?; + let reads = ContextReads::of(context)?; + let sql_analysis_enabled = settings.getattr("sql_analysis")?; + let external_resolver = project.getattr("external_sql_reference_resolver")?; let skip = planner_type(py, RecordId::PlanAction, "SKIP")?; let microbatch = registry::class(py, RecordId::IncrementalMode)?.getattr("MICROBATCH")?; let mut entries: Vec> = Vec::new(); @@ -504,12 +522,12 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult model: model.clone(), snapshot: snapshot.clone(), adapter: adapter.clone(), - context: context.clone(), - sql_analysis_enabled: settings.getattr("sql_analysis")?, + context: &reads, + sql_analysis_enabled: sql_analysis_enabled.clone(), full_refresh: effective_full_refresh, overrides, change_result: resolved.getattr("change")?, - external_resolver: project.getattr("external_sql_reference_resolver")?, + external_resolver: external_resolver.clone(), protect_full_refresh: !freshness_blocked.contains(&name)? && !external_blocked.contains(&name)?, })?; @@ -646,7 +664,7 @@ fn plan_model<'py>( model, snapshot, adapter: args.value("adapter")?, - context: args.value("context")?, + context: &ContextReads::of(args.value("context")?)?, sql_analysis_enabled: args.value("sql_analysis_enabled")?, full_refresh: args.flag("full_refresh")?, overrides: ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 7e21078603..87d8d2f91b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,12 +1,15 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -474,28 +477,26 @@ pub(super) fn selected_test_entries<'py>( } let analysis = project.getattr("settings")?.getattr("sql_analysis")?; let fixture_context = if analysis.is_truthy()? { - native( - py, - NativeAttr::BuildRelationFixtureContext, - &[("project", project.clone())], - )? + Bound::new(py, planning_context(project, None)?)?.into_any() } else { none(py) }; - let results = native( + let results = assemble_sql_test_plans( py, - NativeAttr::PlanSqlTests, - &[ - ("tests", tuple_of(py, selected)?), - ("project", project.clone()), - ("adapter", adapter.clone()), - ("sql_analysis_enabled", analysis), - ( - "validate_fixtures", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("fixture_planning_context", fixture_context), - ], + Some(keywords( + py, + &[ + ("tests", tuple_of(py, selected)?), + ("project", project.clone()), + ("adapter", adapter.clone()), + ("sql_analysis_enabled", analysis), + ( + "validate_fixtures", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("fixture_planning_context", fixture_context), + ], + )?), )?; let mut diagnostics: Vec = Vec::new(); for result in results.try_iter()? { @@ -544,7 +545,10 @@ fn planned_sql_tests<'py>( ) } -fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> { +/// The plan output's fields from planned entries and resolved relation inputs. +pub(crate) fn plan_output_fields<'py>( + args: &KeywordArguments<'py>, +) -> PyResult)>> { let project = args.value("project")?; let py = project.py(); let adapter = args.value("adapter")?; @@ -627,65 +631,59 @@ fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> for warning in planned.getattr("warnings")?.try_iter()? { warnings.push(warning?); } - planner_model( - py, - RecordId::PlanOutput, - &[ - ("execution_order", tuple_of(py, scoped_order)?), - ("model_entries", model_entries.clone()), - ("seed_entries", tuple_of(py, seeds)?), - ( - "source_load_entries", - PyTuple::new( - py, - source_load_entries - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ("function_entries", tuple_of(py, functions)?), - ( - "audit_entries", - PyTuple::new(py, audit_entries.try_iter()?.collect::>>()?)? - .into_any(), - ), - ( - "test_entries", - PyTuple::new(py, test_entries.try_iter()?.collect::>>()?)? - .into_any(), - ), - ("selected_keys", selected_keys), - ("warnings", tuple_of(py, warnings)?), - ("upstream_deps", scope.getattr("upstream_deps")?), - ("downstream_deps", scope.getattr("downstream_deps")?), - ("model_locations", relations.getattr("model_locations")?), - ("seed_locations", relations.getattr("seed_locations")?), - ( - "function_locations", - relations.getattr("function_locations")?, - ), - ("source_map", source_map), - ("source_read_map", relations.getattr("source_read_map")?), - ( - "python_source_read_map", - relations.getattr("python_source_read_map")?, - ), - ("hook_functions", project.getattr("hook_functions")?), - ( - "enforce_explicit_references", - project.getattr("enforce_explicit_references")?, - ), - ( - "provider_usages", - plan_provider_usages(&project, &model_entries, &source_load_entries)?, - ), - ( - "python_identity_fingerprints", - snapshot.getattr("fingerprints")?.getattr("python_nodes")?, - ), - ], - ) + Ok(vec![ + ("execution_order", tuple_of(py, scoped_order)?), + ("model_entries", model_entries.clone()), + ("seed_entries", tuple_of(py, seeds)?), + ( + "source_load_entries", + PyTuple::new( + py, + source_load_entries + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ("function_entries", tuple_of(py, functions)?), + ( + "audit_entries", + PyTuple::new(py, audit_entries.try_iter()?.collect::>>()?)?.into_any(), + ), + ( + "test_entries", + PyTuple::new(py, test_entries.try_iter()?.collect::>>()?)?.into_any(), + ), + ("selected_keys", selected_keys), + ("warnings", tuple_of(py, warnings)?), + ("upstream_deps", scope.getattr("upstream_deps")?), + ("downstream_deps", scope.getattr("downstream_deps")?), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", source_map), + ("source_read_map", relations.getattr("source_read_map")?), + ( + "python_source_read_map", + relations.getattr("python_source_read_map")?, + ), + ("hook_functions", project.getattr("hook_functions")?), + ( + "enforce_explicit_references", + project.getattr("enforce_explicit_references")?, + ), + ( + "provider_usages", + plan_provider_usages(&project, &model_entries, &source_load_entries)?, + ), + ( + "python_identity_fingerprints", + snapshot.getattr("fingerprints")?.getattr("python_nodes")?, + ), + ]) } /// Assemble the plan output from planned entries and resolved relation inputs. @@ -699,21 +697,9 @@ fn build_plan_output<'py>( py, "build_plan_output", kwargs, - ( - &[ - "project", - "adapter", - "scope", - "snapshot", - "relations", - "changes", - "model_entry_results", - "reload_sources", - ], - &["extras"], - ), + (PLAN_OUTPUT_INPUTS, &["extras"]), )?; - plan_output(&args) + planner_model(py, RecordId::PlanOutput, &plan_output_fields(&args)?) } /// Build one canonical function entry from resolved relation inputs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 723a14b74c..07db82f0c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -211,7 +211,7 @@ pub(crate) fn planner_relations_context<'py>( /// Resolve relation locations and source metadata for plan entry construction. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn build_planner_relations_context<'py>( +pub(crate) fn build_planner_relations_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 4b795d067c..9ecd1ba24a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -14,6 +14,7 @@ use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::grap use sqlbuild_analysis::planner_identity::main::model_local_identity_hash::model_local_identity_hash; use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -149,7 +150,7 @@ fn build_direct_model_version_identities<'py>( if model.is_none() { continue; } - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(&model)?; let metadata_json: String = model_version_identity_metadata_json( &model, Some(function_hashes.as_any()), @@ -159,7 +160,7 @@ fn build_direct_model_version_identities<'py>( py, &name, ( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(&model)?.extract::()?, dialect.as_deref(), ), )?; @@ -276,7 +277,7 @@ fn with_honest_model_write_hashes<'py>( .try_iter()? { let model: Bound<'py, PyAny> = model?; - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; graph.nodes.insert( key_of(MODEL_RESOURCE, &name.extract::()?), IdentityNode { @@ -323,7 +324,7 @@ fn with_honest_model_write_hashes<'py>( if model.is_none() { continue; } - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; if local_hashes.call_method1("get", (&name,))?.is_none() { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs index 5ac2e6cc45..35d06a6406 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -14,6 +14,7 @@ use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::functio use sqlbuild_analysis::planner_identity::main::stable_identity_hash::stable_identity_hash; use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords, within}; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::planner_identity::hashing::function_definition_hash; @@ -102,7 +103,7 @@ pub(crate) fn model_version_identity_metadata_json<'py>( let signature: Bound<'py, PyDict> = execution_signature(model, &config_values, hook_version_hashes)?; version_identity_metadata_json( - &model.getattr("name")?, + &model_reads::name_object(model)?, &config_values, ( Some(local_function_hashes.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index b209ee41a1..4ba5097a0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -1,5 +1,6 @@ //! Gather live candidate evidence without changing adapter inspection ownership. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, @@ -34,7 +35,7 @@ pub(super) fn candidates<'py>( if !materialized.eq("incremental")? && !materialized.eq("snapshot")? { continue; } - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; if args.value("full_refresh_model_names")?.contains(&name)? || !snapshot.getattr("existing_relations")?.contains(&name)? { @@ -86,7 +87,7 @@ pub(super) fn candidates<'py>( py, NativeAttr::ParseQueryShape, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(&model)?), ("dialect", dialect.clone()), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 9f5685f3ca..9a8539396a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -1,5 +1,7 @@ //! Materialize column decisions from native proofs and live evidence. +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -52,15 +54,12 @@ pub(super) fn decide<'py>( outputs.push(projection?.getattr("key")?.extract::()?); } } else { - let inferred = candidate.model.getattr("inferred_columns")?; - if !inferred.is_none() { - for column in inferred.try_iter()? { - outputs.push(lower(&column?.getattr("name")?)?); - } + for (name, _, _) in inferred_rows(&candidate.model)?.unwrap_or_default() { + outputs.push(name.to_lowercase()); } } let relation = host::entry(py, ExecutorHost::MigrationRelationForLocation)? - .call1((candidate.model.getattr("destination")?,))?; + .call1((model_reads::destination(&candidate.model)?,))?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; @@ -98,8 +97,8 @@ pub(super) fn decide<'py>( py, RecordId::ColumnMigrationPlanEntry, &[ - ("model_name", candidate.model.getattr("name")?), - ("destination", candidate.model.getattr("destination")?), + ("model_name", model_reads::name_object(&candidate.model)?), + ("destination", model_reads::destination(&candidate.model)?), ( "origin_column", if let Some(origin) = origin { @@ -155,7 +154,7 @@ pub(super) fn checked<'py>( warnings: Vec::new(), }); } - let destination = candidate.model.getattr("destination")?; + let destination = model_reads::destination(&candidate.model)?; let qualified = destination.getattr("qualified_name")?; let relation = if physical.is_truthy()? { physical.clone() @@ -227,7 +226,7 @@ pub(super) fn hints<'py>( py, NativeAttr::RenameHints, &[ - ("model_name", candidate.model.getattr("name")?), + ("model_name", model_reads::name_object(&candidate.model)?), ("previous", candidate.previous.clone()), ("current", candidate.current.clone()), ("excluded", excluded.call_method1("union", (claimed,))?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs index 50b4e856d7..e7f00d1aa4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_objects.rs @@ -1,5 +1,6 @@ //! Boundary conversions for the column migration planner. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::records::ids::{NativeAttr, NativeItem, RecordId}; @@ -57,8 +58,8 @@ pub(super) fn definition_hash<'py>( model.py(), NativeAttr::ModelDefinitionHash, &[ - ("model_name", model.getattr("name")?), - ("query_sql", model.getattr("query_sql")?), + ("model_name", model_reads::name_object(model)?), + ("query_sql", model_reads::query_sql(model)?), ("dialect", snapshot.getattr("column_dialect")?), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 0204ab771f..2aa146b874 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,5 +1,6 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -27,7 +28,7 @@ pub(super) fn overlay<'py>( let stored = snapshot.getattr("fingerprints")?; let fingerprints = imported(py, "builtins", "dict")?.call1((stored.getattr("models")?,))?; for candidate in candidates { - let name = candidate.model.getattr("name")?; + let name = model_reads::name_object(&candidate.model)?; let renames = PyDict::new(py); let pending = PyDict::new(py); let mut blocked = false; @@ -83,7 +84,7 @@ pub(super) fn overlay<'py>( replace( &candidate.fingerprint, &[ - ("definition", candidate.model.getattr("query_sql")?), + ("definition", model_reads::query_sql(&candidate.model)?), ( "definition_hash", definition_hash(&candidate.model, snapshot)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 902ac4cfec..33c059bba3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -1,5 +1,6 @@ //! Host column migration planning while Python owns warehouse inspection. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::replace; @@ -13,7 +14,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; /// Decide column renames and return the existing planning result model. #[pyfunction] #[pyo3(signature = (**options))] -fn plan_column_migrations<'py>( +pub(crate) fn plan_column_migrations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -63,7 +64,7 @@ fn plan_column_migrations<'py>( )?; let schemas = PySet::empty(py)?; for candidate in &candidates { - let schema = candidate.model.getattr("destination")?.getattr("schema")?; + let schema = model_reads::destination(&candidate.model)?.getattr("schema")?; if !schema.is_none() { schemas.add(schema)?; } @@ -92,7 +93,7 @@ fn plan_column_migrations<'py>( )?; let physical = args .value("physical_relations")? - .call_method1("get", (candidate.model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(&candidate.model)?,))?; let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { hints.push(hint?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 86b6de5fd3..da8768b2ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -1,5 +1,7 @@ //! Convert existing planner contracts for origin compatibility checks. +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::analysis::model_reads::inferred_rows; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -135,9 +137,12 @@ fn check_migration_compatibility<'py>( ); } let relations = PyDict::new(py); - relations.set_item(model.getattr("name")?, relation)?; + relations.set_item(model_reads::name_object(&model)?, relation)?; let columns = PyDict::new(py); - columns.set_item(model.getattr("name")?, arguments.value("origin_columns")?)?; + columns.set_item( + model_reads::name_object(&model)?, + arguments.value("origin_columns")?, + )?; let snapshot = call_with( ®istry::class(py, RecordId::WarehouseSnapshot)?, &[ @@ -166,7 +171,7 @@ fn check_migration_compatibility<'py>( ], )?; let schema = model.getattr("schema_entry")?; - let mut has_columns = model.getattr("inferred_columns")?.is_truthy()?; + let mut has_columns = inferred_rows(&model)?.is_some_and(|rows| !rows.is_empty()); if !schema.is_none() { for column in schema.getattr("columns")?.try_iter()? { has_columns |= !column?.getattr("type")?.is_none(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 1a667a1e29..0ce873333a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -1,5 +1,6 @@ //! Migration identity binding with unchanged metadata and hash format. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -101,7 +102,7 @@ fn with_migration_fingerprints<'py>( let fingerprint = call_with( &cache.getattr("fingerprint")?, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(&model)?), ("metadata_json", metadata), ("ref_identities", PyDict::new(py).into_any()), ("dialect", arguments.value("dialect")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs index a8c06036a3..f8106545b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/matching.rs @@ -1,5 +1,6 @@ //! Fingerprint cache callbacks around native one-to-one migration matching. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -14,7 +15,7 @@ fn variants<'py>( model: &Bound<'py, PyAny>, maps: Bound<'py, PyAny>, ) -> PyResult> { - let name = model.getattr("name")?; + let name = model_reads::name_object(model)?; let metadata = arguments.value("metadata_jsons")?.get_item(name)?; let cache = arguments.value("fingerprints")?.getattr("fingerprint")?; let mut hashes: Vec = Vec::new(); @@ -22,7 +23,7 @@ fn variants<'py>( let hash: Option = call_with( &cache, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(model)?), ("metadata_json", metadata.clone()), ("ref_identities", map?), ("dialect", arguments.value("dialect")?), @@ -67,7 +68,7 @@ fn match_migration_fingerprints<'py>( let mut names: Vec = Vec::new(); for model in arguments.value("models")?.try_iter()? { let model = model?; - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(&model)?; if new_names.contains(&name) { names.push(name.clone()); } @@ -131,7 +132,7 @@ fn migration_destination_fingerprints<'py>( let result = PyDict::new(py); for request in arguments.value("requests")?.try_iter()? { let model = request?.getattr("model")?; - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; if arguments .value("metadata_jsons")? .call_method1("get", (&name,))? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index a6a080315f..44516c9d52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -2,7 +2,7 @@ mod column_candidates; mod column_entries; mod column_objects; mod column_overlay; -mod column_planning; +pub(crate) mod column_planning; mod column_warnings; pub(crate) mod columns; mod compatibility; @@ -18,12 +18,12 @@ mod matching; mod model_entries; mod model_handover; mod model_overlay; -mod model_planning; +pub(crate) mod model_planning; mod model_warnings; mod models; mod old_name_entries; mod old_name_objects; -mod old_name_planning; +pub(crate) mod old_name_planning; mod old_name_references; mod old_names; mod planning_boundary; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index 80672798c7..e46ef09a7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -1,5 +1,6 @@ //! Adapt immutable migration fingerprints without changing their persisted representation. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_migrations::column_objects::{native, replace, text}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; @@ -17,7 +18,7 @@ pub(super) fn applies( let own = snapshot .getattr("fingerprints")? .getattr("models")? - .call_method1("get", (model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(model)?,))?; Ok(own.is_none() || presumed_utc(&own.getattr("ts")?)? < presumed_utc(&event.getattr("created_at")?)?) } @@ -44,18 +45,18 @@ pub(super) fn handover<'py>( if let Ok(payload) = payload.cast::() && payload.contains(key)? { - payload.set_item(key, model.getattr("name")?)?; + payload.set_item(key, model_reads::name_object(model)?)?; let kwargs = PyDict::new(py); kwargs.set_item("sort_keys", true)?; kwargs.set_item("separators", (",", ":"))?; kwargs.set_item("default", imported(py, "builtins", "str")?)?; metadata = json.getattr("dumps")?.call((payload,), Some(&kwargs))?; } - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; replace( fingerprint, &[ - ("node_name", model.getattr("name")?), + ("node_name", model_reads::name_object(model)?), ("target_database", destination.getattr("database")?), ("target_schema", destination.getattr("schema")?), ("target_name", destination.getattr("name")?), @@ -89,14 +90,14 @@ pub(super) fn equivalent<'py>( replace( &handover, &[ - ("definition", model.getattr("query_sql")?), + ("definition", model_reads::query_sql(model)?), ( "definition_hash", native( py, NativeAttr::ComputeQueryHash, &[ - ("query_sql", model.getattr("query_sql")?), + ("query_sql", model_reads::query_sql(model)?), ("dialect", dialect.clone()), ], )?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 687f6d4239..f16dd1a153 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -102,7 +102,7 @@ fn decide_model_migration<'py>( #[pyfunction] #[pyo3(signature = (**options))] -fn plan_model_migrations<'py>( +pub(crate) fn plan_model_migrations<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 3fe90cdd55..2b506c7163 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -69,7 +69,7 @@ fn schemas<'py>( #[pyfunction] #[pyo3(signature = (**options))] -fn plan_old_name_views<'py>( +pub(crate) fn plan_old_name_views<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index f8116bae5b..8ebfdd6972 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -1,5 +1,6 @@ //! Existing old-name planner models at the native decision boundary. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -109,7 +110,7 @@ fn old_name_claim_errors<'py>( let mut errors: Vec> = Vec::new(); for model in arguments.value("selected")?.try_iter()? { let model = model?; - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; let existing = arguments .value("snapshot")? .getattr("existing_relations")? @@ -122,7 +123,7 @@ fn old_name_claim_errors<'py>( if !kind.eq("view")? { continue; } - let destination = relation(&model.getattr("destination")?)?; + let destination = relation(&model_reads::destination(&model)?)?; for view in arguments.value("views")?.try_iter()? { let view = view?; if view.getattr("destination_model")?.eq(&name)? @@ -177,11 +178,11 @@ fn old_name_view_skip_reason<'py>( let mut reuser: Option = None; for model in arguments.value("models")?.try_iter()? { let model = model?; - if relation(&model.getattr("destination")?)? + if relation(&model_reads::destination(&model)?)? .call_method1("matches", (&old,))? .is_truthy()? { - reuser = Some(model.getattr("name")?.extract()?); + reuser = Some(model_reads::name_object(&model)?.extract()?); break; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index 5c45d15791..ac9c86d04a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -12,3 +12,14 @@ pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; pub(crate) const RETENTION_DECREASE: &str = "decrease"; pub(crate) const RETENTION_INCREASE: &str = "increase"; pub(crate) const RETENTION_MATCH: &str = "match"; +/// The required inputs of plan output assembly. +pub(crate) const PLAN_OUTPUT_INPUTS: &[&str] = &[ + "project", + "adapter", + "scope", + "snapshot", + "relations", + "changes", + "model_entry_results", + "reload_sources", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs index 63b451f15c..5e578431fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/full_refresh.rs @@ -1,5 +1,6 @@ //! Per-model full-refresh resolution and full-rebuild refusals. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_phases::constants::FULL_REFRESH_DISABLED_REBUILD_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, planner_error, strings}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -40,7 +41,7 @@ pub(crate) fn effectively_full_refreshed_model_names<'py>( for model in project.getattr("models")?.try_iter()? { let model = model?; if resolve_model_full_refresh(&model, cli_full_refresh)? { - names.push(model.getattr("name")?); + names.push(model_reads::name_object(&model)?); } } frozenset(project.py(), &names) @@ -80,7 +81,7 @@ pub(crate) fn check_full_rebuild_allowed( { return Ok(()); } - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(model)?; Err(planner_error( py, &format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index 32482259eb..e750afe4fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod constants; pub(crate) mod execution_changes; pub(crate) mod full_refresh; pub(crate) mod identities; +pub(crate) mod models; pub(crate) mod objects; pub(crate) mod output_assembly; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs new file mode 100644 index 0000000000..0fb08904de --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -0,0 +1,10 @@ +//! Plan output fields gathered across the plan phases before the output is built once. + +use pyo3::prelude::{Bound, PyAny}; + +/// The plan output's fields and metadata entries, each set before the output is built. +#[derive(Debug)] +pub(crate) struct PlanOutputFields<'py> { + pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, + pub(crate) metadata: Vec<(&'static str, Bound<'py, PyAny>)>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 5ce7e353c0..3bf915e1a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,7 +1,12 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; +use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; +use crate::bindings::_helpers::planner_phases::models::PlanOutputFields; use crate::bindings::_helpers::planner_phases::objects::{ model_key_names, native, replace, sorted_names, strings, text, tuple, }; @@ -10,7 +15,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use std::collections::BTreeSet; @@ -24,57 +28,51 @@ fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult( - plan_output: &Bound<'py, PyAny>, - items: &[(&str, Bound<'py, PyAny>)], -) -> PyResult> { - let py = plan_output.py(); - let metadata = PyDict::new(py); - metadata.update( - plan_output - .getattr("metadata")? - .downcast::()?, - )?; - for (name, value) in items { - metadata.set_item(*name, value)?; +impl<'py> PlanOutputFields<'py> { + fn field(&self, name: &str) -> PyResult> { + self.fields + .iter() + .find(|(field, _)| *field == name) + .map(|(_, value)| value.clone()) + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(name.to_owned())) } - replace(plan_output, &[("metadata", metadata.into_any())]) -} -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn assemble_base_plan_output<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "assemble_base_plan_output", - options, - ( - &[ - "runtime", - "warehouse", - "identities", - "overrides", - "pruning", - "reconciliation", - "entries", - "source_freshness", - ], - &["planned_sql_tests"], - ), - )?; - let runtime = &args.value("runtime")?; - let warehouse = &args.value("warehouse")?; - let identities = &args.value("identities")?; - let overrides = &args.value("overrides")?; - let pruning = &args.value("pruning")?; - let reconciliation = &args.value("reconciliation")?; - let entries = &args.value("entries")?; - let source_freshness = &args.value("source_freshness")?; - let planned_sql_tests = &args.value("planned_sql_tests")?; + fn set(&mut self, name: &'static str, value: Bound<'py, PyAny>) { + match self.fields.iter_mut().find(|(field, _)| *field == name) { + Some(existing) => existing.1 = value, + None => self.fields.push((name, value)), + } + } + + /// The `PlanOutput` hand-off, built once from every field the phases set. + pub(crate) fn build(mut self, py: Python<'py>) -> PyResult> { + if !self.metadata.is_empty() { + let metadata = PyDict::new(py); + for (name, value) in &self.metadata { + metadata.set_item(*name, value)?; + } + self.set("metadata", metadata.into_any()); + } + call_with(®istry::class(py, RecordId::PlanOutput)?, &self.fields) + } +} +/// The plan output's base fields from the planned entries, relations and source freshness. +pub(crate) fn base_plan_output<'py>( + (runtime, warehouse, identities, overrides): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + (pruning, reconciliation, entries, source_freshness): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + planned_sql_tests: &Bound<'py, PyAny>, +) -> PyResult> { let py = runtime.py(); let version = identities.getattr("version_identities")?; let extras = native( @@ -92,8 +90,8 @@ pub(crate) fn assemble_base_plan_output<'py>( ("planned_sql_tests", planned_sql_tests.clone()), ], )?; - let plan_output = call_with( - ®istry::native(py, NativeAttr::BuildPlanOutput)?, + let inputs = keywords( + py, &[ ("project", runtime.getattr("project")?), ("adapter", runtime.getattr("adapter")?), @@ -109,18 +107,22 @@ pub(crate) fn assemble_base_plan_output<'py>( ("extras", extras), ], )?; - let plan_output = replace( - &plan_output, - &[ - ("source_freshness", source_freshness.clone()), - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ], - )?; + let mut output = PlanOutputFields { + fields: plan_output_fields(&KeywordArguments::read( + py, + "build_plan_output", + Some(inputs), + (PLAN_OUTPUT_INPUTS, &["extras"]), + )?)?, + metadata: Vec::new(), + }; + output.set("source_freshness", source_freshness.clone()); + output.set("cursor_overrides", overrides.getattr("cursor_overrides")?); let pruned = pruning.getattr("pruned_direct_model_names")?; if pruned.is_truthy()? { - return with_metadata(&plan_output, &[("direct_pruned_model_names", pruned)]); + output.metadata.push(("direct_pruned_model_names", pruned)); } - Ok(plan_output) + Ok(output) } /// Attach near-match column rename hints to the model entries they describe. @@ -158,29 +160,18 @@ pub(crate) fn with_column_rename_hints<'py>( tuple(py, attached) } -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn with_storage_policies<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "with_storage_policies", - options, - ( - &["plan_output", "runtime", "warehouse", "scopes", "policies"], - &[], - ), - )?; - let plan_output = &args.value("plan_output")?; - let runtime = &args.value("runtime")?; - let warehouse = &args.value("warehouse")?; - let scopes = &args.value("scopes")?; - let policies = &args.value("policies")?; - +/// Add migration fingerprints, rename hints, migration, table-type and retention entries. +pub(crate) fn add_storage_policies<'py>( + mut output: PlanOutputFields<'py>, + (runtime, warehouse, scopes, policies): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { let py = runtime.py(); - let mut entries = plan_output.getattr("model_entries")?; + let mut entries = output.field("model_entries")?; if policies .getattr("record_migration_fingerprints")? .is_truthy()? @@ -188,7 +179,7 @@ pub(crate) fn with_storage_policies<'py>( let models = PyDict::new(py); for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } entries = native( py, @@ -215,7 +206,7 @@ pub(crate) fn with_storage_policies<'py>( py, &[ warehouse.getattr("migration_warnings")?, - plan_output.getattr("warnings")?, + output.field("warnings")?, ], )?; let selected = scopes.getattr("selected_scope")?; @@ -226,65 +217,40 @@ pub(crate) fn with_storage_policies<'py>( ]; let table_types = native(py, NativeAttr::PlanTableTypes, &storage)?; let retention = native(py, NativeAttr::PlanRetention, &storage)?; - replace( - plan_output, - &[ - ("model_entries", entries), - ("migration_entries", migration_entries), - ("column_migration_entries", column_entries), - ("old_name_view_entries", old_name_entries), - ("old_name_views", old_name_views), - ("warnings", warnings), - ("table_type_entries", table_types), - ("retention_entries", retention), - ], - ) + output.set("model_entries", entries); + output.set("migration_entries", migration_entries); + output.set("column_migration_entries", column_entries); + output.set("old_name_view_entries", old_name_entries); + output.set("old_name_views", old_name_views); + output.set("warnings", warnings); + output.set("table_type_entries", table_types); + output.set("retention_entries", retention); + Ok(output) } -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn with_plan_warnings<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "with_plan_warnings", - options, - ( - &[ - "runtime", - "scopes", - "warehouse", - "identities", - "stale_warning_changes", - "pruning", - "source_freshness", - "plan_output", - "policies", - ], - &[], - ), - )?; - let runtime = &args.value("runtime")?; - let scopes = &args.value("scopes")?; - let warehouse = &args.value("warehouse")?; - let identities = &args.value("identities")?; - let stale_warning_changes = &args.value("stale_warning_changes")?; - let pruning = &args.value("pruning")?; - let source_freshness = &args.value("source_freshness")?; - let plan_output = &args.value("plan_output")?; - let policies = &args.value("policies")?; - - let py = runtime.py(); - let mut plan_output = plan_output.clone(); +/// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. +pub(crate) fn add_plan_warnings<'py>( + mut output: PlanOutputFields<'py>, + (scopes, warehouse, identities, stale_warning_changes): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + (pruning, source_freshness, policies): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let py = scopes.py(); let freshness = source_freshness_unknown_warnings(source_freshness)?; if freshness.is_truthy()? { - let warnings = concat(py, &[plan_output.getattr("warnings")?, freshness])?; - plan_output = replace(&plan_output, &[("warnings", warnings)])?; + let warnings = concat(py, &[output.field("warnings")?, freshness])?; + output.set("warnings", warnings); } if !policies.getattr("selection_diagnostics")?.is_truthy()? { - return Ok(plan_output); + return Ok(output); } let stale = native( py, @@ -310,10 +276,10 @@ pub(crate) fn with_plan_warnings<'py>( ], )?; if stale.is_truthy()? { - let warnings = concat(py, &[plan_output.getattr("warnings")?, stale])?; - plan_output = replace(&plan_output, &[("warnings", warnings)])?; + let warnings = concat(py, &[output.field("warnings")?, stale])?; + output.set("warnings", warnings); } - Ok(plan_output) + Ok(output) } fn source_names<'py>(items: &Bound<'py, PyAny>) -> PyResult> { @@ -386,15 +352,14 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( - plan_output: &Bound<'py, PyAny>, +/// Add the direct-mode source freshness, remaining stale models and selection diagnostics metadata. +pub(crate) fn add_plan_metadata<'py>( + mut output: PlanOutputFields<'py>, pruning: &Bound<'py, PyAny>, source_freshness: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, -) -> PyResult> { - let py = plan_output.py(); +) -> PyResult> { + let py = pruning.py(); let selected: BTreeSet = model_key_names( &pruning .getattr("inspection_scope")? @@ -411,18 +376,16 @@ pub(crate) fn with_plan_metadata<'py>( let diagnostics = PyDict::new(py); diagnostics.set_item("mode", text(py, "direct"))?; diagnostics.set_item("enabled", policies.getattr("selection_diagnostics")?)?; - with_metadata( - plan_output, - &[ - ( - "direct_source_freshness", - freshness_metadata(source_freshness)?, - ), - ( - "direct_remaining_stale_model_names", - sorted_names(py, remaining)?, - ), - ("selection_diagnostics", diagnostics.into_any()), - ], - ) + output.metadata.push(( + "direct_source_freshness", + freshness_metadata(source_freshness)?, + )); + output.metadata.push(( + "direct_remaining_stale_model_names", + sorted_names(py, remaining)?, + )); + output + .metadata + .push(("selection_diagnostics", diagnostics.into_any())); + Ok(output) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index fe50f5e71e..66fc997a07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -61,26 +61,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(super::retention::plan_retention, module)?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::assemble_base_plan_output, - module - )?)?; module.add_function(wrap_pyfunction!( super::output_assembly::with_column_rename_hints, module )?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::with_storage_policies, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::with_plan_warnings, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::output_assembly::with_plan_metadata, - module - )?)?; module.add_function(wrap_pyfunction!( super::warehouse_state::gather_planner_warehouse_state, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index ff3d379738..0559d5227e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -1,5 +1,6 @@ //! Warehouse-aware time-travel retention and Snowflake table-type planning. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -65,7 +66,7 @@ fn selected_models<'py>( let mut models: Vec<_> = Vec::new(); for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { let model = model?; - if names.contains(&model.getattr("name")?.extract::()?) { + if names.contains(&model_reads::model_name(&model)?) { models.push(model); } } @@ -98,7 +99,7 @@ pub(crate) fn plan_table_types<'py>( .getattr("declared")? .is_truthy()? { - declared.push(model.getattr("name")?.extract::()?); + declared.push(model_reads::model_name(model)?); } } if !declared.is_empty() && !snowflake { @@ -135,7 +136,7 @@ pub(crate) fn plan_table_types<'py>( if !backed.is_truthy()? { continue; } - let relation = relations.call_method1("get", (model.getattr("name")?,))?; + let relation = relations.call_method1("get", (model_reads::name_object(&model)?,))?; if relation.is_none() { continue; } @@ -151,7 +152,7 @@ pub(crate) fn plan_table_types<'py>( continue; } let downgrade = actual == Some(false) && desired_transient; - let destination = model.getattr("destination")?; + let destination = model_reads::destination(&model)?; let limit: i64 = adapter .call_method0("maximum_identifier_length")? .extract()?; @@ -160,7 +161,7 @@ pub(crate) fn plan_table_types<'py>( py, RecordId::TableTypePlanEntry, &[ - ("model_name", model.getattr("name")?), + ("model_name", model_reads::name_object(&model)?), ("destination", destination), ("copy_name", text(py, ©_name(&target_name, limit))), ("desired_type", table_type.getattr("value")?.getattr("value")?), @@ -317,7 +318,7 @@ fn relation_retention<'py>( model: &Bound<'py, PyAny>, ) -> PyResult>> { let py = runtime.py(); - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(model)?; let config = model.getattr("config")?; let retention = config.getattr("time_travel_retention")?; if retention.getattr("desired_days")?.is_none() || retention.getattr("source")?.is_none() { @@ -328,7 +329,7 @@ fn relation_retention<'py>( )); } let desired_days: i64 = retention.getattr("desired_days")?.extract()?; - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; let schema = destination.getattr("schema")?; if schema.is_none() { return Err(planner_error( @@ -341,7 +342,7 @@ fn relation_retention<'py>( py, AdapterHost::RetentionRequest, &[ - ("request_id", model.getattr("name")?), + ("request_id", model_reads::name_object(model)?), ( "scope", member(py, AdapterHost::RetentionScope, "relation")?, @@ -354,7 +355,7 @@ fn relation_retention<'py>( )?; let mut base = RetentionEntry { request: request.clone(), - model_names: tuple(py, vec![model.getattr("name")?])?, + model_names: tuple(py, vec![model_reads::name_object(model)?])?, state: None, source: retention.getattr("source")?.getattr("value")?, }; @@ -363,7 +364,7 @@ fn relation_retention<'py>( let relations = warehouse .getattr("snapshot")? .getattr("existing_relations")?; - if !relations.contains(model.getattr("name")?)? { + if !relations.contains(model_reads::name_object(model)?)? { if snowflake && desired_days > 1 && table_type.eq(TRANSIENT_TABLE_TYPE)? { return Err(transient_error(py, &name, desired_days, "set ")); } @@ -374,7 +375,7 @@ fn relation_retention<'py>( flatten(&changes)?, )?]); } - let relation = relations.get_item(model.getattr("name")?)?; + let relation = relations.get_item(model_reads::name_object(model)?)?; let relation_transient = relation.getattr("is_transient")?; let lookup = call_with( &host::entry(py, AdapterHost::RelationLookup)?.getattr("key")?, @@ -465,7 +466,7 @@ fn bigquery_retention<'py>( for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { let model = model?; let retention = model.getattr("config")?.getattr("time_travel_retention")?; - let destination = model.getattr("destination")?; + let destination = model_reads::destination(&model)?; if retention.getattr("unmanaged")?.is_truthy()? || retention.getattr("desired_days")?.is_none() || destination.getattr("schema")?.is_none() @@ -482,7 +483,7 @@ fn bigquery_retention<'py>( } let mut scopes: BTreeSet<(String, String, Option)> = BTreeSet::new(); for model in models { - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; let database: Option = destination.getattr("database")?.extract()?; let schema: String = destination.getattr("schema")?.str()?.extract()?; scopes.insert((database.clone().unwrap_or_default(), schema, database)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 4338918063..2a4666fb15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,5 +1,6 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_phases::objects::{native, replace, tuple}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use pyo3::IntoPyObject; @@ -66,7 +67,7 @@ pub(crate) fn resolve_planner_scopes<'py>( } let models = PyDict::new(py); for (name, model) in full.getattr("models_by_name")?.downcast::()?.iter() { - if stale_keys.contains(model.getattr("key")?)? { + if stale_keys.contains(model_reads::model_key(&model)?)? { models.set_item(name, model)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index 1ac5f15e7b..e06e19c04c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,12 +1,15 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::planner_entries::relations::build_planner_relations_context; +use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; +use crate::bindings::_helpers::planner_migrations::model_planning::plan_model_migrations; +use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -85,30 +88,35 @@ pub(crate) fn gather_planner_warehouse_state<'py>( deferred_locations: deferral.getattr("deferred_locations")?, cursor_scope, })?; - let migrations = native( + let migrations = plan_model_migrations( py, - NativeAttr::PlanModelMigrations, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", snapshot.clone()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ("fingerprints", fingerprints.clone()), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope.clone()), + ("snapshot", snapshot.clone()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ("fingerprints", fingerprints.clone()), + ], + )?), )?; fingerprints.call_method0("persist")?; - let inspection_relations = call_with( - ®istry::native(py, NativeAttr::BuildPlannerRelationsContext)?, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", runtime.getattr("connection")?), - ("scope", scopes.getattr("inspection_scope")?), - ("deferral", deferral.clone()), - ("project_config", config), - ("local_config", runtime.getattr("local_config")?), - ], + let inspection_relations = build_planner_relations_context( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", runtime.getattr("connection")?), + ("scope", scopes.getattr("inspection_scope")?), + ("deferral", deferral.clone()), + ("project_config", config), + ("local_config", runtime.getattr("local_config")?), + ], + )?), )?; let physical = PyDict::new(py); for entry in migrations.getattr("entries")?.try_iter()? { @@ -130,36 +138,40 @@ pub(crate) fn gather_planner_warehouse_state<'py>( )?; } } - let columns = native( + let columns = plan_column_migrations( py, - NativeAttr::PlanColumnMigrations, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", migrations.getattr("snapshot")?), - ( - "full_refresh_model_names", - effectively_full_refreshed_model_names(&project, full_refresh)?, - ), - ("physical_relations", physical.into_any()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ( - "source_columns", - inspection_relations.getattr("source_warehouse_columns")?, - ), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope.clone()), + ("snapshot", migrations.getattr("snapshot")?), + ( + "full_refresh_model_names", + effectively_full_refreshed_model_names(&project, full_refresh)?, + ), + ("physical_relations", physical.into_any()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ( + "source_columns", + inspection_relations.getattr("source_warehouse_columns")?, + ), + ], + )?), )?; - let old_names = native( + let old_names = plan_old_name_migration_views( py, - NativeAttr::PlanOldNameViews, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope), - ("snapshot", snapshot), - ("migration_entries", migrations.getattr("entries")?), - ("column_entries", columns.getattr("entries")?), - ], + Some(keywords( + py, + &[ + ("runtime", runtime.clone()), + ("scope", selected_scope), + ("snapshot", snapshot), + ("migration_entries", migrations.getattr("entries")?), + ("column_entries", columns.getattr("entries")?), + ], + )?), )?; if !runtime.getattr("on_progress")?.is_none() { progress( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 6bd7cb4f40..a77e1e89d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1013,37 +1013,6 @@ const R_RELATION_FIXTURE_COMPLETION: RecordSpec = RecordSpec { FieldSpec::required("diagnostics"), ], }; -const R_FIXTURE_COLUMN_METADATA: RecordSpec = RecordSpec { - name: "FixtureColumnMetadata", - doc: "Known type and nullability for one relation fixture column.", - fields: &[ - FieldSpec::required("name"), - FieldSpec::required("type"), - FieldSpec::required("nullable"), - ], -}; -const R_FIXTURE_RELATION_METADATA: RecordSpec = RecordSpec { - name: "FixtureRelationMetadata", - doc: "Known columns and whether they form an authoritative relation shape.", - fields: &[ - FieldSpec::required("columns"), - FieldSpec::required("authoritative_names"), - ], -}; -const R_RELATION_FIXTURE_PLANNING_CONTEXT: RecordSpec = RecordSpec { - name: "RelationFixturePlanningContext", - doc: "Project-wide fixture metadata shared across test and scenario planning.", - fields: &[ - FieldSpec::required("models_by_name"), - FieldSpec::required("relations"), - FieldSpec::required("authoritative_columns"), - FieldSpec::required("expected_types"), - FieldSpec::uncompared_shown( - "fixture_inferences", - FieldDefault::NativeClassCall(NativeAttr::FixtureColumnInferences), - ), - ], -}; const R_SELECTOR_EXPANSION: RecordSpec = RecordSpec { name: "SelectorExpansion", doc: "One selector split into graph expansion flags and core text.", @@ -1579,9 +1548,6 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_NATIVE_SQL_TEST_ARTIFACT, R_RELATION_FIXTURE_DIAGNOSTIC, R_RELATION_FIXTURE_COMPLETION, - R_FIXTURE_COLUMN_METADATA, - R_FIXTURE_RELATION_METADATA, - R_RELATION_FIXTURE_PLANNING_CONTEXT, R_SELECTOR_EXPANSION, R_PARSED_SCENARIO_ARTIFACT_NAME, R_MISSING_UPSTREAM, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 7a41207e0b..9626282229 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -1,5 +1,6 @@ //! Model cursor orchestration over live warehouse snapshot facts. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::model_config::config_values::{ config_cursor_bound, config_str as authored_config_str, @@ -42,11 +43,11 @@ fn unbounded(py: Python<'_>) -> PyResult> { fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { let relation = snapshot .getattr("existing_relations")? - .call_method1("get", (model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(model)?,))?; if relation.is_none() { return Ok(false); } - let destination = model.getattr("destination")?; + let destination = model_reads::destination(model)?; let key = call_with( &host::entry(model.py(), AdapterHost::RelationLookup)?.getattr("key")?, &[ @@ -61,12 +62,12 @@ fn destination_exists(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> PyResult { let target = snapshot.getattr("target_relation")?; Ok(!target.is_none() - && target.ne(model.getattr("destination")?.getattr("qualified_name")?)?) + && target.ne(model_reads::destination(model)?.getattr("qualified_name")?)?) } #[pyfunction] #[pyo3(signature = (**kwargs))] -fn compute_model_cursor_bounds<'py>( +pub(crate) fn compute_model_cursor_bounds<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -118,7 +119,7 @@ fn compute_model_cursor_bounds<'py>( let snapshot = args.value("snapshot")?; let mut cursor_snapshot = snapshot .getattr("cursor_snapshots")? - .call_method1("get", (model.getattr("name")?,))?; + .call_method1("get", (model_reads::name_object(&model)?,))?; let roles = call_with( ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, &[("model", model.clone())], @@ -147,7 +148,7 @@ fn compute_model_cursor_bounds<'py>( let refusal = call_with( ®istry::native(py, NativeAttr::EmptyInputRebuildRefusal)?, &[ - ("model_name", model.getattr("name")?), + ("model_name", model_reads::name_object(&model)?), ("input_names", empty.clone()), ("cursor_type", cursor_type.clone()), ], @@ -183,7 +184,7 @@ fn compute_model_cursor_bounds<'py>( return Err(PyErr::from_value(call_with( ®istry::native(py, NativeAttr::UnreadableCursorInputsError)?, &[ - ("model_name", model.getattr("name")?), + ("model_name", model_reads::name_object(&model)?), ("cursor_type", cursor_type), ("input_names", names), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 33ba90bdd1..8c069d8dd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -1,5 +1,6 @@ //! Selected-model inspection and untyped override conversion. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_resolution::values::{method, string}; @@ -14,7 +15,7 @@ use sqlbuild_cursor::planner::main::override_kind::override_kind; #[pyfunction] #[pyo3(signature = (**kwargs))] -fn type_cursor_overrides<'py>( +pub(crate) fn type_cursor_overrides<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -45,7 +46,10 @@ fn type_cursor_overrides<'py>( } let kind = config("cursor_type")?; if kind.eq("integer")? || kind.eq("timestamp")? { - models.push((model.getattr("name")?.extract()?, kind.extract()?)); + models.push(( + model_reads::name_object(&model)?.extract()?, + kind.extract()?, + )); } } let mut integer_literals = true; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index c13ec8d03f..da8a171299 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -1,5 +1,6 @@ //! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -353,7 +354,7 @@ fn model_cursor_snapshot<'py>( ) -> PyResult> { snapshot .getattr("cursor_snapshots")? - .call_method1("get", (model.getattr("name")?,)) + .call_method1("get", (model_reads::name_object(model)?,)) } fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { @@ -566,7 +567,7 @@ pub(crate) fn model_cursor_overrides<'py>( } } let session = CursorSession::new(py)?; - let name = model.getattr("name")?.str()?; + let name = model_reads::name_object(model)?.str()?; let cursor_start = config_bound(model, "cursor_start")?; let cursor_end = config_bound(model, "cursor_end")?; if !resolved_start.is_none() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index df9dfd5f7e..d5f276315b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -3,7 +3,7 @@ mod constants; mod cursor_bounds; mod cursor_model; -mod cursor_overrides; +pub(crate) mod cursor_overrides; pub(crate) mod cursor_policies; pub(crate) mod cursor_ranges; mod cursor_safety; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 862b1f7b5e..f4f0192106 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -1,96 +1,103 @@ //! Model, function and SQL-hook orchestration through adapter-owned rendering. +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_intrinsics; +use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; +use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, + KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; -use crate::bindings::_helpers::planner_resolution::values::string; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; +use crate::bindings::_helpers::planner_resolution::refs::{ + resolve_dbt_ref_references, resolve_ref_references, resolve_table_function_references, + resolve_udf_references, +}; +use crate::bindings::_helpers::planner_resolution::sources::resolve_source_references; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +/// What resolving one model's, hook's or function's references reads. struct Resolution<'py> { adapter: Bound<'py, PyAny>, - context: Bound<'py, PyAny>, external: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, + source_warehouse_columns: Bound<'py, PyAny>, + star_exclude_keyword: Bound<'py, PyAny>, + model_locations: Bound<'py, PyAny>, + seed_locations: Bound<'py, PyAny>, + function_locations: Bound<'py, PyAny>, } impl<'py> Resolution<'py> { + /// Resolution against a `ModelPlanContext`'s relations. + fn of_context( + adapter: Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external: Bound<'py, PyAny>, + ) -> PyResult { + Ok(Self { + adapter, + external, + source_map: context.getattr("source_map")?, + source_warehouse_columns: context.getattr("source_warehouse_columns")?, + star_exclude_keyword: context.getattr("star_exclude_keyword")?, + model_locations: context.getattr("model_locations")?, + seed_locations: context.getattr("seed_locations")?, + function_locations: context.getattr("function_locations")?, + }) + } + fn resolve( &self, sql: Bound<'py, PyAny>, cursor: &[(&str, Bound<'py, PyAny>)], - ) -> PyResult> { + ) -> PyResult { let py = self.adapter.py(); - let mut source = vec![("query_sql", sql), ("adapter", self.adapter.clone())]; - for key in [ - "source_map", - "source_warehouse_columns", - "star_exclude_keyword", - ] { - source.push((key, self.context.getattr(key)?)); - } + let mut source = vec![ + ("query_sql", sql), + ("adapter", self.adapter.clone()), + ("source_map", self.source_map.clone()), + ( + "source_warehouse_columns", + self.source_warehouse_columns.clone(), + ), + ("star_exclude_keyword", self.star_exclude_keyword.clone()), + ]; source.extend_from_slice(cursor); - let mut resolved = call_with( - ®istry::native(py, NativeAttr::ResolveSourceReferences)?, - &source, - )?; - let mut refs = vec![("query_sql", resolved), ("adapter", self.adapter.clone())]; - for key in ["model_locations", "seed_locations"] { - refs.push((key, self.context.getattr(key)?)); - } + let resolved: String = resolve_source_references(py, Some(keywords(py, &source)?))?; + let mut refs = vec![ + ("query_sql", PyString::new(py, &resolved).into_any()), + ("adapter", self.adapter.clone()), + ("model_locations", self.model_locations.clone()), + ("seed_locations", self.seed_locations.clone()), + ]; refs.extend_from_slice(cursor); - resolved = call_with( - ®istry::native(py, NativeAttr::ResolveRefReferences)?, - &refs, - )?; - resolved = call_with( - ®istry::native(py, NativeAttr::ResolveDbtRefReferences)?, - &[ - ("query_sql", resolved), - ("external_sql_reference_resolver", self.external.clone()), - ], - )?; - let locations = self.context.getattr("function_locations")?; - let locations = if locations.is_truthy()? { - locations + let resolved: String = resolve_ref_references(py, Some(keywords(py, &refs)?))?; + let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; + let locations = if self.function_locations.is_truthy()? { + self.function_locations.clone() } else { PyDict::new(py).into_any() }; - for name in [ - NativeAttr::ResolveUdfReferences, - NativeAttr::ResolveTableFunctionReferences, - ] { - resolved = call_with( - ®istry::native(py, name)?, - &[ - ("query_sql", resolved), - ("function_locations", locations.clone()), - ("adapter", self.adapter.clone()), - ], - )?; - } - Ok(resolved) + let resolved: String = resolve_udf_references(&resolved, &locations, &self.adapter)?; + resolve_table_function_references(&resolved, &locations, &self.adapter) } - fn assert_resolved(&self, sql: &Bound<'py, PyAny>, label: &str) -> PyResult<()> { - call_with( - ®istry::native(self.adapter.py(), NativeAttr::AssertNoUnresolvedSqlMarkers)?, - &[ - ("sql", sql.clone()), - ( - "context", - string(self.adapter.py(), &format!("{label} planned SQL")), - ), - ], - )?; - Ok(()) + fn assert_resolved(&self, sql: &str, label: &str) -> PyResult<()> { + let py = self.adapter.py(); + assert_no_unresolved_sql_markers( + py, + &PyString::new(py, sql), + &format!("{label} planned SQL"), + ) } - fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult> { + fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { let py = self.adapter.py(); let resolved = self.resolve( sql, @@ -162,50 +169,49 @@ pub(crate) fn resolve_model_sql<'py>( ), )?; let model = args.value("model")?; - let resolution = Resolution { - adapter: args.value("adapter")?, - context: args.value("context")?, - external: args.value("external_sql_reference_resolver")?, - }; + let context = args.value("context")?; + let resolution = Resolution::of_context( + args.value("adapter")?, + &context, + args.value("external_sql_reference_resolver")?, + )?; let cursor_type = authored_config_str(&model.getattr("config")?.getattr("values")?, "cursor_type")? .unwrap_or_else(|| py.None().into_bound(py)); let overrides = args.value("cursor_overrides")?; - let bounds = call_with( - ®istry::native(py, NativeAttr::ComputeModelCursorBounds)?, - &[ - ("model", model.clone()), - ("snapshot", args.value("snapshot")?), - ("backfill", args.value("backfill")?), - ("full_refresh", args.value("full_refresh")?), - ( - "start_cursor_override", - overrides.getattr("start_cursor_override")?, - ), - ( - "end_cursor_override", - overrides.getattr("end_cursor_override")?, - ), - ( - "runtime_cursor_producer_names", - resolution - .context - .getattr("runtime_cursor_producer_names")?, - ), - ("context", resolution.context.clone()), - ( - "replaces_relation", - args.value_or_false("replaces_relation")?, - ), - ], - )?; - let roles = call_with( - ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, - &[("model", model.clone())], + let bounds = compute_model_cursor_bounds( + py, + Some(keywords( + py, + &[ + ("model", model.clone()), + ("snapshot", args.value("snapshot")?), + ("backfill", args.value("backfill")?), + ("full_refresh", args.value("full_refresh")?), + ( + "start_cursor_override", + overrides.getattr("start_cursor_override")?, + ), + ( + "end_cursor_override", + overrides.getattr("end_cursor_override")?, + ), + ( + "runtime_cursor_producer_names", + context.getattr("runtime_cursor_producer_names")?, + ), + ("context", context.clone()), + ( + "replaces_relation", + args.value_or_false("replaces_relation")?, + ), + ], + )?), )?; + let roles = resolve_cursor_input_roles(&model)?; let inclusive = append_cursor_inclusive(&model)?; let mut sql = resolution.resolve( - model.getattr("query_sql")?, + model_reads::query_sql(&model)?, &[ ("cursor_bounds", bounds.clone()), ("cursor_filter_inputs", roles.getattr("filter_inputs")?), @@ -213,12 +219,9 @@ pub(crate) fn resolve_model_sql<'py>( ("lower_bound_inclusive", inclusive), ], )?; - let intrinsics = call_with( - ®istry::native(py, NativeAttr::ResolveCursorIntrinsics)?, - &[("sql", sql.clone())], - )?; - let name: String = model.getattr("name")?.extract()?; - if intrinsics.get_item(1)?.is_truthy()? { + let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; + let name: String = model_reads::model_name(&model)?; + if uses_intrinsics { if bounds.is_none() { let cause = args.value("full_rebuild_cause")?; let cause = if cause.is_truthy()? { @@ -233,7 +236,7 @@ pub(crate) fn resolve_model_sql<'py>( sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; - Ok(sql) + Ok(PyString::new(py, &sql).into_any()) } /// Effective append cursor lower-bound inclusivity. @@ -255,13 +258,13 @@ fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult( - sql: &Bound<'py, PyAny>, + sql: &str, bounds: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, -) -> PyResult> { - let py = sql.py(); - let literal = |field: &str| -> PyResult> { +) -> PyResult { + let py = adapter.py(); + let literal = |field: &str| -> PyResult { call_with( &adapter.getattr("render_cursor_bound_literal")?, &[ @@ -271,19 +274,12 @@ fn intrinsic_bounds<'py>( ), ("cursor_type", cursor_type.clone()), ], - ) + )? + .extract() }; let start_sql = literal("start")?; let end_sql = literal("end")?; - call_with( - ®istry::native(py, NativeAttr::ResolveCursorIntrinsics)?, - &[ - ("sql", sql.clone()), - ("start_sql", start_sql), - ("end_sql", end_sql), - ], - )? - .get_item(0) + Ok(resolve_cursor_intrinsics(py, sql, Some(&start_sql), Some(&end_sql))?.0) } fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> PyResult { @@ -324,25 +320,21 @@ fn resolve_function_sql<'py>( &[], ), )?; - let mut context = vec![("models_by_name", PyDict::new(py).into_any())]; - for key in [ - "model_locations", - "seed_locations", - "function_locations", - "source_map", - "source_warehouse_columns", - "star_exclude_keyword", - ] { - context.push((key, args.value(key)?)); - } let resolution = Resolution { adapter: args.value("adapter")?, - context: call_with(®istry::class(py, RecordId::ModelPlanContext)?, &context)?, external: py.None().into_bound(py), + source_map: args.value("source_map")?, + source_warehouse_columns: args.value("source_warehouse_columns")?, + star_exclude_keyword: args.value("star_exclude_keyword")?, + model_locations: args.value("model_locations")?, + seed_locations: args.value("seed_locations")?, + function_locations: args.value("function_locations")?, }; let function = args.value("function")?; let name: String = function.getattr("name")?.extract()?; - resolution.unbounded(function.getattr("body_sql")?, &format!("Function '{name}'")) + let sql: String = + resolution.unbounded(function.getattr("body_sql")?, &format!("Function '{name}'"))?; + Ok(PyString::new(py, &sql).into_any()) } #[pyfunction] @@ -353,13 +345,9 @@ pub(crate) fn resolve_model_hook_entries<'py>( context: Bound<'py, PyAny>, external_sql_reference_resolver: Bound<'py, PyAny>, ) -> PyResult> { - let resolution = Resolution { - adapter, - context, - external: external_sql_reference_resolver, - }; + let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; let values = model.getattr("config")?.getattr("values")?; - let name: String = model.getattr("name")?.extract()?; + let name: String = model_reads::model_name(model)?; let mut hooks: Vec> = Vec::new(); for key in ["pre_hooks", "post_hooks"] { hooks.push(resolution.hook( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index e212bbf96f..63475c1b12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -14,7 +14,10 @@ use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans:: #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { +pub(crate) fn resolve_ref_references( + py: Python<'_>, + kwargs: Option>, +) -> PyResult { let args = KeywordArguments::read( py, "resolve_ref_references", @@ -72,7 +75,7 @@ fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> #[pyfunction] #[pyo3(signature = (*, query_sql, external_sql_reference_resolver=None))] -fn resolve_dbt_ref_references( +pub(crate) fn resolve_dbt_ref_references( query_sql: &str, external_sql_reference_resolver: Option<&Bound<'_, PyAny>>, ) -> PyResult { @@ -195,7 +198,7 @@ fn resolve_function_calls( #[pyfunction] #[pyo3(signature = (*, query_sql, function_locations, adapter))] -fn resolve_udf_references( +pub(crate) fn resolve_udf_references( query_sql: &str, function_locations: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, @@ -205,7 +208,7 @@ fn resolve_udf_references( #[pyfunction] #[pyo3(signature = (*, query_sql, function_locations, adapter))] -fn resolve_table_function_references( +pub(crate) fn resolve_table_function_references( query_sql: &str, function_locations: &Bound<'_, PyAny>, adapter: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index 8c09e4e2b5..d6d44f8596 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -279,7 +279,7 @@ fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyRes #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_source_references( +pub(crate) fn resolve_source_references( py: Python<'_>, kwargs: Option>, ) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 86669773d4..9e7835d0c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -1,5 +1,6 @@ //! Assemble CLI scenario plans while retaining adapter and executor boundaries. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; @@ -122,12 +123,12 @@ fn namespace<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } for field in ["target_model_names", "model_names"] { for name in graph.getattr(field)?.try_iter()? { if let Some(model) = models.get_item(name?)? { - let destination = model.getattr("destination")?; + let destination = model_reads::destination(&model)?; return Ok(( destination.getattr("database")?, destination.getattr("schema")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index ba484b2854..1303f5e260 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -1,5 +1,6 @@ //! Assemble scenario execution plans through existing planner and adapter APIs. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -12,6 +13,8 @@ use crate::bindings::_helpers::planner_scenario::{ sql::{required_target, resolve_model_dbt_refs, resolve_sql, source_label}, values::{error, string}, }; +use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures as complete_relation_fixtures; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; @@ -116,11 +119,11 @@ pub(crate) fn build_scenario_execution_plan<'py>( let destination = required_target( &context.relations, "model_locations", - &model.getattr("name")?.extract::()?, + &model_reads::model_name(&model)?, "model", )?; let query = resolve_model_dbt_refs( - &model.getattr("query_sql")?.extract::()?, + &model_reads::query_sql(&model)?.extract::()?, &context.relations, )?; let record = registry::native(py, NativeAttr::CompiledModelRecord)?.call1((&model,))?; @@ -207,31 +210,22 @@ fn complete_fixtures<'py>( .iter() .map(|key| key.getattr("name")) .collect::>>()?; - let completion = call_with( - ®istry::native(py, NativeAttr::BuildRelationFixtureCompletion)?, - &[ - ("project", project.clone()), - ("adapter", context.adapter.clone()), - ( - "request", - PyTuple::new( - py, - [ - PyTuple::new(py, names)?.into_any(), - PyTuple::new(py, groups)?.into_any(), - ], - )? - .into_any(), - ), - ], + let names = names + .iter() + .map(|name| name.extract()) + .collect::>>()?; + let planning = planning_context(project, None)?; + let completion = complete_relation_fixtures( + &planning, + &context.adapter, + (&names, PyTuple::new(py, groups)?.as_any()), )?; - let diagnostics = completion.getattr("diagnostics")?; - if diagnostics.is_truthy()? { + if !completion.diagnostics.is_empty() { let mut details: Vec = Vec::new(); - for diagnostic in diagnostics.try_iter()? { + for diagnostic in &completion.diagnostics { details.push(format!( "- {}", - diagnostic?.getattr("message")?.extract::()? + diagnostic.getattr("message")?.extract::()? )); } let name: String = scenario.getattr("name")?.extract()?; @@ -246,7 +240,7 @@ fn complete_fixtures<'py>( None, )?); } - completion.getattr("fixture_sql_by_key") + Ok(completion.sql_by_key.into_any()) } fn model_context<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 4be6a925ee..e550a8d14e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -1,5 +1,6 @@ //! Convert project views into pure scenario dependency facts. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; @@ -41,7 +42,9 @@ pub(crate) fn plan_scenario_graph<'py>( .try_iter()? .map(|dep| native_key(&dep?)) .collect::>()?; - input.models.insert(model.getattr("name")?.extract()?, deps); + input + .models + .insert(model_reads::name_object(&model)?.extract()?, deps); } for source in project.getattr("sources")?.try_iter()? { input diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index 4646fba279..a2bc231951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -13,3 +13,11 @@ pub(crate) const MODEL_REFERENCE: &str = "ref"; pub(crate) const SEED_REFERENCE: &str = "seed"; pub(crate) const SOURCE_REFERENCE: &str = "source"; pub(crate) const DDL_SEPARATOR: &str = ";\n\n"; +/// The lineage confidence whose reads fixtures must provide. +pub(crate) const HIGH_CONFIDENCE: &str = "high"; +/// The lineage transform kind that passes an input column's type through. +pub(crate) const DIRECT_TRANSFORM: &str = "direct"; +/// Resource type values keying fixture relations. +pub(crate) const MODEL_REFERENCE_KIND: &str = "model"; +pub(crate) const SOURCE_REFERENCE_KIND: &str = "source"; +pub(crate) const SEED_REFERENCE_KIND: &str = "seed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 40cc22382d..2497ab3b8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -7,6 +7,7 @@ use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::planner_strategy::main::cursor_input_column::cursor_input_column_failure; use sqlbuild_analysis::planner_strategy::models::{CursorInputColumnCheck, CursorInputColumnOwner}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::call_with; @@ -74,7 +75,7 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; let source_map = or_empty(request.optional("source_map")?, py)?; let warehouse_columns: Bound<'py, PyAny> = request.get("source_warehouse_columns")?; - let model_name: String = text(&model.getattr("name")?)?; + let model_name: String = model_reads::model_name(&model)?; for role in ["filter_inputs", "watermark_inputs"] { for item in roles.getattr(role)?.call_method0("items")?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index b20c8c11e1..29fcf254be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -69,7 +69,7 @@ fn classify_cursor_sql_type<'py>( /// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. #[pyfunction] #[pyo3(signature = (**options))] -fn check_cursor_type_consistency<'py>( +pub(crate) fn check_cursor_type_consistency<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs index f614f02137..7d034d3faf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_completion.rs @@ -1,12 +1,14 @@ -//! Native-hosted relation fixture completion over Python boundary models. +//! Relation fixture completion over the Rust fixture planning context. + +use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::align_columns::align_columns; -use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, RequiredColumn}; +use sqlbuild_analysis::planner_fixtures::models::{AlignmentInput, FixtureColumn}; -use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context as project_planning_context; use crate::bindings::_helpers::planner_strategy::fixture_preparation::{PreparedFixtures, prepare}; use crate::bindings::_helpers::planner_strategy::fixture_reads::required_fixture_columns; use crate::bindings::_helpers::planner_strategy::fixture_selection::{ @@ -15,190 +17,217 @@ use crate::bindings::_helpers::planner_strategy::fixture_selection::{ use crate::bindings::_helpers::planner_strategy::fixture_sql::{CompletionSql, render_completed}; use crate::bindings::_helpers::planner_strategy::fixture_types::expected_fixture_types; use crate::bindings::_helpers::planner_strategy::fixture_values::{ - columns, diagnostic, folded, label, record, strings, + columns, diagnostic, label, record, +}; +use crate::bindings::_helpers::planner_strategy::models::{ + ExpectedTypes, RelationFixturePlanningContext, }; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::records::ids::RecordId; -#[pyfunction] -#[pyo3(signature = (*, project, adapter, request, planning_context = None))] -pub(crate) fn build_relation_fixture_completion<'py>( - project: &Bound<'py, PyAny>, +/// Completed fixture SQL and inferred columns by Python fixture key, expected types and diagnostics. +pub(crate) struct FixtureCompletion<'py, 'a> { + pub(crate) sql_by_key: Bound<'py, PyDict>, + pub(crate) inferred_by_fixture: Bound<'py, PyDict>, + pub(crate) expected_types: ExpectedTypes<'a>, + pub(crate) diagnostics: Vec>, +} + +impl ExpectedTypes<'_> { + /// The relation's expected column types: the chain's propagated types, else the context's. + pub(crate) fn get(&self, key: &FixtureKey) -> Option<&[(String, String)]> { + self.overlay + .iter() + .find(|(known, _)| known == key) + .map(|(_, types)| types.as_slice()) + .or_else(|| self.base.get(key).map(Vec::as_slice)) + } +} + +/// Complete the fixture groups of one chain against the planning context. +pub(crate) fn complete_fixtures<'py, 'a>( + context: &'a RelationFixturePlanningContext, adapter: &Bound<'py, PyAny>, - request: (Bound<'py, PyAny>, Bound<'py, PyAny>), - planning_context: Option>, -) -> PyResult> { - let (ordered_model_names, fixture_groups) = request; - let context = match planning_context { - Some(context) if context.is_truthy()? => context, - _ => build_relation_fixture_context(project, None)?, - }; - let relations = context.getattr("relations")?.cast_into::()?; - let prepared = prepare( - &fixture_groups, - &relations, - adapter, - &context.getattr("fixture_inferences")?, - )?; - let fixture_keys = fixture_keys_requiring_analysis( - &prepared.inferred, - &context - .getattr("authoritative_columns")? - .cast_into::()?, - )?; - let (required, stars) = required_fixture_columns( - ( - ordered_model_names.extract()?, - context.getattr("models_by_name")?.cast_into::()?, - ), - ( - fixture_keys.into_any(), - relations.clone(), - context - .getattr("authoritative_columns")? - .cast_into::()?, - ), - adapter, - )?; - let expected = expected_fixture_types( - &context.getattr("expected_types")?.cast_into::()?, - ordered_model_names.extract()?, - &context.getattr("models_by_name")?.cast_into::()?, - )?; - complete_prepared( - prepared, - (relations, required, stars.into_any()), - expected.into_any(), - adapter, - ) + (ordered_model_names, fixture_groups): (&[String], &Bound<'py, PyAny>), +) -> PyResult> { + let py = adapter.py(); + let prepared = prepare(fixture_groups, context, adapter)?; + let fixture_keys = fixture_keys_requiring_analysis(&prepared, context)?; + let required = required_fixture_columns(context, ordered_model_names, &fixture_keys, adapter)?; + let expected = expected_fixture_types(context, py, ordered_model_names)?; + complete_prepared(prepared, (context, &required), expected, adapter) } -fn complete_prepared<'py>( +fn complete_prepared<'py, 'a>( mut prepared: PreparedFixtures<'py>, - plan: (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyAny>), - expected: Bound<'py, PyAny>, + (context, required): ( + &'a RelationFixturePlanningContext, + &sqlbuild_analysis::planner_fixtures::models::FixtureReadRequirements, + ), + expected: ExpectedTypes<'a>, adapter: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py = adapter.py(); - let (relations, required, stars) = plan; + let python_keys: Vec<(FixtureKey, Bound<'py, PyAny>)> = prepared.keys.clone(); + let python_key = |key: &FixtureKey| -> Option<&Bound<'py, PyAny>> { + python_keys + .iter() + .find(|(known, _)| known == key) + .map(|(_, python)| python) + }; let typed_nulls = PyDict::new(py); - for (key, names) in prepared.null_names.iter() { - let types = expected.call_method1("get", (&key, PyDict::new(py)))?; + for (key, original) in &python_keys { + let Some(names) = prepared.null_names.get_item(original)? else { + continue; + }; + let types: &[(String, String)] = expected.get(key).unwrap_or_default(); let selected = PyDict::new(py); for name in names.try_iter()? { let name = name?; - let value = types.call_method1("get", (&name,))?; - if !value.is_none() { - selected.set_item(name, value)?; + let folded: String = name.extract()?; + if let Some((_, sql_type)) = types.iter().find(|(column, _)| *column == folded) { + selected.set_item(name, sql_type)?; } } if !selected.is_empty() { - typed_nulls.set_item(key, selected)?; + typed_nulls.set_item(original, selected)?; } } - for value in unknown_fixture_column_diagnostics(&prepared.inferred, &relations)?.try_iter()? { - prepared.diagnostics.push(value?); + for value in unknown_fixture_column_diagnostics(&prepared, context)? { + prepared.diagnostics.push(value); } - let completed_keys = PySet::empty(py)?; - for (key, required_columns) in required.iter() { - let Some(inferred) = prepared.inferred.get_item(&key)? else { + let mut completed_keys: HashSet = HashSet::new(); + for (key, requirements) in &required.columns { + let Some(original) = python_key(key) else { + continue; + }; + let Some(inferred) = prepared.inferred.get_item(original)? else { continue; }; let typed = typed_nulls - .get_item(&key)? + .get_item(original)? .map(|value| value.cast_into::()) .transpose()? .unwrap_or_else(|| PyDict::new(py)); - let metadata = match relations.get_item(&key)? { - Some(relation) => relation.getattr("columns")?, - None => PyTuple::empty(py).into_any(), - }; - let requirements = required_columns - .cast_into::()? - .iter() - .map(|(name, readers)| { - Ok(RequiredColumn { - name: name.extract()?, - folded_name: folded(&name)?, - readers: strings(&readers)?, - }) - }) - .collect::>>()?; + let metadata: &[FixtureColumn] = context + .relations + .get(key) + .map(|relation| relation.columns.as_slice()) + .unwrap_or_default(); let alignment = align_columns( - &label(&key)?, + &label(original)?, AlignmentInput { supplied: &columns(&inferred, true)?, - metadata: &columns(&metadata, false)?, - required: &requirements, - star: stars.contains(&key)?, + metadata, + required: requirements, + star: required.stars.contains(key), typed_nulls: !typed.is_empty(), }, ); for message in alignment.diagnostics { - prepared.diagnostics.push(diagnostic(&key, &message)?); + prepared.diagnostics.push(diagnostic(original, &message)?); } let Some(completed) = alignment.completed else { continue; }; - let completed = completed + let completed: Vec = completed .into_iter() - .map(|index| metadata.get_item(index)) - .collect::>>()?; - let sql: String = prepared - .sql - .get_item(&key)? - .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? - .extract()?; - let quoted = prepared - .quoted_names - .get_item(&key)? - .unwrap_or(PyFrozenSet::empty(py)?.into_any()); + .map(|index| metadata[index].clone()) + .collect(); let sql = render_completed( CompletionSql { - sql: &sql, + sql: &fixture_sql(&prepared, original)?, inferred, typed_nulls: typed, - quoted_names: quoted, - completed: PyTuple::new(py, completed)?.into_any(), + quoted_names: quoted_names(&prepared, original)?, + completed, }, adapter, )?; - prepared.sql.set_item(&key, sql)?; - completed_keys.add(key)?; + prepared.sql.set_item(original, sql)?; + completed_keys.insert(key.clone()); } - for (key, typed) in typed_nulls.iter() { - if completed_keys.contains(&key)? { + for (key, original) in &python_keys { + let Some(typed) = typed_nulls.get_item(original)? else { + continue; + }; + if completed_keys.contains(key) { continue; } let inferred = prepared .inferred - .get_item(&key)? + .get_item(original)? .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture inference"))?; - let sql: String = prepared - .sql - .get_item(&key)? - .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? - .extract()?; - let quoted = prepared - .quoted_names - .get_item(&key)? - .unwrap_or(PyFrozenSet::empty(py)?.into_any()); let sql = render_completed( CompletionSql { - sql: &sql, + sql: &fixture_sql(&prepared, original)?, inferred, typed_nulls: typed.cast_into::()?, - quoted_names: quoted, - completed: PyTuple::empty(py).into_any(), + quoted_names: quoted_names(&prepared, original)?, + completed: Vec::new(), }, adapter, )?; - prepared.sql.set_item(key, sql)?; + prepared.sql.set_item(original, sql)?; + } + Ok(FixtureCompletion { + sql_by_key: prepared.sql, + inferred_by_fixture: prepared.inferred, + expected_types: expected, + diagnostics: prepared.diagnostics, + }) +} + +fn fixture_sql(prepared: &PreparedFixtures<'_>, key: &Bound<'_, PyAny>) -> PyResult { + prepared + .sql + .get_item(key)? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("fixture SQL"))? + .extract() +} + +fn quoted_names<'py>( + prepared: &PreparedFixtures<'py>, + key: &Bound<'py, PyAny>, +) -> PyResult> { + Ok(prepared + .quoted_names + .get_item(key)? + .unwrap_or(PyFrozenSet::empty(key.py())?.into_any())) +} + +/// Complete one chain's fixture groups, building the planning context when none is given. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, request, planning_context = None))] +fn build_relation_fixture_completion<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + request: (Vec, Bound<'py, PyAny>), + planning_context: Option>, +) -> PyResult> { + let py = project.py(); + let built: RelationFixturePlanningContext; + let context: &RelationFixturePlanningContext = match &planning_context { + Some(context) => context.get(), + None => { + built = project_planning_context(project, None)?; + &built + } + }; + let completion = complete_fixtures(context, adapter, (&request.0, &request.1))?; + let expected = PyDict::new(py); + for (key, types) in &completion.expected_types.overlay { + let columns = PyDict::new(py); + for (name, sql_type) in types { + columns.set_item(name, sql_type)?; + } + expected.set_item(key, columns)?; } let kwargs = PyDict::new(py); - kwargs.set_item("fixture_sql_by_key", prepared.sql)?; - kwargs.set_item("inferred_by_fixture", prepared.inferred)?; + kwargs.set_item("fixture_sql_by_key", completion.sql_by_key)?; + kwargs.set_item("inferred_by_fixture", completion.inferred_by_fixture)?; kwargs.set_item("expected_types", expected)?; - kwargs.set_item("diagnostics", PyTuple::new(py, prepared.diagnostics)?)?; + kwargs.set_item("diagnostics", PyTuple::new(py, completion.diagnostics)?)?; record(py, RecordId::RelationFixtureCompletion, &kwargs) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs index 3d2187cabe..e259839244 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_context.rs @@ -1,118 +1,252 @@ -//! Project fixture context materialization and schema conversions. +//! The project-wide fixture planning context: relation metadata and per-model facts as Rust data. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::fixture_schema::fixture_schema; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; +use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_metadata; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata, record}; -use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::analysis_session::fixture_inferences::FixtureColumnInferences; +use crate::bindings::_helpers::planner_strategy::constants::{ + MODEL_REFERENCE_KIND, SEED_REFERENCE_KIND, SOURCE_REFERENCE_KIND, +}; +use crate::bindings::_helpers::planner_strategy::fixture_metadata::model_fixture_relation; +use crate::bindings::_helpers::planner_strategy::fixture_reads::lineage_reads; +use crate::bindings::_helpers::planner_strategy::fixture_types::projections; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::{ + FixtureReadCacheHost, FixtureRelation, ModelFixtureFacts, RelationFixturePlanningContext, +}; +use crate::bindings::_helpers::planner_strategy::types::{FixtureKey, FixtureRelationInput}; +use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::records::registry; -#[pyfunction] -#[pyo3(signature = (*, project, models_by_name = None))] -pub(crate) fn build_relation_fixture_context<'py>( - project: &Bound<'py, PyAny>, - models_by_name: Option>, -) -> PyResult> { +/// Count one batch resolution of SQL-test model chains. +pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { + work_counts(py)?.borrow_mut().chain_resolutions += 1; + Ok(()) +} + +fn work_counts(py: Python<'_>) -> PyResult> { + Ok(registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?) +} + +/// The planning context of a project, over its models or the given name-to-model map. +pub(crate) fn planning_context( + project: &Bound<'_, PyAny>, + models_by_name: Option>, +) -> PyResult { let py = project.py(); + work_counts(py)?.borrow_mut().context_builds += 1; let model_map = match models_by_name { Some(models) => models, None => { let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, &model)?; + models.set_item(model_reads::name_object(&model)?, &model)?; } models } }; - let relations = relation_metadata(project, &model_map)?; - let kwargs = PyDict::new(py); - kwargs.set_item("models_by_name", model_map)?; - kwargs.set_item( - "authoritative_columns", - authoritative_fixture_columns(&relations)?, - )?; - kwargs.set_item("expected_types", base_expected_fixture_types(&relations)?)?; - kwargs.set_item("relations", relations)?; - record(py, RecordId::RelationFixturePlanningContext, &kwargs) -} - -fn relation_metadata<'py>( - project: &Bound<'py, PyAny>, - models: &Bound<'py, PyDict>, -) -> PyResult> { - let py = project.py(); - let kinds = py - .import("sqlbuild.compiler.compile.types")? - .getattr("CompiledResourceType")?; - let relations = PyDict::new(py); - for (name, model) in models.iter() { - relations.set_item( - (kinds.getattr("MODEL")?, name), - model_fixture_metadata(&model)?, - )?; + let mut relations: HashMap = HashMap::new(); + for (name, model) in model_map.iter() { + relations.insert( + (MODEL_REFERENCE_KIND.to_owned(), name.extract()?), + model_fixture_relation(&model)?, + ); } for (collection, entry, kind, source) in [ - ("sources", "source_entry", "SOURCE", true), - ("seeds", "schema_entry", "SEED", false), + ("sources", "source_entry", SOURCE_REFERENCE_KIND, true), + ("seeds", "schema_entry", SEED_REFERENCE_KIND, false), ] { for resource in project.getattr(collection)?.try_iter()? { let resource = resource?; let entry = resource.getattr(entry)?; - let columns = columns(&entry.getattr("columns")?, false)?; + let columns: Vec = columns(&entry.getattr("columns")?, false)?; let authoritative = if source { entry.getattr("contract")?.eq("enforced")? } else { !columns.is_empty() }; - relations.set_item( - (kinds.getattr(kind)?, resource.getattr("name")?), - metadata(py, &columns, authoritative)?, - )?; + relations.insert( + (kind.to_owned(), resource.getattr("name")?.extract()?), + FixtureRelation { + columns, + authoritative, + }, + ); } } - Ok(relations) + Ok(context_of( + model_map.unbind(), + relations, + Py::new(py, FixtureColumnInferences::new())?.into_any(), + )) } -#[pyfunction] -#[pyo3(signature = (*, relations))] -pub(crate) fn authoritative_fixture_columns<'py>( - relations: &Bound<'py, PyDict>, -) -> PyResult> { - let result = PyDict::new(relations.py()); - for (key, relation) in relations.iter() { - let schema = fixture_schema( - &columns(&relation.getattr("columns")?, false)?, - relation.getattr("authoritative_names")?.extract()?, +fn context_of( + models_by_name: Py, + relations: HashMap, + fixture_inferences: Py, +) -> RelationFixturePlanningContext { + let mut authoritative_columns: HashMap> = HashMap::new(); + let mut expected_types: HashMap> = HashMap::new(); + for (key, relation) in &relations { + if let Some(names) = + fixture_schema(&relation.columns, relation.authoritative).authoritative_names + { + authoritative_columns.insert(key.clone(), names); + } + expected_types.insert( + key.clone(), + fixture_schema(&relation.columns, false).expected_types, ); - if let Some(names) = schema.authoritative_names { - result.set_item(key, PyFrozenSet::new(relations.py(), names)?)?; + } + RelationFixturePlanningContext { + models_by_name, + relations, + authoritative_columns, + expected_types, + fixture_inferences, + model_facts: Mutex::new(HashMap::new()), + } +} + +impl RelationFixturePlanningContext { + /// One model's lineage reads and type projections, derived once per context. + pub(crate) fn model_facts( + &self, + py: Python<'_>, + name: &str, + ) -> PyResult>> { + if let Some(found) = self.facts_cache()?.get(name) { + return Ok(Some(Arc::clone(found))); + } + let Some(model) = self.models_by_name.bind(py).get_item(name)? else { + return Ok(None); + }; + let facts = Arc::new(ModelFixtureFacts { + reads: lineage_reads(&model)?, + projections: projections(&model)?, + }); + self.facts_cache()? + .insert(name.to_owned(), Arc::clone(&facts)); + Ok(Some(facts)) + } + + fn facts_cache( + &self, + ) -> PyResult>>> { + self.model_facts + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture model facts")) + } +} + +#[pymethods] +impl RelationFixturePlanningContext { + /// A context over explicit relation metadata: `{key: ((name, type, nullable), ...), authoritative}`. + #[new] + #[pyo3(signature = (*, models_by_name, relations))] + fn new( + py: Python<'_>, + models_by_name: Bound<'_, PyDict>, + relations: HashMap, + ) -> PyResult { + let mut known: HashMap = HashMap::new(); + for (key, (columns, authoritative)) in relations { + let mut fixture_columns: Vec = Vec::with_capacity(columns.len()); + for (name, sql_type, nullable) in columns { + fixture_columns.push(FixtureColumn { + folded_name: python_casefold(&name), + name, + sql_type, + nullable, + }); + } + known.insert( + key, + FixtureRelation { + columns: fixture_columns, + authoritative, + }, + ); + } + Ok(context_of( + models_by_name.unbind(), + known, + Py::new(py, FixtureColumnInferences::new())?.into_any(), + )) + } + + /// Whether the relation's columns form an authoritative shape, `None` for unknown relations. + fn authoritative_names(&self, key: FixtureKey) -> Option { + self.relations + .get(&key) + .map(|relation| relation.authoritative) + } + + /// The relation's known column types by name, `None` for unknown relations. + fn expected_types<'py>( + &self, + py: Python<'py>, + key: FixtureKey, + ) -> PyResult>> { + let Some(types) = self.expected_types.get(&key) else { + return Ok(None); + }; + let result = PyDict::new(py); + for (name, sql_type) in types { + result.set_item(name, sql_type)?; } + Ok(Some(result)) + } + + /// The relation's authoritative column names, `None` when the shape is not authoritative. + fn authoritative_columns<'py>( + &self, + py: Python<'py>, + key: FixtureKey, + ) -> PyResult>> { + self.authoritative_columns + .get(&key) + .map(|names| PyFrozenSet::new(py, names)) + .transpose() } - Ok(result) } +/// Build the project-wide fixture planning context once for many SQL-test plans. #[pyfunction] -#[pyo3(signature = (*, relations))] -pub(crate) fn base_expected_fixture_types<'py>( - relations: &Bound<'py, PyDict>, -) -> PyResult> { - let result = PyDict::new(relations.py()); - for (key, relation) in relations.iter() { - let schema = fixture_schema(&columns(&relation.getattr("columns")?, false)?, false); - let types = PyDict::new(relations.py()); - for (name, sql_type) in schema.expected_types { - types.set_item(name, sql_type)?; - } - result.set_item(key, types)?; +#[pyo3(signature = (*, project, models_by_name = None))] +fn build_relation_fixture_context( + project: &Bound<'_, PyAny>, + models_by_name: Option>, +) -> PyResult { + planning_context(project, models_by_name) +} + +/// Planning contexts built and SQL-test chain batches resolved, optionally resetting both. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +fn fixture_planning_work_info(py: Python<'_>, reset: bool) -> PyResult<(u64, u64)> { + let host = work_counts(py)?; + let mut host = host.borrow_mut(); + let counts = (host.context_builds, host.chain_resolutions); + if reset { + host.context_builds = 0; + host.chain_resolutions = 0; } - Ok(result) + Ok(counts) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; module.add_function(wrap_pyfunction!(build_relation_fixture_context, module)?)?; - module.add_function(wrap_pyfunction!(authoritative_fixture_columns, module)?)?; - module.add_function(wrap_pyfunction!(base_expected_fixture_types, module)?) + module.add_function(wrap_pyfunction!(fixture_planning_work_info, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs index 8839867676..ac20594665 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_metadata.rs @@ -1,41 +1,42 @@ -//! Model fixture schema materialization at the Python boundary. +//! A model's fixture relation metadata from its inferred and declared columns. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::PyTuple; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_fixtures::main::merge_metadata::merge_metadata; +use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; +use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, metadata}; +use crate::bindings::_helpers::analysis::model_reads::{inferred_rows, star_flags}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::InferredRow; +use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; -#[pyfunction] -#[pyo3(signature = (*, model))] -pub(crate) fn model_fixture_metadata<'py>( - model: &Bound<'py, PyAny>, -) -> PyResult> { - let py = model.py(); - let inferred = model.getattr("inferred_columns")?; +/// The model's merged inferred and declared columns, authoritative unless its analysis saw a star. +pub(crate) fn model_fixture_relation(model: &Bound<'_, PyAny>) -> PyResult { + let inferred: Option> = inferred_rows(model)?; let schema = model.getattr("schema_entry")?; - let declared = if schema.is_none() { - PyTuple::empty(py).into_any() - } else { - schema.getattr("columns")? - }; - let inferred_columns = if inferred.is_none() { + let declared: Vec = if schema.is_none() { Vec::new() } else { - columns(&inferred, true)? + columns(&schema.getattr("columns")?, false)? }; - let merged = merge_metadata(inferred_columns, columns(&declared, false)?); - let authoritative = !model.getattr("fast_lineage_has_star")?.is_truthy()? + let mut inferred_columns: Vec = Vec::new(); + for (name, sql_type, _) in inferred.iter().flatten() { + inferred_columns.push(FixtureColumn { + name: name.clone(), + folded_name: python_casefold(name), + sql_type: sql_type.clone(), + nullable: None, + }); + } + let authoritative = !star_flags(model)?.0 && (model .getattr("config")? .getattr("values")? .call_method1("get", ("contract",))? .eq("enforced")? - || !inferred.is_none()); - metadata(py, &merged, authoritative) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(model_fixture_metadata, module)?) + || inferred.is_some()); + Ok(FixtureRelation { + columns: merge_metadata(inferred_columns, declared), + authoritative, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs index b0b1446345..ff108f28d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_preparation.rs @@ -7,8 +7,12 @@ use sqlbuild_analysis::planner_fixtures::main::empty_fixture_marker::empty_fixtu use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::fixture_sql::complete_empty_fixture_sql; use crate::bindings::_helpers::planner_strategy::fixture_values::{diagnostic, label}; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; pub(crate) struct PreparedFixtures<'py> { + /// Each fixture's key with its Python key, in request order. + pub(crate) keys: Vec<(FixtureKey, Bound<'py, PyAny>)>, pub(crate) sql: Bound<'py, PyDict>, pub(crate) inferred: Bound<'py, PyDict>, pub(crate) null_names: Bound<'py, PyDict>, @@ -18,12 +22,13 @@ pub(crate) struct PreparedFixtures<'py> { pub(crate) fn prepare<'py>( groups: &Bound<'py, PyAny>, - relations: &Bound<'py, PyDict>, + context: &RelationFixturePlanningContext, adapter: &Bound<'py, PyAny>, - inferences: &Bound<'py, PyAny>, ) -> PyResult> { let py = groups.py(); + let inferences = context.fixture_inferences.bind(py); let mut prepared = PreparedFixtures { + keys: Vec::new(), sql: PyDict::new(py), inferred: PyDict::new(py), null_names: PyDict::new(py), @@ -36,12 +41,12 @@ pub(crate) fn prepare<'py>( let fixtures = group.get_item(1)?.cast_into::()?; for (name, sql) in fixtures.iter() { let key = PyTuple::new(py, [&kind, &name])?.into_any(); + let fixture_key: FixtureKey = key.extract()?; + prepared.keys.push((fixture_key.clone(), key.clone())); let mut sql: String = sql.extract()?; if empty_fixture_marker(&sql).map_err(value_error)? { - let relation = relations - .get_item(&key)? - .unwrap_or_else(|| py.None().into_bound(py)); - let (empty_sql, error) = complete_empty_fixture_sql(&sql, &relation, adapter)?; + let (empty_sql, error) = + complete_empty_fixture_sql(&sql, context.relations.get(&fixture_key), adapter)?; if let Some(error) = error { prepared .diagnostics diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index ce01565e3c..21d982b07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -1,67 +1,65 @@ //! Boundary conversion for native fixture read analysis. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet}; +use pyo3::types::{PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ - FixtureReadKey, FixtureReadModel, FixtureReadSchemas, FixtureStarRelation, + FixtureReadKey, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, + FixtureStarRelation, }; use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::planner_strategy::fixture_values::columns; -use crate::bindings::_helpers::planner_strategy::models::FixtureReadCacheHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; +use crate::bindings::_helpers::planner_strategy::models::{ + FixtureReadCacheHost, RelationFixturePlanningContext, +}; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; +use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use crate::bindings::types::CompilerDetach; -#[pyfunction] -#[pyo3(signature = (*, model_inputs, fixture_inputs, adapter))] -pub(crate) fn required_fixture_columns<'py>( - model_inputs: (Vec, Bound<'py, PyDict>), - fixture_inputs: (Bound<'py, PyAny>, Bound<'py, PyDict>, Bound<'py, PyDict>), - adapter: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyFrozenSet>)> { - let (ordered_model_names, model_map) = model_inputs; - let (fixture_keys, relations, authoritative_columns) = fixture_inputs; +/// The columns each fixture must supply for the chain's models, and the fixtures read through a star. +pub(crate) fn required_fixture_columns( + context: &RelationFixturePlanningContext, + ordered_model_names: &[String], + fixture_keys: &HashSet, + adapter: &Bound<'_, PyAny>, +) -> PyResult { let py = adapter.py(); - let mut keys: HashMap<(String, String), Bound<'py, PyAny>> = HashMap::new(); - for key in fixture_keys.try_iter()? { - let key = key?; - keys.insert(key.extract::<(String, String)>()?, key); - } let mut schema: Vec = Vec::new(); - let mut authoritative: HashMap<(String, String), HashSet> = HashMap::new(); - for (key, original) in &keys { - if let Some(value) = relations.get_item(original)? { - schema.push(FixtureStarRelation { - key: key.clone(), - columns: columns(&value.getattr("columns")?, false)?, - authoritative: value.getattr("authoritative_names")?.extract()?, - }); - } else { - schema.push(FixtureStarRelation { - key: key.clone(), - columns: Vec::new(), - authoritative: false, - }); - } - if let Some(value) = authoritative_columns.get_item(original)? { - authoritative.insert(key.clone(), value.extract()?); + let mut authoritative: HashMap> = HashMap::new(); + for key in fixture_keys { + let relation = context.relations.get(key); + schema.push(FixtureStarRelation { + key: key.clone(), + columns: relation + .map(|relation| relation.columns.clone()) + .unwrap_or_default(), + authoritative: relation.is_some_and(|relation| relation.authoritative), + }); + if let Some(names) = context.authoritative_columns.get(key) { + authoritative.insert(key.clone(), names.iter().cloned().collect()); } } let dialect = adapter .call_method0("expression_inference_profile")? .getattr("sql_analysis_dialect")? .extract::>()?; + let models_by_name = context.models_by_name.bind(py); let mut models: Vec = Vec::new(); for name in ordered_model_names { - let Some(model) = model_map.get_item(&name)? else { + let Some(model) = models_by_name.get_item(name)? else { + continue; + }; + let Some(facts) = context.model_facts(py, name)? else { continue; }; let mut references: Vec = Vec::new(); @@ -75,13 +73,13 @@ pub(crate) fn required_fixture_columns<'py>( )); } models.push(FixtureReadModel { - name, + name: name.clone(), key: FixtureReadKey { - sql: model.getattr("query_sql")?.extract()?, + sql: model_reads::query_sql(&model)?.extract()?, references, dialect: dialect.clone(), }, - high_confidence: lineage_reads(&model)?, + high_confidence: facts.reads.clone(), stars: model_stars(&model, &schema, adapter)?, }); } @@ -89,13 +87,12 @@ pub(crate) fn required_fixture_columns<'py>( registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?; let mut owner = owner.borrow_mut(); let cache = std::mem::take(&mut owner.cache); - let fixture_keys: HashSet<(String, String)> = keys.keys().cloned().collect(); let (cache, result) = catch_compiler_panic(|| { Ok(required_columns( cache, &models, FixtureReadSchemas { - keys: &fixture_keys, + keys: fixture_keys, authoritative: &authoritative, relations: &schema, }, @@ -103,41 +100,20 @@ pub(crate) fn required_fixture_columns<'py>( }) .map_err(value_error)?; owner.cache = cache; - let result = result.map_err(|error| value_error(format!("{error:?}")))?; - let required = PyDict::new(py); - for (key, columns) in result.columns { - let values = PyDict::new(py); - for column in columns { - values.set_item(column.name, PySet::new(py, column.readers)?)?; - } - required.set_item(&keys[&key], values)?; - } - let stars = PyFrozenSet::new(py, result.stars.iter().map(|key| &keys[key]))?; - Ok((required, stars)) + result.map_err(|error| value_error(format!("{error:?}"))) } -fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let mut reads: Vec = Vec::new(); - let facts = model.getattr("fast_lineage_columns")?; - if facts.is_none() { - return Ok(reads); - } - for fact in facts.try_iter()? { - let fact = fact?; - if !fact.getattr("confidence")?.eq(registry::class( - model.py(), - RecordId::ColumnLineageConfidence, - )? - .getattr("HIGH")?)? - { + for (_, _, confidence, sources) in lineage_facts(model)?.unwrap_or_default() { + if confidence != HIGH_CONFIDENCE { continue; } - for source in fact.getattr("upstream_columns")?.try_iter()? { - let source = source?; + for (resource_type, resource_name, column_name) in sources { reads.push(ColumnRead { - resource_type: source.getattr("resource_type")?.extract()?, - resource_name: source.getattr("resource_name")?.extract()?, - column_name: source.getattr("column_name")?.extract()?, + resource_type, + resource_name, + column_name, }); } } @@ -149,16 +125,14 @@ fn model_stars( schema: &[FixtureStarRelation], adapter: &Bound<'_, PyAny>, ) -> PyResult> { - if !model.getattr("fast_lineage_has_star")?.is_truthy()? { + if !star_flags(model)?.0 { return Ok(Vec::new()); } let mut deps: Vec<(String, String)> = Vec::new(); - for dependency in model.getattr("deps")?.try_iter()? { - let dependency = dependency?; - let key = ( - dependency.getattr("resource_type")?.extract::()?, - dependency.getattr("name")?.extract::()?, - ); + for key in model + .getattr("dep_rows")? + .extract::>()? + { if schema.iter().any(|relation| relation.key == key) { deps.push(key); } @@ -175,13 +149,7 @@ fn model_stars( }); } } - let contract = model.getattr("dynamic_column_contract")?; - let inputs: Option> = - if !contract.is_none() && contract.getattr("output_proven")?.is_truthy()? { - Some(contract.getattr("input_relations")?.extract()?) - } else { - None - }; + let inputs: Option> = proven_inputs(model)?; let dialect: Option = if inputs.is_none() && selected.iter().any(|relation| !relation.columns.is_empty()) { adapter @@ -191,7 +159,7 @@ fn model_stars( } else { None }; - let sql: String = model.getattr("query_sql")?.extract()?; + let sql: String = model_reads::query_sql(model)?.extract()?; model .py() .compiler_detach(|| { @@ -210,8 +178,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { "_fixture_read_cache", pyo3::Py::new(module.py(), FixtureReadCacheHost::default())?, )?; - module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?)?; - module.add_function(wrap_pyfunction!(required_fixture_columns, module)?) + module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) } /// Report resolver misses and retained entries, optionally clearing prior fixture reuse. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs index c21012b96e..3f25962c17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_selection.rs @@ -1,66 +1,60 @@ -//! Fixture analysis selection and diagnostic boundary conversions. +//! Fixture analysis selection and unknown-column diagnostics over the planning context. -use std::collections::BTreeSet; +use std::collections::HashSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::PyDictMethods; use sqlbuild_analysis::planner_fixtures::main::{ fixture_schema::fixture_schema, requires_analysis::requires_analysis, unknown_columns::unknown_columns, }; +use crate::bindings::_helpers::planner_strategy::fixture_preparation::PreparedFixtures; use crate::bindings::_helpers::planner_strategy::fixture_values::{columns, diagnostic, label}; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; -#[pyfunction] -#[pyo3(signature = (*, inferred_by_fixture, authoritative_columns))] -pub(crate) fn fixture_keys_requiring_analysis<'py>( - inferred_by_fixture: &Bound<'py, PyDict>, - authoritative_columns: &Bound<'py, PyDict>, -) -> PyResult> { - let mut selected: Vec> = Vec::new(); - for (key, inferred) in inferred_by_fixture.iter() { - let expected: Option> = authoritative_columns - .get_item(&key)? - .map(|names| names.extract()) - .transpose()?; - if requires_analysis(&columns(&inferred, true)?, expected.as_ref()) { - selected.push(key); +/// Fixtures whose inferred columns do not already match an authoritative relation shape. +pub(crate) fn fixture_keys_requiring_analysis( + prepared: &PreparedFixtures<'_>, + context: &RelationFixturePlanningContext, +) -> PyResult> { + let mut selected: HashSet = HashSet::new(); + for (key, python_key) in &prepared.keys { + let Some(inferred) = prepared.inferred.get_item(python_key)? else { + continue; + }; + if requires_analysis( + &columns(&inferred, true)?, + context.authoritative_columns.get(key), + ) { + selected.insert(key.clone()); } } - PyFrozenSet::new(inferred_by_fixture.py(), selected) + Ok(selected) } -#[pyfunction] -#[pyo3(signature = (*, inferred_by_fixture, relations))] +/// Diagnostics for fixtures that supply columns an authoritative relation does not have. pub(crate) fn unknown_fixture_column_diagnostics<'py>( - inferred_by_fixture: &Bound<'py, PyDict>, - relations: &Bound<'py, PyDict>, -) -> PyResult> { - let mut diagnostics = Vec::>::new(); - for (key, inferred) in inferred_by_fixture.iter() { - let Some(relation) = relations.get_item(&key)? else { + prepared: &PreparedFixtures<'py>, + context: &RelationFixturePlanningContext, +) -> PyResult>> { + let mut diagnostics: Vec> = Vec::new(); + for (key, python_key) in &prepared.keys { + let Some(inferred) = prepared.inferred.get_item(python_key)? else { continue; }; - let schema = fixture_schema( - &columns(&relation.getattr("columns")?, false)?, - relation.getattr("authoritative_names")?.extract()?, - ); + let Some(relation) = context.relations.get(key) else { + continue; + }; + let schema = fixture_schema(&relation.columns, relation.authoritative); if let Some(message) = unknown_columns( - &label(&key)?, + &label(python_key)?, &columns(&inferred, true)?, schema.authoritative_names.as_ref(), ) { - diagnostics.push(diagnostic(&key, &message)?); + diagnostics.push(diagnostic(python_key, &message)?); } } - PyTuple::new(inferred_by_fixture.py(), diagnostics) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_keys_requiring_analysis, module)?)?; - module.add_function(wrap_pyfunction!( - unknown_fixture_column_diagnostics, - module - )?) + Ok(diagnostics) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs index e5f4eb6767..7f8f327f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_sql.rs @@ -7,38 +7,37 @@ use sqlbuild_analysis::planner_fixtures::main::{ empty_fixture_error::empty_fixture_error, empty_fixture_marker::empty_fixture_marker, render_completed::render_completed as native_render_completed, render_empty::render_empty, }; -use sqlbuild_analysis::planner_fixtures::models::{FixtureSqlInput, SuppliedProjection}; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureColumn, FixtureSqlInput, SuppliedProjection, +}; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::planner_strategy::fixture_values::columns; +use crate::bindings::_helpers::planner_strategy::models::FixtureRelation; #[pyfunction(name = "is_empty_fixture_query")] pub(crate) fn fixture_marker(sql: &str) -> PyResult { empty_fixture_marker(sql).map_err(value_error) } -#[pyfunction] -#[pyo3(signature = (*, sql, relation, adapter))] +/// An empty-fixture marker rendered as the relation's typed empty projection, or why it cannot be. pub(crate) fn complete_empty_fixture_sql( sql: &str, - relation: &Bound<'_, PyAny>, + relation: Option<&FixtureRelation>, adapter: &Bound<'_, PyAny>, ) -> PyResult<(String, Option)> { if !fixture_marker(sql)? { return Ok((sql.into(), None)); } - let authoritative = - !relation.is_none() && relation.getattr("authoritative_names")?.is_truthy()?; - let metadata = if authoritative { - columns(&relation.getattr("columns")?, false)? - } else { - Vec::new() + let authoritative = relation.is_some_and(|relation| relation.authoritative); + let metadata: &[FixtureColumn] = match relation { + Some(relation) if authoritative => &relation.columns, + _ => &[], }; - if let Some(error) = empty_fixture_error(authoritative, &metadata) { + if let Some(error) = empty_fixture_error(authoritative, metadata) { return Ok((sql.into(), Some(error))); } Ok(( - render_empty(&metadata, |name| { + render_empty(metadata, |name| { adapter .call_method1("render_identifier", (name,))? .extract::() @@ -52,7 +51,7 @@ pub(crate) struct CompletionSql<'py, 'a> { pub(crate) inferred: Bound<'py, PyAny>, pub(crate) typed_nulls: Bound<'py, PyDict>, pub(crate) quoted_names: Bound<'py, PyAny>, - pub(crate) completed: Bound<'py, PyAny>, + pub(crate) completed: Vec, } pub(crate) fn render_completed( @@ -81,7 +80,7 @@ pub(crate) fn render_completed( sql: input.sql, supplied: &supplied, typed_nulls: !input.typed_nulls.is_empty(), - completed: &columns(&input.completed, false)?, + completed: &input.completed, }, |name, exact| { adapter @@ -98,19 +97,19 @@ pub(crate) fn render_completed( ) } -#[pyfunction] -#[pyo3(signature = (*, sql, inference, expected_types, adapter))] +/// Fixture SQL with its untyped null columns cast to the expected types. pub(crate) fn complete_typed_null_columns( sql: &str, inference: &Bound<'_, PyAny>, - expected_types: &Bound<'_, PyDict>, + expected_types: &[(String, String)], adapter: &Bound<'_, PyAny>, ) -> PyResult { let py = inference.py(); let typed_nulls = PyDict::new(py); for name in inference.getattr("null_literal_names")?.try_iter()? { let name = name?; - if let Some(expected) = expected_types.get_item(&name)? { + let folded: String = name.extract()?; + if let Some((_, expected)) = expected_types.iter().find(|(column, _)| *column == folded) { typed_nulls.set_item(name, expected)?; } } @@ -123,14 +122,12 @@ pub(crate) fn complete_typed_null_columns( inferred: inference.getattr("columns")?, typed_nulls, quoted_names: inference.getattr("quoted_names")?, - completed: pyo3::types::PyTuple::empty(py).into_any(), + completed: Vec::new(), }, adapter, ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(fixture_marker, module)?)?; - module.add_function(wrap_pyfunction!(complete_empty_fixture_sql, module)?)?; - module.add_function(wrap_pyfunction!(complete_typed_null_columns, module)?) + module.add_function(wrap_pyfunction!(fixture_marker, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index beb34dfd93..3a6a1dd758 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -1,43 +1,49 @@ //! Expected fixture type propagation boundary conversions. use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use sqlbuild_analysis::planner_fixtures::main::propagate_types::propagate_types; use sqlbuild_analysis::planner_fixtures::models::{ FixtureTypeModel, FixtureTypeProjection, FixtureTypeRead, FixtureTypeState, }; -use crate::bindings::_helpers::planner_strategy::fixture_values::folded; -use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::records::registry; +use sqlbuild_core::text::main::python_casefold::python_casefold; -#[pyfunction] -#[pyo3(signature = (*, base_expected_types, ordered_model_names, model_map))] -pub(crate) fn expected_fixture_types<'py>( - base_expected_types: &Bound<'py, PyDict>, - ordered_model_names: Vec, - model_map: &Bound<'py, PyDict>, -) -> PyResult> { +use crate::bindings::_helpers::analysis::model_reads::lineage_facts; +use crate::bindings::_helpers::planner_strategy::constants::{ + DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, +}; +use crate::bindings::_helpers::planner_strategy::models::{ + ExpectedTypes, RelationFixturePlanningContext, +}; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; + +/// Each chain model's known column types with direct lineage types propagated from its inputs. +pub(crate) fn expected_fixture_types<'a>( + context: &'a RelationFixturePlanningContext, + py: Python<'_>, + ordered_model_names: &[String], +) -> PyResult> { + let base = &context.expected_types; let mut state = FixtureTypeState::default(); - let result = PyDict::new(base_expected_types.py()); + let mut overlay: Vec<(FixtureKey, Vec<(String, String)>)> = Vec::new(); let mut models: Vec = Vec::new(); for name in ordered_model_names { - let Some(model) = model_map.get_item(&name)? else { + let Some(facts) = context.model_facts(py, name)? else { continue; }; - let key = ("model".to_owned(), name); - let types = base_expected_types - .get_item(&key)? + let key: FixtureKey = (MODEL_REFERENCE_KIND.to_owned(), name.clone()); + let types = base + .get(&key) .ok_or_else(|| PyKeyError::new_err(key.clone()))? - .cast_into::()? - .copy()?; - state.known.insert(key.clone(), types.extract()?); - result.set_item(&key, types)?; + .clone(); + state + .known + .insert(key.clone(), types.iter().cloned().collect()); + overlay.push((key.clone(), types)); models.push(FixtureTypeModel { key, - projections: projections(&model)?, + projections: facts.projections.clone(), }); } for model in &models { @@ -50,63 +56,53 @@ pub(crate) fn expected_fixture_types<'py>( { continue; } - if let Some(types) = base_expected_types.get_item(&read.key)? - && let Some(value) = types.cast_into::()?.get_item(&read.column)? - { + if let Some(sql_type) = base.get(&read.key).and_then(|types| { + types + .iter() + .find(|(name, _)| *name == read.column) + .map(|(_, sql_type)| sql_type.clone()) + }) { state .known .entry(read.key.clone()) .or_default() - .insert(read.column.clone(), value.extract()?); + .insert(read.column.clone(), sql_type); } } } } let state = propagate_types(state, &models); for (read, sql_type) in state.additions { - result - .get_item(read.key)? - .ok_or_else(|| PyKeyError::new_err("fixture types"))? - .set_item(read.column, sql_type)?; + let types = overlay + .iter_mut() + .find(|(key, _)| *key == read.key) + .map(|(_, types)| types) + .ok_or_else(|| PyKeyError::new_err("fixture types"))?; + match types.iter_mut().find(|(name, _)| *name == read.column) { + Some(existing) => existing.1 = sql_type, + None => types.push((read.column, sql_type)), + } } - base_expected_types - .py() - .import("collections")? - .getattr("ChainMap")? - .call1((result, base_expected_types)) + Ok(ExpectedTypes { overlay, base }) } -fn projections(model: &Bound<'_, PyAny>) -> PyResult> { - let lineage = model.getattr("fast_lineage_columns")?; +pub(crate) fn projections(model: &Bound<'_, PyAny>) -> PyResult> { let mut projections: Vec = Vec::new(); - if lineage.is_none() { - return Ok(projections); - } - let direct = registry::class(model.py(), RecordId::ColumnTransformKind)?.getattr("DIRECT")?; - for column in lineage.try_iter()? { - let column = column?; - if !column.getattr("transform_kind")?.eq(&direct)? { + for (output_column, transform, _, sources) in lineage_facts(model)?.unwrap_or_default() { + if transform != DIRECT_TRANSFORM { continue; } - let mut upstream: Vec = Vec::new(); - for source in column.getattr("upstream_columns")?.try_iter()? { - let source = source?; + let mut upstream: Vec = Vec::with_capacity(sources.len()); + for (resource_type, resource_name, column_name) in sources { upstream.push(FixtureTypeRead { - key: ( - source.getattr("resource_type")?.extract()?, - source.getattr("resource_name")?.extract()?, - ), - column: folded(&source.getattr("column_name")?)?, + key: (resource_type, resource_name), + column: python_casefold(&column_name), }); } projections.push(FixtureTypeProjection { - output_column: folded(&column.getattr("output_column")?)?, + output_column: python_casefold(&output_column), upstream, }); } Ok(projections) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(expected_fixture_types, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs index 2177f21723..5d621a37c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs @@ -3,17 +3,13 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; pub(crate) fn folded(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("casefold")?.extract() } -pub(crate) fn strings(values: &Bound<'_, PyAny>) -> PyResult> { - values.try_iter()?.map(|value| value?.extract()).collect() -} - pub(crate) fn columns(values: &Bound<'_, PyAny>, inferred: bool) -> PyResult> { values .try_iter()? @@ -42,27 +38,6 @@ pub(crate) fn record<'py>( registry::class(py, name)?.call((), Some(kwargs)) } -pub(crate) fn metadata<'py>( - py: Python<'py>, - columns: &[FixtureColumn], - authoritative: bool, -) -> PyResult> { - let values = columns - .iter() - .map(|column| { - let kwargs = PyDict::new(py); - kwargs.set_item("name", &column.name)?; - kwargs.set_item("type", &column.sql_type)?; - kwargs.set_item("nullable", column.nullable)?; - record(py, RecordId::FixtureColumnMetadata, &kwargs) - }) - .collect::>>()?; - let kwargs = PyDict::new(py); - kwargs.set_item("columns", PyTuple::new(py, values)?)?; - kwargs.set_item("authoritative_names", authoritative)?; - record(py, RecordId::FixtureRelationMetadata, &kwargs) -} - pub(crate) fn label(key: &Bound<'_, PyAny>) -> PyResult { Ok(format!( "mock {} '{}'", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs index 9fc4edac60..c790f9de30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/lineage.rs @@ -12,6 +12,7 @@ use sqlbuild_analysis::planner_strategy::main::lineage_reference::lineage_refere use sqlbuild_analysis::planner_strategy::models::{ProducerFacts, ReferenceIdentity}; use sqlbuild_analysis::planner_strategy::types::ReferenceGraph; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::model_config::config_values::config_str; use crate::bindings::_helpers::planner_strategy::microbatches::{ @@ -126,7 +127,7 @@ pub(crate) fn resolve_lineage_reference<'py>( models_by_name, functions_by_name, }; - let name: String = text(&model.getattr("name")?)?; + let name: String = model_reads::model_name(model)?; lineage_reference(&graph, (&name, references(model)?), input_name) .map_err(|failure| lineage_error(py, failure)) } @@ -134,7 +135,7 @@ pub(crate) fn resolve_lineage_reference<'py>( /// Reject capped producers feeding a microbatch model's watermark inputs. #[pyfunction(name = "validate_capped_watermark_inputs")] #[pyo3(signature = (*, model, models_by_name, functions_by_name = None))] -fn reject_capped_watermark_inputs<'py>( +pub(crate) fn reject_capped_watermark_inputs<'py>( model: &Bound<'py, PyAny>, models_by_name: Bound<'py, PyAny>, functions_by_name: Option>, @@ -151,7 +152,7 @@ fn reject_capped_watermark_inputs<'py>( models_by_name, functions_by_name, }; - let name: String = text(&model.getattr("name")?)?; + let name: String = model_reads::model_name(model)?; let inputs: Vec = watermark_names(model)?; match capped_watermark_input(&graph, (&name, references(model)?), inputs) { Ok(None) => Ok(()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs index ca7401805f..b29e2442b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/mod.rs @@ -19,4 +19,5 @@ pub(crate) mod model_entries; pub(crate) mod models; pub(crate) mod registration; pub(crate) mod strategy; +pub(crate) mod types; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 0b69c266ef..9143134c54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -9,8 +9,9 @@ use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_l use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; use sqlbuild_analysis::planner_strategy::models::LookbackConfig; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::{ config_bool, config_cursor_bound, config_int, config_str, config_string_tuple, @@ -22,10 +23,14 @@ use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; +use crate::bindings::_helpers::planner_strategy::cursor_types::check_cursor_type_consistency; use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; +use crate::bindings::_helpers::planner_strategy::strategy::{ + build_model_warnings, resolve_schema_actions, +}; use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; /// Model config keys copied to the plan entry as optional strings, with their entry fields. @@ -165,7 +170,7 @@ fn on_schema_change<'py>(model: &Bound<'py, PyAny>) -> PyResult( let py: Python<'py> = request.py(); let adapter: Bound<'py, PyAny> = request.get("adapter")?; let model: Bound<'py, PyAny> = request.get("model")?; - let destination: Bound<'py, PyAny> = model.getattr("destination")?; + let destination: Bound<'py, PyAny> = model_reads::destination(&model)?; let qualified: Bound<'py, PyAny> = destination.getattr("qualified_name")?; let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { qualified @@ -332,34 +337,40 @@ fn model_warnings<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let model: Bound<'py, PyAny> = request.get("model")?; - let name: Bound<'py, PyAny> = model.getattr("name")?; - let mut warnings: Vec> = call_with( - ®istry::native(py, NativeAttr::BuildModelWarnings)?, - &[ - ("model_name", name.clone()), - ("materialization_type", request.get("materialization_type")?), - ("change_result", request.get("change_result")?), - ("schema_actions", schema_actions.clone()), - ("on_schema_change", policy.clone()), - ("type_enforcement", enforcement.clone()), - ], + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; + let mut warnings: Vec> = build_model_warnings( + py, + Some(keywords( + py, + &[ + ("model_name", name.clone()), + ("materialization_type", request.get("materialization_type")?), + ("change_result", request.get("change_result")?), + ("schema_actions", schema_actions.clone()), + ("on_schema_change", policy.clone()), + ("type_enforcement", enforcement.clone()), + ], + )?), )? .try_iter()? .collect::>()?; - let cursor_warning: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CheckCursorTypeConsistency)?, - &[ - ("model_name", name), - ("cursor_column", request.get("cursor_column")?), - ( - "cursor_type", - or_none(py, config_str(&model_values(&model)?, "cursor_type")?), - ), - ("warehouse_columns", warehouse_columns.clone()), - ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), - ], + let cursor_warning: Option> = check_cursor_type_consistency( + py, + Some(keywords( + py, + &[ + ("model_name", name), + ("cursor_column", request.get("cursor_column")?), + ( + "cursor_type", + or_none(py, config_str(&model_values(&model)?, "cursor_type")?), + ), + ("warehouse_columns", warehouse_columns.clone()), + ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), + ], + )?), )?; - if !cursor_warning.is_none() { + if let Some(cursor_warning) = cursor_warning.filter(|warning| !warning.is_none()) { warnings.push(cursor_warning); } PyTuple::new(py, warnings) @@ -490,15 +501,10 @@ pub(crate) fn assemble_model_plan_entry<'py>( let values: Bound<'py, PyAny> = model_values(&model)?; let change: Bound<'py, PyAny> = request.get("change_result")?; let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; - let name: Bound<'py, PyAny> = model.getattr("name")?; + let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; let policy: Bound<'py, PyAny> = on_schema_change(&model)?; - let schema_actions: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveSchemaActions)?, - &[ - ("schema_findings", change.getattr("schema_findings")?), - ("on_schema_change", policy.clone()), - ], - )?; + let schema_actions: Bound<'py, PyAny> = + resolve_schema_actions(&change.getattr("schema_findings")?, &policy)?.into_any(); let enforcement: Bound<'py, PyAny> = type_enforcement(&model)?; let warehouse_columns: Bound<'py, PyAny> = snapshot .getattr("existing_columns")? @@ -522,10 +528,11 @@ pub(crate) fn assemble_model_plan_entry<'py>( (&warehouse_columns, ddl_cursor_bounds(&request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); - for field in ["key", "name", "relative_path", "destination"] { - entry.set_item(field, model.getattr(field)?)?; - } - entry.set_item("fingerprint_query_sql", model.getattr("query_sql")?)?; + entry.set_item("key", model_reads::model_key(&model)?)?; + entry.set_item("name", &name)?; + entry.set_item("relative_path", model.getattr("relative_path")?)?; + entry.set_item("destination", model_reads::destination(&model)?)?; + entry.set_item("fingerprint_query_sql", model_reads::query_sql(&model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; for field in PASSED_FIELDS { entry.set_item(field, request.get(field)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs index f3abaa0b26..6c998ddebe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -1,11 +1,58 @@ -//! Extension-owned fixture analysis reuse, with pure state passed into analysis. +//! Extension-owned fixture analysis reuse and the project-wide fixture planning context. +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, Mutex}; + +use pyo3::prelude::Py; use pyo3::pyclass; -use sqlbuild_analysis::planner_fixtures::models::FixtureReadCache; +use pyo3::types::{PyAny, PyDict}; +use sqlbuild_analysis::planner_fixtures::models::{ + FixtureColumn, FixtureReadCache, FixtureTypeProjection, +}; +use sqlbuild_analysis::required_columns::models::ColumnRead; + +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; /// Per-extension cache replacing the Python fixture resolver's LRU. #[pyclass] #[derive(Debug, Default)] pub(crate) struct FixtureReadCacheHost { pub(crate) cache: FixtureReadCache, + /// Fixture planning contexts built since the last reset. + pub(crate) context_builds: u64, + /// SQL-test model chain batches resolved since the last reset. + pub(crate) chain_resolutions: u64, +} + +/// One relation's known fixture columns and whether they form an authoritative shape. +#[derive(Clone, Debug)] +pub(crate) struct FixtureRelation { + pub(crate) columns: Vec, + pub(crate) authoritative: bool, +} + +/// One model's high-confidence lineage reads and direct type projections. +#[derive(Debug)] +pub(crate) struct ModelFixtureFacts { + pub(crate) reads: Vec, + pub(crate) projections: Vec, +} + +/// Project-wide fixture metadata shared across test and scenario planning. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct RelationFixturePlanningContext { + pub(crate) models_by_name: Py, + pub(crate) relations: HashMap, + pub(crate) authoritative_columns: HashMap>, + pub(crate) expected_types: HashMap>, + pub(crate) fixture_inferences: Py, + pub(crate) model_facts: Mutex>>, +} + +/// The expected column types of one completion: its chain's propagated types over the context's. +#[derive(Debug)] +pub(crate) struct ExpectedTypes<'a> { + pub(crate) overlay: Vec<(FixtureKey, Vec<(String, String)>)>, + pub(crate) base: &'a HashMap>, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index c5bdbebbc5..89538ed245 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -4,8 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use pyo3::{prelude::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::planner_strategy::{ - audits, cursor_types, fixture_completion, fixture_metadata, fixture_sql, lineage, microbatches, - strategy, + audits, cursor_types, fixture_completion, fixture_sql, lineage, microbatches, strategy, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -53,12 +52,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audits::register(module)?; cursor_types::register(module)?; fixture_completion::register(module)?; - fixture_metadata::register(module)?; fixture_sql::register(module)?; super::fixture_context::register(module)?; super::fixture_reads::register(module)?; - super::fixture_selection::register(module)?; - super::fixture_types::register(module)?; lineage::register(module)?; microbatches::register(module)?; strategy::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index cf13bb91c8..ad4e0aa5eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -1,5 +1,6 @@ //! Per-model plan actions, schema actions and plan warnings. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; @@ -61,7 +62,7 @@ fn schema_policy(on_schema_change: &Bound<'_, PyAny>) -> PyResult /// The `MaterializationType` of a model's `materialized` config: table when unset, custom when unknown. #[pyfunction(name = "get_materialization_type")] #[pyo3(signature = (model))] -fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn materialization_of<'py>(model: &Bound<'py, PyAny>) -> PyResult> { planner_enum( model.py(), RecordId::MaterializationType, @@ -80,7 +81,7 @@ pub(crate) fn resolve_model_plan_action<'py>( let py: Python<'py> = model.py(); let enabled: Bound<'py, PyAny> = config_value(model, "enabled")?; let strategy = ModelStrategy { - name: text(&model.getattr("name")?)?, + name: model_reads::model_name(model)?, disabled: enabled.is_instance_of::() && !enabled.is_truthy()?, materialization: model_materialization(model)?, incremental_strategy: optional_string(&config_value(model, "incremental_strategy")?)?, @@ -108,7 +109,7 @@ pub(crate) fn resolve_model_plan_action<'py>( /// Concrete schema change actions from findings and `on_schema_change`. #[pyfunction] #[pyo3(signature = (*, schema_findings, on_schema_change))] -fn resolve_schema_actions<'py>( +pub(crate) fn resolve_schema_actions<'py>( schema_findings: &Bound<'py, PyAny>, on_schema_change: &Bound<'py, PyAny>, ) -> PyResult> { @@ -140,7 +141,7 @@ fn resolve_schema_actions<'py>( /// Warnings for one model from its change detection and plan decisions. #[pyfunction] #[pyo3(signature = (**options))] -fn build_model_warnings<'py>( +pub(crate) fn build_model_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs new file mode 100644 index 0000000000..2ae899fbc1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/types.rs @@ -0,0 +1,7 @@ +//! Fixture planning key and input shapes. + +/// `(resource type value, name)` of one fixture relation. +pub(crate) type FixtureKey = (String, String); + +/// One relation's `((name, type, nullable), ...)` columns and whether they are authoritative. +pub(crate) type FixtureRelationInput = (Vec<(String, Option, Option)>, bool); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index f316e0e19d..5fff2d990a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -1,5 +1,6 @@ //! Cursor MIN/MAX inspection of selected incremental models and their watermark inputs. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_phases::objects::{ @@ -346,7 +347,7 @@ fn collect_models<'py>( let mut models: Vec<_> = Vec::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - let name_value = model.getattr("name")?; + let name_value = model_reads::name_object(&model)?; let model_name: String = name_value.extract()?; if !inputs.selected_keys.is_none() && !inputs.selected_keys.contains(model_key(py, &name_value)?)? @@ -390,7 +391,7 @@ fn collect_models<'py>( None => None, }; let target_info = existing_relations.get_item(&name_value)?; - let qualified = model.getattr("destination")?.getattr("qualified_name")?; + let qualified = model_reads::destination(&model)?.getattr("qualified_name")?; if let Some(redirected) = redirected.filter(|value| !value.is_none()) { target_tag = Some(format!("{model_name}__target__max")); target_relation = Some(redirected.extract::()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs index f91f79d345..64ecf9b14b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/semantic_sources.rs @@ -1,5 +1,6 @@ //! Complete missing source metadata in bounded, database-grouped batches. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; @@ -192,7 +193,7 @@ pub(crate) fn semantic_source_columns<'py>( .getattr("config")? .getattr("values")? .call_method1("get", ("sql_analysis",))?; - if !selected_keys.contains(model.getattr("key")?)? + if !selected_keys.contains(model_reads::model_key(&model)?)? || analysis.is(pyo3::types::PyBool::new(py, false)) { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index 40177197f8..d0a9af43c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -1,5 +1,6 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; @@ -193,7 +194,11 @@ fn metadata_name_filter( for key in selected.try_iter()? { let name = key?.getattr("name")?; if let Some(model) = models.get_item(&name)? { - names.insert(model.getattr("destination")?.getattr("name")?.extract()?); + names.insert( + model_reads::destination(&model)? + .getattr("name")? + .extract()?, + ); names.extend(model_upstream_names( &model, (&models, &seeds), @@ -461,7 +466,7 @@ fn shares_unfiltered(project: &Bound<'_, PyAny>, selected: &Bound<'_, PyAny>) -> let mut any_migration = false; for model in project.getattr("models")?.try_iter()? { let model = model?; - if !names.contains(&model.getattr("name")?.extract::()?) { + if !names.contains(&model_reads::model_name(&model)?) { all_selected = false; } if model @@ -847,7 +852,7 @@ pub(crate) fn gather_redirected_cursor_snapshots<'py>( &snapshot, &[( "target_relation", - model.getattr("destination")?.getattr("qualified_name")?, + model_reads::destination(&model)?.getattr("qualified_name")?, )], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs index d31120fc7b..b55cad2956 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_deferral.rs @@ -1,5 +1,6 @@ //! Source read deferral for managed source loaders. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -83,10 +84,10 @@ fn selected_managed_refs( let mut names: BTreeSet<_> = BTreeSet::new(); for model in project.getattr("models")?.try_iter()? { let model = model?; - if !selected.contains(model.getattr("key")?)? { + if !selected.contains(model_reads::model_key(&model)?)? { continue; } - names.extend(sql_source_reads(&model.getattr("query_sql")?, managed)?); + names.extend(sql_source_reads(&model_reads::query_sql(&model)?, managed)?); names.extend(hook_source_reads(&model, managed)?); } for function in project.getattr("functions")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 6fa37b875c..cb56a20b1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -1,5 +1,6 @@ //! Direct planner source freshness observation and the warnings for unknown freshness. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::freshness::planning::{ @@ -93,7 +94,7 @@ fn state_schemas( if model.is_none() { continue; } - let schema = model.getattr("destination")?.getattr("schema")?; + let schema = model_reads::destination(&model)?.getattr("schema")?; if !schema.is_none() { schemas.insert(schema.extract()?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs index ab046e7c97..f9c629d6fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/watermarks.rs @@ -1,5 +1,6 @@ //! Validate reliable watermark column contracts before warehouse inspection. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::planner_phases::objects::{config_str, planner_error, text}; use crate::bindings::_helpers::planner_warehouse::constants::{ CURSOR_TYPE_INTEGER, CURSOR_TYPE_TIMESTAMP, WATERMARK_CONTRACT_CODE, @@ -62,7 +63,7 @@ pub(crate) fn validate_watermark_column<'py>( let py = model.py(); let kind = reference.getattr("ref_kind")?; let ref_name: String = reference.getattr("ref_name")?.extract()?; - let model_name: String = model.getattr("name")?.extract()?; + let model_name: String = model_reads::model_name(model)?; let lowered = |value: &str| -> PyResult { text(py, value).call_method0("lower")?.extract() }; let (names, types) = if kind.eq("ref")? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 78b2df5d8d..e23a03ebc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -250,8 +250,6 @@ native_ids! { FingerprintSet => "FingerprintSet", FixVerdict => "FixVerdict", FixtureColumnInference => "FixtureColumnInference", - FixtureColumnMetadata => "FixtureColumnMetadata", - FixtureRelationMetadata => "FixtureRelationMetadata", FreshnessCommandRequest => "FreshnessCommandRequest", FunctionArgument => "FunctionArgument", FunctionChangeResult => "FunctionChangeResult", @@ -446,7 +444,6 @@ native_ids! { RelatedLocation => "RelatedLocation", RelationFixtureCompletion => "RelationFixtureCompletion", RelationFixtureDiagnostic => "RelationFixtureDiagnostic", - RelationFixturePlanningContext => "RelationFixturePlanningContext", ResolvedDbtConfig => "ResolvedDbtConfig", ResolvedDbtProfileOutput => "ResolvedDbtProfileOutput", ResolvedModelAction => "ResolvedModelAction", @@ -698,7 +695,6 @@ native_ids! { ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleBasePlanOutput => "assemble_base_plan_output", AssembleModelPlanEntry => "assemble_model_plan_entry", AssembleNativeProjectResources => "assemble_native_project_resources", AssembleNativeSqlTests => "assemble_native_sql_tests", @@ -752,7 +748,6 @@ native_ids! { BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", BuildPlanEntries => "build_plan_entries", - BuildPlanOutput => "build_plan_output", BuildPlannerEntryResults => "build_planner_entry_results", BuildPlannerIdentityContext => "build_planner_identity_context", BuildPlannerRelationsContext => "build_planner_relations_context", @@ -1249,9 +1244,6 @@ native_ids! { WithDuplicateLiteralHints => "with_duplicate_literal_hints", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", - WithPlanMetadata => "with_plan_metadata", - WithPlanWarnings => "with_plan_warnings", - WithStoragePolicies => "with_storage_policies", WithTypeRecovery => "with_type_recovery", WithoutDeclaredColumnsHash => "without_declared_columns_hash", WithoutDestinationCursor => "without_destination_cursor", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 19c80e379a..327da1b23d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -312,14 +312,4 @@ impl FieldSpec { shown: false, } } - - /// A field left out of equality and hashing but shown in repr. - pub(crate) const fn uncompared_shown(name: &'static str, default: FieldDefault) -> Self { - Self { - name, - default, - compared: false, - shown: true, - } - } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs index 872e7c9fc0..536be5e414 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/cursor_window.rs @@ -8,6 +8,7 @@ use sqlbuild_cursor::algebra::main::compare::compare; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::algebra::models::{Grain, Scalar}; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::attachments::cursor_intrinsics::{ compile_input_error, resolve_cursor_intrinsics, }; @@ -86,7 +87,7 @@ pub(super) fn declared_window_cursor_models<'py>( let py = test.py(); let mut cursor_models: Vec> = Vec::new(); for model in chain_models { - if uses_cursor_intrinsics(py, &model.getattr("query_sql")?.extract::()?)? { + if uses_cursor_intrinsics(py, &model_reads::query_sql(&model)?.extract::()?)? { cursor_models.push(model); } } @@ -255,7 +256,7 @@ fn model_cursor<'py>(model: &Bound<'py, PyAny>) -> PyResult> { _ => None, }; Ok(ModelCursor { - name: model.getattr("name")?.extract()?, + name: model_reads::name_object(model)?.extract()?, cursor_type, type_name, grain, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs index 6915959b3b..fce670a9e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -10,11 +10,13 @@ use sqlbuild_analysis::planner_fixtures::main::{ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_strategy::fixture_completion::build_relation_fixture_completion; -use crate::bindings::_helpers::planner_strategy::fixture_context::build_relation_fixture_context; +use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures; use crate::bindings::_helpers::planner_strategy::fixture_sql::{ complete_empty_fixture_sql, complete_typed_null_columns, }; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, FIXTURE_PREFIXES}; /// Report every statically provable fixture defect or return the completed fixture SQL groups. @@ -22,11 +24,11 @@ pub(super) fn validated_fixtures<'py>( test: &Bound<'py, PyAny>, request: (&Bound<'py, PyAny>, [Bound<'py, PyDict>; 4]), inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, + context: &RelationFixturePlanningContext, ) -> PyResult; 4], Vec>> { let py = test.py(); let (chain, [refs, sources, seeds, expected]) = request; - let (project, adapter) = inputs; + let adapter = inputs.1; let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; let model = resource_types.getattr("MODEL")?; let kinds = [ @@ -42,20 +44,10 @@ pub(super) fn validated_fixtures<'py>( PyTuple::new(py, [kinds[2].clone(), seeds.clone().into_any()])?, ], )?; - let context = if context.is_truthy()? { - context.clone() - } else { - build_relation_fixture_context(project, None)? - }; - let completion = build_relation_fixture_completion( - project, - adapter, - (chain.clone(), groups.into_any()), - Some(context.clone()), - )?; + let chain: Vec = chain.extract()?; + let completion = complete_fixtures(context, adapter, (&chain, groups.as_any()))?; let mut errors: BTreeSet = BTreeSet::new(); - for diagnostic in completion.getattr("diagnostics")?.try_iter()? { - let diagnostic = diagnostic?; + for diagnostic in &completion.diagnostics { let key = diagnostic.getattr("key")?; errors.insert(format!( "{}: {}", @@ -67,8 +59,7 @@ pub(super) fn validated_fixtures<'py>( diagnostic.getattr("message")?.extract::()? )); } - let expected_types = completion.getattr("expected_types")?; - let inferred_by_fixture = completion.getattr("inferred_by_fixture")?; + let inferred_by_fixture = &completion.inferred_by_fixture; let completed_expected = expected.copy()?; let sql_test = resource_types.getattr("SQL_TEST")?; let type_groups = [ @@ -83,12 +74,12 @@ pub(super) fn validated_fixtures<'py>( let name: String = key_name.extract()?; let location = || fixture_location(test, kind, &name); let inferred = if expected_output { - let model_key = PyTuple::new(py, [&model, &key_name])?; - let relation = context - .getattr("relations")? - .call_method1("get", (&model_key,))?; - let (completed_sql, empty_error) = - complete_empty_fixture_sql(&sql.extract::()?, &relation, adapter)?; + let model_key: FixtureKey = (MODEL_REFERENCE_KIND.to_owned(), name.clone()); + let (completed_sql, empty_error) = complete_empty_fixture_sql( + &sql.extract::()?, + context.relations.get(&model_key), + adapter, + )?; if let Some(error) = empty_error { errors.insert(format!("{}: expected output '{name}' {error}", location()?)); } @@ -99,20 +90,21 @@ pub(super) fn validated_fixtures<'py>( "inference_profile", adapter.call_method0("expression_inference_profile")?, )?; - let inference = context.getattr("fixture_inferences")?.call_method( - "infer", - (), - Some(&kwargs), - )?; + let inference = + context + .fixture_inferences + .bind(py) + .call_method("infer", (), Some(&kwargs))?; if inference.is_none() { None } else { - let types = expected_types - .call_method1("get", (&model_key, PyDict::new(py)))? - .cast_into::()?; + let types: &[(String, String)] = completion + .expected_types + .get(&model_key) + .unwrap_or_default(); completed_expected.set_item( &name, - complete_typed_null_columns(&completed_sql, &inference, &types, adapter)?, + complete_typed_null_columns(&completed_sql, &inference, types, adapter)?, )?; Some(inference.getattr("columns")?) } @@ -124,15 +116,23 @@ pub(super) fn validated_fixtures<'py>( let Some(columns) = inferred else { continue; }; - let type_kind = if expected_output { &model } else { kind }; - let type_key = PyTuple::new(py, [type_kind, &key_name])?; - let types = expected_types.call_method1("get", (type_key, PyDict::new(py)))?; + let type_kind: String = if expected_output { + MODEL_REFERENCE_KIND.to_owned() + } else { + kind.extract()? + }; + let types: &[(String, String)] = completion + .expected_types + .get(&(type_kind, name.clone())) + .unwrap_or_default(); for column in columns.try_iter()? { let column = column?; let column_name = column.getattr("name")?; + let folded_name: String = column_name.call_method0("casefold")?.extract()?; let expected_type: Option = types - .call_method1("get", (column_name.call_method0("casefold")?,))? - .extract()?; + .iter() + .find(|(column, _)| *column == folded_name) + .map(|(_, sql_type)| sql_type.clone()); let inferred_type: Option = column.getattr("type")?.extract()?; if !collection_mismatch(expected_type.as_deref(), inferred_type.as_deref()) { continue; @@ -159,7 +159,7 @@ pub(super) fn validated_fixtures<'py>( .map(|error| format!("SQL test '{test_name}': {error}")) .collect())); } - let fixture_sql = completion.getattr("fixture_sql_by_key")?; + let fixture_sql = &completion.sql_by_key; let completed = |kind: &Bound<'py, PyAny>, fixtures: &Bound<'py, PyDict>| -> PyResult> { let result = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 17e0dd54a1..b067230958 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -4,8 +4,8 @@ mod compiled_tests; pub(crate) mod constants; mod cursor_window; mod fixture_validation; -mod plan_assembly; -mod plan_driver; +pub(crate) mod plan_assembly; +pub(crate) mod plan_driver; mod plan_entries; mod plan_errors; mod plan_fixtures; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index f88373bfe9..dca5cd13fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -3,9 +3,15 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::plan_driver::plan_sql_tests_natively; #[pyfunction(name = "plan_sql_tests")] #[pyo3(signature = (**kwargs))] @@ -29,40 +35,45 @@ pub(crate) fn assemble_sql_test_plans<'py>( let tests: Vec> = args.value("tests")?.extract()?; let project = args.value("project")?; let adapter = args.value("adapter")?; - let mut context = py.None().into_bound(py); - let (planned, diagnostics) = - if args.flag("sql_analysis_enabled")? && args.flag("validate_fixtures")? { - context = args.value("fixture_planning_context")?; - if !context.is_truthy()? { - context = call_with( - ®istry::native(py, NativeAttr::BuildRelationFixtureContext)?, - &[("project", project.clone())], - )?; - } - super::plan_fixtures::validate(&tests, (&project, &adapter), &context)? + let mut context: Option> = None; + let (planned, diagnostics) = if args.flag("sql_analysis_enabled")? + && args.flag("validate_fixtures")? + { + let given = args.value("fixture_planning_context")?; + let shared = if given.is_none() { + Bound::new(py, planning_context(&project, None)?)? } else { - (tests.clone(), Default::default()) + given.cast_into::()? }; + let validated = super::plan_fixtures::validate(&tests, (&project, &adapter), shared.get())?; + context = Some(shared); + validated + } else { + (tests.clone(), Default::default()) + }; let plannable: Vec<_> = planned .into_iter() .enumerate() .filter_map(|(index, test)| (!diagnostics.contains_key(&index)).then_some(test)) .collect(); - let plans = call_with( - ®istry::native(py, NativeAttr::PlanSqlTestsNatively)?, - &[ - ("project", project.clone()), - ("tests", PyTuple::new(py, plannable)?.into_any()), - ("adapter", adapter), - ( - "sql_analysis_enabled", - args.value_or_false("sql_analysis_enabled")?, - ), - ( - "render_sql", - pyo3::types::PyBool::new(py, false).to_owned().into_any(), - ), - ], + let plans = plan_sql_tests_natively( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("tests", PyTuple::new(py, plannable)?.into_any()), + ("adapter", adapter), + ( + "sql_analysis_enabled", + args.value_or_false("sql_analysis_enabled")?, + ), + ( + "render_sql", + pyo3::types::PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?), )?; let plans: Vec> = plans.extract()?; if plans.len() != tests.len() - diagnostics.len() { @@ -74,7 +85,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } let result_class = registry::class(py, RecordId::SqlTestPlanResult)?; let mut results: Vec> = Vec::new(); @@ -96,7 +107,9 @@ pub(crate) fn assemble_sql_test_plans<'py>( }; let mut errors: Vec = plan.getattr("error_messages")?.extract()?; errors.extend(super::plan_entries::expected_diagnostics( - test, &plan, &context, + test, + &plan, + context.as_ref().map(Bound::get), )?); if !errors.is_empty() { results.push(call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index a9095ed179..4e5d1a0190 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::compile_timings::add_compile_timi use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; @@ -58,6 +59,7 @@ pub(crate) fn resolve_sql_test_model_chains<'py>( kwargs, (&["project", "tests"], &[]), )?; + count_chain_resolution(py)?; chains(&args.value("project")?, &args.value("tests")?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index 153ce6dedc..7d60c531ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -3,11 +3,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use crate::bindings::_helpers::analysis::sql_test_comparisons::describe_missing_expected_columns; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_TYPES; use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; +use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -132,29 +134,32 @@ pub(super) fn entry<'py>( pub(super) fn expected_diagnostics( test: &Bound<'_, PyAny>, plan: &Bound<'_, PyAny>, - context: &Bound<'_, PyAny>, + context: Option<&RelationFixturePlanningContext>, ) -> PyResult> { let mut result: Vec = Vec::new(); - if context.is_none() { + let Some(context) = context else { return Ok(result); - } + }; let py = test.py(); let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; for step in plan.getattr("chain")?.try_iter()? { let step = step?; let name = step.getattr("model_name")?; + let model_name: String = name.extract()?; let available = context - .getattr("authoritative_columns")? - .call_method1("get", ((resource_types.getattr("MODEL")?, &name),))?; + .authoritative_columns + .get(&(MODEL_REFERENCE_KIND.to_owned(), model_name)); let expected = step.getattr("expected_columns")?; - if expected.is_none() || available.is_none() { + let Some(available) = available else { + continue; + }; + if expected.is_none() { continue; } - let Some(message) = describe_missing_expected_columns( - &name.extract::()?, - expected.extract()?, - available, - )? + let available: Vec = available.iter().cloned().collect(); + let expected: Vec = expected.extract()?; + let Some(message) = + missing_expected_columns(&name.extract::()?, &expected, &available) else { continue; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs index 78a9c510a2..4e8d3f69c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -3,18 +3,18 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; +use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; pub(super) fn validated_test<'py>( test: &Bound<'py, PyAny>, chain: &Bound<'py, PyAny>, inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, + context: &RelationFixturePlanningContext, ) -> PyResult, Vec>> { let py = test.py(); let payload = test.getattr("payload")?; @@ -78,9 +78,9 @@ pub(super) fn validated_test<'py>( pub(super) fn validate<'py>( tests: &[Bound<'py, PyAny>], inputs: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, + context: &RelationFixturePlanningContext, ) -> PyResult> { - let py = context.py(); + let py = inputs.0.py(); let mut model_tests: Vec> = Vec::new(); let mut indexes: Vec = Vec::new(); for (index, test) in tests.iter().enumerate() { @@ -92,12 +92,15 @@ pub(super) fn validate<'py>( if indexes.is_empty() { return Ok((tests.to_vec(), Default::default())); } - let chains = call_with( - ®istry::native(py, NativeAttr::ResolveSqlTestModelChains)?, - &[ - ("project", inputs.0.clone()), - ("tests", PyTuple::new(py, model_tests)?.into_any()), - ], + let chains = resolve_sql_test_model_chains( + py, + Some(keywords( + py, + &[ + ("project", inputs.0.clone()), + ("tests", PyTuple::new(py, model_tests)?.into_any()), + ], + )?), )?; let chains: Vec> = chains.extract()?; if chains.len() != indexes.len() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs index 350760a6b0..ed52d26a49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs @@ -1,5 +1,6 @@ //! Adapter rendering and declared cursor-window boundaries for native test planning. +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::sql_test_glue::plan_values::{model_payload, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; @@ -26,7 +27,7 @@ pub(super) fn models<'py>( let result = PyDict::new(py); for (index, model) in project.getattr("models")?.try_iter()?.enumerate() { let model = model?; - let sql = model.getattr("query_sql")?; + let sql = model_reads::query_sql(&model)?; if mentions(&sql.extract::()?) { result.set_item( index, @@ -46,7 +47,7 @@ pub(super) fn overrides<'py>( let models = PyDict::new(py); for model in project.getattr("models")?.try_iter()? { let model = model?; - models.set_item(model.getattr("name")?, model)?; + models.set_item(model_reads::name_object(&model)?, model)?; } let tests: Vec> = tests.extract()?; let mut indexes: Vec = Vec::new(); @@ -116,12 +117,12 @@ pub(super) fn overrides<'py>( } } for model in declared_window_cursor_models(test, chain_models)? { - let name = model.getattr("name")?; + let name = model_reads::name_object(&model)?; if !rendered.contains(&name)? { rendered.set_item( name, render_test_cursor_intrinsics( - model.getattr("query_sql")?, + model_reads::query_sql(&model)?, &model, adapter, Some(test), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index 43b10157f0..e7bf16eaf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::compiler::models::{ SqlTestPlanTest, }; +use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; @@ -99,7 +100,7 @@ fn resolve_compiled_sql_test_chains( .iter() .map(|model| { Ok(SqlTestChainModel { - name: model.getattr("name")?.extract()?, + name: model_reads::name_object(model)?.extract()?, model_dependencies: model_dependencies(model)?, }) }) @@ -165,10 +166,10 @@ fn plan_model( rendered_sql: Option, ) -> PyResult { Ok(SqlTestPlanModel { - name: model.getattr("name")?.extract()?, + name: model_reads::name_object(model)?.extract()?, query_sql: match rendered_sql { Some(sql) => sql, - None => model.getattr("query_sql")?.extract()?, + None => model_reads::query_sql(model)?.extract()?, }, model_dependencies: model_dependencies(model)?, }) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 54f6ce2037..bbe387de6a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -768,42 +768,23 @@ class RelationFixtureCompletion(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> RelationFixtureCompletion: ... -class FixtureColumnMetadata(NativeRecord): - """Known type and nullability for one relation fixture column.""" - - name: str - type: str | None - nullable: bool | None - def __init__(self, name: str, type: str | None, nullable: bool | None) -> None: ... - def __replace__(self, **changes: Any) -> FixtureColumnMetadata: ... - -class FixtureRelationMetadata(NativeRecord): - """Known columns and whether they form an authoritative relation shape.""" - - columns: tuple[FixtureColumnMetadata, ...] - authoritative_names: bool - def __init__( - self, columns: tuple[FixtureColumnMetadata, ...], authoritative_names: bool - ) -> None: ... - def __replace__(self, **changes: Any) -> FixtureRelationMetadata: ... - -class RelationFixturePlanningContext(NativeRecord): +class RelationFixturePlanningContext: """Project-wide fixture metadata shared across test and scenario planning.""" - models_by_name: dict[str, CompiledModelView] - relations: dict[FixtureKey, FixtureRelationMetadata] - authoritative_columns: dict[FixtureKey, frozenset[str]] - expected_types: dict[FixtureKey, dict[str, str]] - fixture_inferences: FixtureColumnInferences def __init__( self, + *, models_by_name: dict[str, CompiledModelView], - relations: dict[FixtureKey, FixtureRelationMetadata], - authoritative_columns: dict[FixtureKey, frozenset[str]], - expected_types: dict[FixtureKey, dict[str, str]], - fixture_inferences: FixtureColumnInferences = ..., + relations: dict[ + tuple[CompiledResourceType, str], + tuple[tuple[tuple[str, str | None, bool | None], ...], bool], + ], ) -> None: ... - def __replace__(self, **changes: Any) -> RelationFixturePlanningContext: ... + def authoritative_names(self, key: tuple[CompiledResourceType, str]) -> bool | None: ... + def expected_types(self, key: tuple[CompiledResourceType, str]) -> dict[str, str] | None: ... + def authoritative_columns( + self, key: tuple[CompiledResourceType, str] + ) -> frozenset[str] | None: ... class SelectorExpansion(NativeRecord): """One selector split into graph expansion flags and core text.""" @@ -4179,58 +4160,13 @@ def lint_interpolation_text_at(body: str, start: int, dialect: str) -> str | Non def scan_fixture_typed_null_candidates( project_dir: Path, selected_paths: frozenset[Path] | None = None ) -> tuple[frozenset[Path], frozenset[str]]: ... -def simplify_fixture_null_projections(*, sql: str, relation: FixtureRelationMetadata) -> str: ... +def simplify_fixture_null_projections(*, sql: str, relation: Any) -> str: ... def is_empty_fixture_query(sql: str) -> bool: ... -def complete_empty_fixture_sql( - *, sql: str, relation: FixtureRelationMetadata | None, adapter: BaseAdapter -) -> tuple[str, str | None]: ... -def complete_typed_null_columns( - *, - sql: str, - inference: FixtureColumnInference, - expected_types: dict[str, str], - adapter: BaseAdapter, -) -> str: ... -def model_fixture_metadata(*, model: CompiledModelView) -> FixtureRelationMetadata: ... -def expected_fixture_types( - *, - base_expected_types: dict[tuple[CompiledResourceType, str], dict[str, str]], - ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModelView], -) -> Mapping[tuple[CompiledResourceType, str], dict[str, str]]: ... def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... -def required_fixture_columns( - *, - model_inputs: tuple[tuple[str, ...], dict[str, CompiledModelView]], - fixture_inputs: tuple[ - frozenset[tuple[CompiledResourceType, str]], - dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], - dict[tuple[CompiledResourceType, str], frozenset[str]], - ], - adapter: BaseAdapter, -) -> tuple[ - dict[tuple[CompiledResourceType, str], dict[str, set[str]]], - frozenset[tuple[CompiledResourceType, str]], -]: ... +def fixture_planning_work_info(*, reset: bool = False) -> tuple[int, int]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... -def authoritative_fixture_columns( - *, relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata] -) -> dict[tuple[CompiledResourceType, str], frozenset[str]]: ... -def base_expected_fixture_types( - *, relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata] -) -> dict[tuple[CompiledResourceType, str], dict[str, str]]: ... -def fixture_keys_requiring_analysis( - *, - inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]], - authoritative_columns: dict[tuple[CompiledResourceType, str], frozenset[str]], -) -> frozenset[tuple[CompiledResourceType, str]]: ... -def unknown_fixture_column_diagnostics( - *, - inferred_by_fixture: dict[tuple[CompiledResourceType, str], tuple[InferredColumn, ...]], - relations: dict[tuple[CompiledResourceType, str], FixtureRelationMetadata], -) -> tuple[RelationFixtureDiagnostic, ...]: ... def build_relation_fixture_completion( *, project: CompiledProjectView, @@ -7940,48 +7876,9 @@ def plan_table_types( def plan_retention( *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, scope: PlannerScope ) -> tuple[RetentionPlanEntry, ...]: ... -def assemble_base_plan_output( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - pruning: PlannerScopePruningResult, - reconciliation: PlannerChangeReconciliation, - entries: PlannerEntryResults, - source_freshness: DirectSourceFreshnessPlanningResult, - planned_sql_tests: PlannedSqlTests | None = None, -) -> PlanOutput: ... def with_column_rename_hints( *, entries: tuple[ModelPlanEntry, ...], hints: tuple[ColumnRenameHint, ...] ) -> tuple[ModelPlanEntry, ...]: ... -def with_storage_policies( - *, - plan_output: PlanOutput, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - scopes: PlannerScopeResolution, - policies: PlannerPolicies, -) -> PlanOutput: ... -def with_plan_warnings( - *, - runtime: PlannerRuntime, - scopes: PlannerScopeResolution, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - stale_warning_changes: PlannerChangeResults, - pruning: PlannerScopePruningResult, - source_freshness: DirectSourceFreshnessPlanningResult, - plan_output: PlanOutput, - policies: PlannerPolicies, -) -> PlanOutput: ... -def with_plan_metadata( - *, - plan_output: PlanOutput, - pruning: PlannerScopePruningResult, - source_freshness: DirectSourceFreshnessPlanningResult, - policies: PlannerPolicies, -) -> PlanOutput: ... def gather_planner_warehouse_state( *, runtime: PlannerRuntime, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py new file mode 100644 index 0000000000..d70c2abf67 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/helpers.py @@ -0,0 +1,11 @@ +"""Helpers for fixture type-scope integration tests.""" + + +def unrelated_source_entries(*, count: int) -> str: + """Source YAML entries for relations no fixture chain reads.""" + return "".join( + f" - name: unrelated_orders_{index}\n description: Unrelated input.\n" + " expression: SELECT 1 AS order_id\n" + " columns:\n - name: order_id\n type: INTEGER\n" + for index in range(count) + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py index 601f40a9c9..bb8df9c6ab 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -1,6 +1,4 @@ -from collections import ChainMap -from collections.abc import Callable, Mapping -from copy import deepcopy +from collections.abc import Callable from pathlib import Path import pytest @@ -18,14 +16,16 @@ from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures._test_types import ( FixtureTypeScopeTestCase, ) +from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures.helpers import ( + unrelated_source_entries, +) @pytest.mark.parametrize( "test_case", ( FixtureTypeScopeTestCase(description="small metadata map", unrelated_relations=10), - FixtureTypeScopeTestCase(description="large metadata map", unrelated_relations=1000), - FixtureTypeScopeTestCase(description="very large metadata map", unrelated_relations=10000), + FixtureTypeScopeTestCase(description="large metadata map", unrelated_relations=200), ), ids=lambda case: case.description, ) @@ -48,6 +48,7 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i " contract: enforced\n columns:\n" " - name: order_id\n type: INTEGER\n nullable: false\n" " - name: status\n type: VARCHAR\n nullable: true\n" + + unrelated_source_entries(count=test_case.unrelated_relations) ), }, ) @@ -56,14 +57,9 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) context: RelationFixturePlanningContext = build_relation_fixture_context(project=project) - # Model the shared project metadata supplied to every individual SQL-test plan. - for index in range(test_case.unrelated_relations): - context.expected_types[(CompiledResourceType.SOURCE, f"unrelated_orders_{index}")] = { - "order_id": "INTEGER", - "status": "VARCHAR", - } - before: dict[tuple[CompiledResourceType, str], dict[str, str]] = deepcopy( - context.expected_types + unrelated_key: tuple[CompiledResourceType, str] = ( + CompiledResourceType.SOURCE, + "unrelated_orders_0", ) fixture_key: tuple[CompiledResourceType, str] = (CompiledResourceType.SOURCE, "raw_orders") completed: RelationFixtureCompletion = build_relation_fixture_completion( @@ -77,20 +73,12 @@ def test_given_unrelated_project_types_when_completing_fixture_then_eager_work_i ) assert completed.diagnostics == () assert 'CAST(NULL AS VARCHAR) AS "status"' in completed.fixture_sql_by_key[fixture_key] - assert dict(completed.expected_types) == before - assert list(completed.expected_types) == list(before) - assert context.expected_types == before - # Count eagerly materialized relation/type entries, independent of machine timing. - eager: Mapping[tuple[CompiledResourceType, str], dict[str, str]] = getattr( - completed.expected_types, "maps", (completed.expected_types,) - )[0] - assert len(eager) == test_case.expected_eager_relations - assert sum(len(columns) for columns in eager.values()) == test_case.expected_eager_columns - assert isinstance(completed.expected_types, ChainMap) - assert completed.expected_types.maps[-1] is context.expected_types - assert completed.expected_types[fixture_key] is context.expected_types[fixture_key] - model_key: tuple[CompiledResourceType, str] = (CompiledResourceType.MODEL, "orders") - assert completed.expected_types[model_key] is not context.expected_types[model_key] + assert context.expected_types(unrelated_key) == {"order_id": "INTEGER"} + assert len(completed.expected_types) == test_case.expected_eager_relations + assert ( + sum(len(columns) for columns in completed.expected_types.values()) + == test_case.expected_eager_columns + ) if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 6cf18838c5..98f64d28f0 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -226,8 +226,8 @@ def test_given_source_contract_when_completing_fixture_then_respects_authoritati ), planning_context=context, ) - assert context.relations[key].authoritative_names is test_case.expected_authoritative - assert context.expected_types[key] == {"order_id": "INTEGER"} + assert context.authoritative_names(key) is test_case.expected_authoritative + assert context.expected_types(key) == {"order_id": "INTEGER"} assert ( tuple(diagnostic.message for diagnostic in completed.diagnostics) == test_case.expected_diagnostics diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 5da157a363..826bc5e16d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -11,8 +11,6 @@ CompiledObjectKey, CompiledSqlTest, CompileSqlReference, - FixtureColumnMetadata, - FixtureRelationMetadata, PlanWarning, RelationFixtureCompletion, RelationFixturePlanningContext, @@ -55,7 +53,6 @@ ) def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is_built_once( test_case: FixturePlanningWorkTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: compiled_test, project = build_test_and_project( PlanTestChainTestCase( @@ -94,11 +91,8 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is sql_tests=tests, settings=SettingsConfig(sql_analysis=True), ) - metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) - monkeypatch.setattr(fixture_completion, "build_relation_fixture_context", metadata_builder) fixture_completion.fixture_read_cache_info(reset=True) - topology_builder: Mock = Mock(wraps=sql_test_assembly.resolve_sql_test_model_chains) - monkeypatch.setattr(sql_test_assembly, "resolve_sql_test_model_chains", topology_builder) + fixture_completion.fixture_planning_work_info(reset=True) entries: list[SqlTestPlanEntry] warnings: list[PlanWarning] @@ -111,9 +105,12 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is assert len(entries) == test_case.test_count assert warnings == [] - assert metadata_builder.call_count == test_case.expected_metadata_builds + metadata_builds: int + topology_builds: int + metadata_builds, topology_builds = fixture_completion.fixture_planning_work_info() + assert metadata_builds == test_case.expected_metadata_builds assert fixture_completion.fixture_read_cache_info()[0] == test_case.expected_analysis_calls - assert topology_builder.call_count == test_case.expected_topology_builds + assert topology_builds == test_case.expected_topology_builds @pytest.mark.parametrize( @@ -216,15 +213,8 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ context: RelationFixturePlanningContext = RelationFixturePlanningContext( models_by_name={model.name: model}, relations={ - source_key: FixtureRelationMetadata( - columns=(FixtureColumnMetadata(name="order_id", type="INTEGER", nullable=False),), - authoritative_names=True, - ) - }, - authoritative_columns={source_key: frozenset(("order_id",))}, - expected_types={ - source_key: {"order_id": "INTEGER"}, - (CompiledResourceType.MODEL, model.name): {}, + source_key: ((("order_id", "INTEGER", False),), True), + (CompiledResourceType.MODEL, model.name): ((), False), }, ) fixture_completion.fixture_read_cache_info(reset=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index fc73db0258..463fa7cd55 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -39,14 +39,7 @@ def slow_plan_selected_sql_tests(**arguments): _FAILED_BUILD: str = dedent( """ from sqlbuild.cli.entry.main.entry import main - from sqlbuild._native import PlannerInputError - - def failing_reconciliation(**arguments): - raise PlannerInputError("forced planning failure") - - - native.reconcile_execution_changes = failing_reconciliation sys.exit(main(["--project-dir", sys.argv[1], "--no-color", "build"])) """ ) @@ -73,7 +66,7 @@ def failing_reconciliation(**arguments): "plan fails while SQL tests are still planning", _SLOW_TEST_PLANNING + _FAILED_BUILD, 1, - "error[S000]: forced planning failure", + "error[S405]: cannot build selected scope", ), PlanningExitTestCase( "SQL test planning never joined", @@ -95,8 +88,15 @@ def test_given_sql_test_planning_running_when_interpreter_finalizes_then_process '[connection]\ndatabase = "orders_project.duckdb"\n', encoding="utf-8", ) + (project_dir / "sources").mkdir() + (project_dir / "sources" / "raw.yml").write_text( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: raw\n table: orders\n", + encoding="utf-8", + ) (project_dir / "models" / "orders.sql").write_text( - "MODEL (description 'Test model orders.', materialized table);\n\nSELECT 1 AS order_id\n", + "MODEL (description 'Test model orders.', materialized table);\n\n" + 'SELECT * FROM __source("raw_orders")\n', encoding="utf-8", ) script: Path = tmp_path / "planning_exit.py"