From 0afceb217769e735902060ac348b7b3a0c96abd2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:49:45 +0100 Subject: [PATCH 01/33] refactor: host plan entry cursor windows and model entries natively Port plan-entry cursor bound helpers, per-model plan entry construction, relation context assembly and the planner resolution facades to native code, deleting the Python plan_entry module and resolve/cursor-window wrappers. --- .../src/planner/main/cursor_end_clamp.rs | 25 + .../sqlbuild-cursor/src/planner/main/mod.rs | 2 + .../src/planner/main/rolling_window.rs | 44 + crates/sqlbuild-cursor/src/planner/models.rs | 11 + crates/sqlbuild-cursor/src/planner/tests.rs | 4 + .../src/planner/tests/helpers.rs | 30 +- .../planner/tests/test_cursor_end_clamp.rs | 43 + .../src/planner/tests/test_rolling_window.rs | 40 + .../src/planner/tests/test_types.rs | 17 + .../_helpers/build_command/execution.rs | 5 +- .../bindings/_helpers/load/load_command.rs | 5 +- .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/planner_entries/mod.rs | 6 + .../_helpers/planner_entries/model_entries.rs | 705 +++++++++++ .../_helpers/planner_entries/registration.rs | 8 + .../_helpers/planner_entries/relations.rs | 245 ++++ .../_helpers/planner_entries/values.rs | 80 ++ .../_helpers/planner_phases/constants.rs | 1 - .../planner_phases/execution_changes.rs | 6 +- .../planner_phases/warehouse_state.rs | 4 +- .../planner_registration/registration.rs | 3 +- .../planner_resolution/cursor_model.rs | 23 +- .../planner_resolution/cursor_overrides.rs | 9 +- .../planner_resolution/cursor_policies.rs | 106 ++ .../planner_resolution/cursor_ranges.rs | 712 +++++++++++ .../planner_resolution/cursor_warnings.rs | 115 ++ .../_helpers/planner_resolution/mod.rs | 3 + .../planner_resolution/orchestration.rs | 70 +- .../planner_resolution/registration.rs | 3 + .../_helpers/planner_scenario/entries.rs | 6 +- .../_helpers/planner_scenario/execution.rs | 6 +- .../_helpers/planner_warehouse/constants.rs | 19 +- python-boundary.toml | 2 +- scripts/fensu_policy/constants.py | 1 - src/sqlbuild/_native.pyi | 87 +- .../_helpers/command_planning/static.py | 2 +- .../planner/_helpers/output/plan_entry.py | 1101 ----------------- .../planner/_helpers/output/plan_output.py | 8 +- .../planner/_helpers/resolve/__init__.py | 0 .../planner/_helpers/resolve/config.py | 19 - .../planner/_helpers/resolve/cursor.py | 38 - .../_helpers/resolve/cursor_intrinsics.py | 25 - .../_helpers/resolve/cursor_policies.py | 46 - .../_helpers/resolve/future_cursor_safety.py | 3 - .../planner/_helpers/resolve/maximum_start.py | 4 - .../compiler/planner/_helpers/resolve/refs.py | 59 - .../planner/_helpers/resolve/resolve.py | 5 - .../planner/_helpers/resolve/sources.py | 4 - .../compiler/planner/_helpers/reuse/clone.py | 14 +- .../planner/main/cursor_window/__init__.py | 0 .../cursor_window/_typed_cursor_overrides.py | 21 - .../cursor_window_override_flags.py | 13 - .../empty_cursor_input_decision.py | 25 - .../main/cursor_window/empty_cursor_window.py | 14 - .../empty_input_rebuild_refusal.py | 17 - .../main/execution/cursor_replay_policy.py | 32 - .../planner/main/execution/execution.py | 4 +- .../main/execution/future_cursor_safety.py | 37 - .../main/execution/future_cursor_warning.py | 20 - .../main/execution/maximum_start_safety.py | 41 - .../main/execution/maximum_start_warning.py | 30 - .../_helpers/materializations/incremental.py | 3 +- .../_helpers/materializations/microbatch.py | 6 +- .../_helpers/materializations/table_cursor.py | 8 +- .../run/_helpers/validation/cursor_bounds.py | 24 +- .../planner/_helpers/resolve/helpers.py | 8 +- .../resolve/test_adapter_overrides.py | 2 +- .../test_microbatch_reconciliation.py | 2 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../_helpers/test_microbatch_redesign.py | 8 +- .../_helpers/test_parameterized_sql_tests.py | 3 +- .../planner/_helpers/resolve/test_cursor.py | 5 +- .../resolve/test_cursor_intrinsics.py | 2 +- .../resolve/test_cursor_override_types.py | 2 +- .../planner/_helpers/resolve/test_hooks.py | 2 +- .../planner/_helpers/resolve/test_refs.py | 17 +- .../planner/_helpers/resolve/test_sources.py | 4 +- .../test_cursor_override_resolution.py | 2 +- .../planner/_helpers/test_cursor_start.py | 2 +- .../test_plan_entry_cursor_overrides.py | 100 +- .../planner/_helpers/test_source_columns.py | 2 +- .../planner/main/test_future_cursor_safety.py | 4 +- .../planner/main/test_maximum_start_safety.py | 4 +- .../run/_helpers/test_cursor_bounds.py | 2 +- 84 files changed, 2482 insertions(+), 1761 deletions(-) create mode 100644 crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs create mode 100644 crates/sqlbuild-cursor/src/planner/main/rolling_window.rs create mode 100644 crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs create mode 100644 crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/config.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/refs.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/resolve/sources.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py delete mode 100644 src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py diff --git a/crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs b/crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs new file mode 100644 index 0000000000..11f8ff8396 --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/cursor_end_clamp.rs @@ -0,0 +1,25 @@ +//! Clamping a concrete cursor window to an exclusive configured `cursor_end`. + +use std::cmp::Ordering; + +use crate::algebra::errors::CursorError; +use crate::algebra::main::compare::compare; +use crate::algebra::models::Scalar; +use crate::algebra::types::ZoneRules; +use crate::planner::models::CursorEndClamp; + +/// How a `[start, end)` window changes so it never reaches past `cursor_end`. +pub fn cursor_end_clamp( + window: (&Scalar, &Scalar), + cursor_end: &Scalar, + rules: &impl ZoneRules, +) -> Result { + let (start, end) = window; + if compare(start, cursor_end, rules)? != Ordering::Less { + return Ok(CursorEndClamp::Empty); + } + if compare(end, cursor_end, rules)? == Ordering::Greater { + return Ok(CursorEndClamp::Capped); + } + Ok(CursorEndClamp::Unchanged) +} diff --git a/crates/sqlbuild-cursor/src/planner/main/mod.rs b/crates/sqlbuild-cursor/src/planner/main/mod.rs index 9c989bded1..ca5f77b454 100644 --- a/crates/sqlbuild-cursor/src/planner/main/mod.rs +++ b/crates/sqlbuild-cursor/src/planner/main/mod.rs @@ -1,8 +1,10 @@ //! Planner cursor algebra entry points. +pub mod cursor_end_clamp; pub mod eligible_start; pub mod empty_inputs; pub mod override_kind; pub mod raw_bounds; pub mod replay_start; +pub mod rolling_window; pub mod safety_horizon; diff --git a/crates/sqlbuild-cursor/src/planner/main/rolling_window.rs b/crates/sqlbuild-cursor/src/planner/main/rolling_window.rs new file mode 100644 index 0000000000..c4501e570c --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/main/rolling_window.rs @@ -0,0 +1,44 @@ +//! Invocation-anchored rolling microbatch window. + +use crate::algebra::errors::CursorError; +use crate::algebra::main::floor_to_grain::floor_to_grain; +use crate::algebra::main::shift_by_duration::shift_by_duration; +use crate::algebra::models::{DurationSpan, Grain, Moment, Scalar, ScalarValue}; + +/// The `[clock floored to grain - span, clock floored to grain)` window of a UTC clock. +pub fn rolling_window( + clock: &Scalar, + span: &DurationSpan, + grain: Grain, +) -> Result<(Scalar, Scalar), CursorError> { + let anchor: Scalar = floor_to_grain(clock, Some(grain)); + let ScalarValue::Timestamp(anchored) = &anchor.value else { + return Err(CursorError::Type( + "invocation clock must be a timestamp".into(), + )); + }; + let Moment::Timestamp { local, fold } = shift_by_duration( + Moment::Timestamp { + local: anchored.local, + fold: anchored.fold, + }, + span, + -1, + )? + else { + return Err(CursorError::Type( + "invocation clock must be a timestamp".into(), + )); + }; + let mut start = anchored.clone(); + start.local = local; + start.fold = fold; + start.source_text = None; + Ok(( + Scalar { + value: ScalarValue::Timestamp(start), + handle: None, + }, + anchor, + )) +} diff --git a/crates/sqlbuild-cursor/src/planner/models.rs b/crates/sqlbuild-cursor/src/planner/models.rs index 0c51018bcc..570af6b417 100644 --- a/crates/sqlbuild-cursor/src/planner/models.rs +++ b/crates/sqlbuild-cursor/src/planner/models.rs @@ -31,3 +31,14 @@ pub struct ReplayPolicy { pub has_backfill: bool, pub has_start_override: bool, } + +/// The change a configured exclusive `cursor_end` makes to a concrete window. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CursorEndClamp { + /// The window already ends at or before `cursor_end`. + Unchanged, + /// The window starts at or after `cursor_end`, so it collapses to `[cursor_end, cursor_end)`. + Empty, + /// The window ends after `cursor_end`, so its end becomes `cursor_end`. + Capped, +} diff --git a/crates/sqlbuild-cursor/src/planner/tests.rs b/crates/sqlbuild-cursor/src/planner/tests.rs index d695d13453..ef9c3c585e 100644 --- a/crates/sqlbuild-cursor/src/planner/tests.rs +++ b/crates/sqlbuild-cursor/src/planner/tests.rs @@ -2,6 +2,10 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_cursor_end_clamp.rs"] +mod test_cursor_end_clamp; +#[path = "tests/test_rolling_window.rs"] +mod test_rolling_window; #[path = "tests/test_safety_horizon.rs"] mod test_safety_horizon; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-cursor/src/planner/tests/helpers.rs b/crates/sqlbuild-cursor/src/planner/tests/helpers.rs index b9b6ef1bdd..53ae3202c6 100644 --- a/crates/sqlbuild-cursor/src/planner/tests/helpers.rs +++ b/crates/sqlbuild-cursor/src/planner/tests/helpers.rs @@ -1,6 +1,9 @@ +use crate::algebra::errors::CursorError; use crate::algebra::models::{ - CivilDate, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, + CivilDate, DurationSpan, LocalDateTime, Scalar, ScalarValue, Timestamp, ZoneOffset, }; +use crate::algebra::types::ZoneRules; +use crate::planner::tests::test_types::UtcZones; pub(super) fn timestamp(month: u32, day: u32, hour: u32, minute: u32) -> Scalar { Scalar { @@ -24,3 +27,28 @@ pub(super) fn timestamp(month: u32, day: u32, hour: u32, minute: u32) -> Scalar }), } } + +impl ZoneRules for UtcZones { + fn to_utc( + &self, + _: usize, + local: &LocalDateTime, + _: bool, + ) -> Result { + Ok(*local) + } + + fn utc_offset(&self, _: usize, _: &LocalDateTime, _: bool) -> Result, CursorError> { + Ok(Some(0)) + } +} + +pub(super) fn days(days: i64) -> DurationSpan { + DurationSpan { + months: 0, + days, + hours: 0, + minutes: 0, + seconds: 0, + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs b/crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs new file mode 100644 index 0000000000..41476933ca --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/test_cursor_end_clamp.rs @@ -0,0 +1,43 @@ +use crate::planner::main::cursor_end_clamp::cursor_end_clamp; +use crate::planner::models::CursorEndClamp; +use crate::planner::tests::helpers::timestamp; +use crate::planner::tests::test_types::{CursorEndClampTestCase, UtcZones}; + +#[test] +fn given_window_and_cursor_end_when_clamping_then_window_never_passes_end() { + let test_cases = [ + CursorEndClampTestCase { + description: "window before the end is unchanged", + start_day: 1, + end_day: 5, + expected_clamp: CursorEndClamp::Unchanged, + }, + CursorEndClampTestCase { + description: "window crossing the end is capped", + start_day: 5, + end_day: 15, + expected_clamp: CursorEndClamp::Capped, + }, + CursorEndClampTestCase { + description: "window starting at the end is empty", + start_day: 10, + end_day: 15, + expected_clamp: CursorEndClamp::Empty, + }, + ]; + for test_case in test_cases { + assert_eq!( + cursor_end_clamp( + ( + ×tamp(1, test_case.start_day, 0, 0), + ×tamp(1, test_case.end_day, 0, 0) + ), + ×tamp(1, 10, 0, 0), + &UtcZones, + ), + Ok(test_case.expected_clamp), + "{}", + test_case.description, + ); + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs b/crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs new file mode 100644 index 0000000000..a544fc896c --- /dev/null +++ b/crates/sqlbuild-cursor/src/planner/tests/test_rolling_window.rs @@ -0,0 +1,40 @@ +use crate::algebra::models::Grain; +use crate::planner::main::rolling_window::rolling_window; +use crate::planner::tests::helpers::{days, timestamp}; +use crate::planner::tests::test_types::RollingWindowTestCase; + +#[test] +fn given_invocation_clock_when_rolling_window_then_window_ends_at_floored_clock() { + let test_cases = [ + RollingWindowTestCase { + description: "day grain floors to midnight and steps back", + days: 2, + grain: Grain::Day, + expected_window: (timestamp(1, 29, 0, 0), timestamp(1, 31, 0, 0)), + }, + RollingWindowTestCase { + description: "hour grain keeps the hour", + days: 1, + grain: Grain::Hour, + expected_window: (timestamp(1, 30, 12, 0), timestamp(1, 31, 12, 0)), + }, + RollingWindowTestCase { + description: "zero span over minute grain is empty", + days: 0, + grain: Grain::Minute, + expected_window: (timestamp(1, 31, 12, 30), timestamp(1, 31, 12, 30)), + }, + ]; + for test_case in test_cases { + assert_eq!( + rolling_window( + ×tamp(1, 31, 12, 30), + &days(test_case.days), + test_case.grain + ), + Ok(test_case.expected_window), + "{}", + test_case.description, + ); + } +} diff --git a/crates/sqlbuild-cursor/src/planner/tests/test_types.rs b/crates/sqlbuild-cursor/src/planner/tests/test_types.rs index f8d56d203a..1e3cccc00a 100644 --- a/crates/sqlbuild-cursor/src/planner/tests/test_types.rs +++ b/crates/sqlbuild-cursor/src/planner/tests/test_types.rs @@ -1,4 +1,5 @@ use crate::algebra::models::{Grain, Scalar}; +use crate::planner::models::CursorEndClamp; pub(super) struct HorizonTestCase { pub(super) description: &'static str, @@ -7,3 +8,19 @@ pub(super) struct HorizonTestCase { pub(super) date_only: bool, pub(super) expected_horizon: Scalar, } + +pub(super) struct UtcZones; + +pub(super) struct RollingWindowTestCase { + pub(super) description: &'static str, + pub(super) days: i64, + pub(super) grain: Grain, + pub(super) expected_window: (Scalar, Scalar), +} + +pub(super) struct CursorEndClampTestCase { + pub(super) description: &'static str, + pub(super) start_day: u32, + pub(super) end_day: u32, + pub(super) expected_clamp: CursorEndClamp, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 865ea135f9..9ce0cd9904 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -22,8 +22,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CURSOR_OVERRIDES_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; +const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; @@ -179,7 +178,7 @@ fn cursors<'py>( let mut overrides: Bound<'py, PyAny> = plan.getattr("cursor_overrides")?; if !overrides.is_truthy()? { overrides = call_with( - &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &imported(py, CURSOR_OVERRIDES_MODULE, "type_cursor_overrides")?, &[( "cursor_overrides", context.request.getattr("cursor_overrides")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index b477e79fbd..1565112d7f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -35,8 +35,7 @@ const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const CURSOR_OVERRIDES_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; +const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; @@ -324,7 +323,7 @@ fn prepare<'py>( requested.extract()? }; let typed: Bound<'py, PyAny> = call_with( - &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &imported(py, CURSOR_OVERRIDES_MODULE, "type_cursor_overrides")?, &[("cursor_overrides", request.getattr("cursor_overrides")?)], )?; let cursor_overrides: Bound<'py, PyAny> = if typed.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index e7a77b676b..7499911e21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -46,6 +46,7 @@ pub(crate) mod model_config; pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; +pub(crate) mod planner_entries; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; pub(crate) mod planner_phases; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs new file mode 100644 index 0000000000..9950582c5a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -0,0 +1,6 @@ +//! Plan entry and plan output assembly over native planner resolution. + +mod model_entries; +pub(crate) mod registration; +mod relations; +mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs new file mode 100644 index 0000000000..82e93fa088 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -0,0 +1,705 @@ +//! Per-model plan entries from resolved change results and warehouse snapshot facts. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::values::{ + COMPILE_EXCEPTIONS, COMPILE_TYPES, PLANNER_CONSTANTS, PLANNER_EXCEPTIONS, PLANNER_TYPES, + boolean, error_of, native, none, planner_model, planner_type, replaced, string, +}; +use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ + CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, + model_cursor_overrides, plan_cursor_bounds, +}; +use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_safety_warnings; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; + +/// One model's planning request after change detection. +struct ModelPlanRequest<'py> { + model: Bound<'py, PyAny>, + snapshot: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, + sql_analysis_enabled: Bound<'py, PyAny>, + full_refresh: bool, + overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + change_result: Bound<'py, PyAny>, + external_resolver: Bound<'py, PyAny>, + protect_full_refresh: bool, +} + +/// Build a model plan entry and its warnings from a resolved change result. +fn plan_model_from_change<'py>( + request: &ModelPlanRequest<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let model = &request.model; + let context = &request.context; + let py = model.py(); + let full_refresh = boolean(py, request.full_refresh)?; + let backfill = if request.full_refresh { + planner_model( + py, + "BackfillResult", + &[("action", planner_type(py, "BackfillAction", "FULL")?)], + )? + } else { + request.change_result.getattr("backfill")? + }; + let cursor_column = config_str(model, "cursor")?; + native( + py, + "validate_source_cursor_input_columns", + &[ + ("model", model.clone()), + ("cursor_column", cursor_column.clone()), + ("models_by_name", context.getattr("models_by_name")?), + ("functions_by_name", context.getattr("functions_by_name")?), + ("source_map", context.getattr("source_map")?), + ( + "source_warehouse_columns", + context.getattr("source_warehouse_columns")?, + ), + ], + )?; + let resolved = native( + py, + "resolve_model_plan_action", + &[ + ("model", model.clone()), + ("change_result", request.change_result.clone()), + ("full_refresh", full_refresh.clone()), + ], + )?; + let (action, reason) = (resolved.get_item(0)?, resolved.get_item(1)?); + let materialization_type = native(py, "get_materialization_type", &[("model", model.clone())])?; + let replaces_relation = native( + py, + "replaces_incremental_relation", + &[ + ("materialization_type", materialization_type.clone()), + ("action", action.clone()), + ], + )? + .is_truthy()?; + let full_rebuild_cause = if replaces_relation { + native( + py, + "describe_full_rebuild_cause", + &[ + ("change_result", request.change_result.clone()), + ("full_refresh", full_refresh.clone()), + ], + )? + } else { + none(py) + }; + if !full_rebuild_cause.is_none() && request.protect_full_refresh && !request.full_refresh { + native( + py, + "check_full_rebuild_allowed", + &[ + ("model", model.clone()), + ("change_result", request.change_result.clone()), + ("full_rebuild_cause", full_rebuild_cause.clone()), + ], + )?; + } + let pair = planner_model( + py, + "CursorOverridePair", + &[ + ("start_cursor_override", request.overrides.0.clone()), + ("end_cursor_override", request.overrides.1.clone()), + ], + )?; + let resolved_sql = native( + py, + "resolve_model_sql", + &[ + ("adapter", request.adapter.clone()), + ("model", model.clone()), + ("snapshot", request.snapshot.clone()), + ("context", context.clone()), + ("backfill", backfill.clone()), + ("full_refresh", full_refresh.clone()), + ("cursor_overrides", pair), + ( + "external_sql_reference_resolver", + request.external_resolver.clone(), + ), + ("replaces_relation", boolean(py, replaces_relation)?), + ("full_rebuild_cause", full_rebuild_cause), + ], + )?; + let hooks = native( + py, + "resolve_model_hook_entries", + &[ + ("model", model.clone()), + ("adapter", request.adapter.clone()), + ("context", context.clone()), + ( + "external_sql_reference_resolver", + request.external_resolver.clone(), + ), + ], + )?; + let (future_config, start_config) = cursor_safety_configs(model, context)?; + native( + py, + "validate_capped_watermark_inputs", + &[ + ("model", model.clone()), + ("models_by_name", context.getattr("models_by_name")?), + ("functions_by_name", context.getattr("functions_by_name")?), + ], + )?; + let cursor_input_relations = native( + py, + "build_cursor_input_relations", + &[ + ("model", model.clone()), + ("adapter", request.adapter.clone()), + ("model_locations", context.getattr("model_locations")?), + ("models_by_name", context.getattr("models_by_name")?), + ("functions_by_name", context.getattr("functions_by_name")?), + ("seed_locations", context.getattr("seed_locations")?), + ("source_map", context.getattr("source_map")?), + ("cursor_column", cursor_column.clone()), + ( + "runtime_cursor_producer_names", + context.getattr("runtime_cursor_producer_names")?, + ), + ], + )?; + let mut any_runtime_owned = false; + for relation in cursor_input_relations.try_iter()? { + if relation?.getattr("is_runtime_owned")?.is_truthy()? { + any_runtime_owned = true; + break; + } + } + let complete_override = !request.overrides.0.is_none() && !request.overrides.1.is_none(); + let runtime_owned = any_runtime_owned && !complete_override; + let window = CursorWindowInputs { + snapshot: request.snapshot.clone(), + backfill: backfill.clone(), + overrides: request.overrides.clone(), + runtime_owned, + policies: (future_config.clone(), start_config.clone()), + invocation_time: context.getattr("invocation_time")?, + full_refresh: request.full_refresh, + replaces_relation, + cursor_input_relations: cursor_input_relations.clone(), + }; + let cursor_bounds = plan_cursor_bounds(model, &window)?; + let (empty_inputs, waiting_on_empty_inputs) = empty_cursor_inputs(model, &window)?; + let range = microbatch_range(model, &window)?; + let assembled = native( + py, + "assemble_model_plan_entry", + &[ + ("model", model.clone()), + ("adapter", request.adapter.clone()), + ("snapshot", request.snapshot.clone()), + ("change_result", request.change_result.clone()), + ("action", action), + ("reason", reason), + ("materialization_type", materialization_type), + ("resolved_sql", resolved_sql), + ("sql_analysis_enabled", request.sql_analysis_enabled.clone()), + ("pre_hooks", hooks.get_item(0)?), + ("post_hooks", hooks.get_item(1)?), + ("cursor_column", cursor_column), + ("cursor_input_relations", cursor_input_relations), + ("runtime_owned_cursor_bounds", boolean(py, runtime_owned)?), + ("cursor_bounds", cursor_bounds.clone()), + ("microbatch_range", range.clone()), + ("empty_cursor_inputs", empty_inputs), + ( + "waiting_on_empty_inputs", + boolean(py, waiting_on_empty_inputs)?, + ), + ("backfill", backfill), + ("start_cursor_override", request.overrides.0.clone()), + ("end_cursor_override", request.overrides.1.clone()), + ("future_cursor_config", future_config), + ("start_cursor_config", start_config), + ("invocation_time", context.getattr("invocation_time")?), + ], + )?; + let bounds = if range.is_truthy()? { + range + } else { + cursor_bounds + }; + let warnings = + with_cursor_safety_warnings(&assembled.get_item(1)?, &model.getattr("name")?, &bounds)?; + Ok((assembled.get_item(0)?, warnings)) +} + +/// The binding-diagnostic failure that stops planning, if any diagnostic is an error. +fn check_binding_diagnostics<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = project.py(); + let sql_test = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SQL_TEST")?; + let mut errors: Vec> = Vec::new(); + for diagnostic in project.getattr("diagnostics")?.try_iter()? { + let diagnostic = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? + && diagnostic.getattr("resource_type")?.ne(&sql_test)? + { + errors.push(diagnostic); + } + } + let bound = native( + py, + "source_binding_diagnostics", + &[ + ("project", project.clone()), + ("columns", relations.getattr("source_warehouse_columns")?), + ( + "profile", + adapter.call_method0("expression_inference_profile")?, + ), + ("selected_keys", selected_keys.clone()), + ], + )?; + for diagnostic in bound.try_iter()? { + let diagnostic = diagnostic?; + if diagnostic.getattr("is_error")?.is_truthy()? { + errors.push(diagnostic); + } + } + let Some(first) = errors.first() else { + return Ok(()); + }; + let mut lines: Vec = Vec::new(); + for diagnostic in &errors { + let position = |field: &str| -> PyResult { + let value = diagnostic.getattr(field)?; + Ok(if value.is_truthy()? { + value.str()?.to_string() + } else { + "1".to_owned() + }) + }; + lines.push(format!( + "error[{}]: {}\n --> {}:{}:{}", + diagnostic.getattr("code")?.str()?, + diagnostic.getattr("message")?.str()?, + diagnostic.getattr("path")?.str()?, + position("line")?, + position("column")?, + )); + } + Err(error_of( + py, + (PLANNER_EXCEPTIONS, "PlannerInputError"), + &lines.join("\n"), + &[("code", first.getattr("code")?)], + )) +} + +fn names_of(entries: &[Bound<'_, PyAny>]) -> PyResult { + Ok(entries + .iter() + .map(|entry| Ok(entry.getattr("name")?.str()?.to_string())) + .collect::>>()? + .join(", ")) +} + +/// Concurrent microbatch configuration checks, and the warning concurrency complexity earns. +fn check_microbatch_concurrency<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + entries: &[Bound<'py, PyAny>], +) -> PyResult>> { + let py = project.py(); + let skip = planner_type(py, "PlanAction", "SKIP")?; + let create = planner_type(py, "PlanAction", "CREATE_TABLE")?; + let mut configured: Vec> = Vec::new(); + for entry in entries { + if entry.getattr("batch_concurrency")?.gt(1)? && entry.getattr("action")?.ne(&skip)? { + configured.push(entry.clone()); + } + } + let settings = project.getattr("settings")?; + if !configured.is_empty() && !settings.getattr("microbatch_concurrency")?.is_truthy()? { + let guidance = native(py, "microbatch_guidance", &[])?; + let prefix: String = + imported(py, PLANNER_CONSTANTS, "MICROBATCH_CONCURRENCY_MESSAGE")?.extract()?; + return Err(error_of( + py, + (COMPILE_EXCEPTIONS, "CompileInputError"), + &format!( + "{prefix}{}; {}", + names_of(&configured)?, + guidance.get_item(0)?.str()? + ), + &[("help", guidance.get_item(1)?)], + )); + } + if !configured.is_empty() + && !adapter + .call_method0("supports_concurrent_microbatch_dml")? + .is_truthy()? + { + return Err(error_of( + py, + (COMPILE_EXCEPTIONS, "CompileInputError"), + &format!( + "adapter '{}' does not support concurrent microbatch delete/insert DML; set \ + batch_concurrency = 1", + adapter.getattr("adapter_name")?.str()? + ), + &[], + )); + } + let mut concurrent: Vec> = Vec::new(); + for entry in configured { + if entry.getattr("action")?.ne(&create)? { + concurrent.push(entry); + } + } + if concurrent.is_empty() || !settings.getattr("concurrency")?.gt(1)? { + return Ok(None); + } + let count = concurrent.len(); + planner_model( + py, + "PlanWarning", + &[ + ("model_name", none(py)), + ("severity", planner_type(py, "WarningSeverity", "WARNING")?), + ("code", string(py, MICROBATCH_COMPLEXITY_CODE)), + ( + "message", + string( + py, + &format!( + "Concurrent microbatch execution is enabled for {count} selected \ + model{}: {}. Concurrent batches may complete out of order and rely \ + on persistent microbatch history for gap recovery and fingerprint \ + provenance. State deletion or corruption can require synthetic \ + reconciliation and weaken version guarantees. Keep \ + batch_concurrency at 1 unless the runtime benefit justifies this \ + complexity; see the microbatch concurrency and recovery \ + documentation.", + if count == 1 { "" } else { "s" }, + names_of(&concurrent)? + ), + ), + ), + ], + ) + .map(Some) +} + +/// Build model plan entries from snapshot and per-model resolved actions. +fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let adapter = args.value("adapter")?; + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let relations = args.value("relations")?; + let models = args.value("resolved_actions")?.getattr("models")?; + let cursor_overrides = args.value("cursor_overrides")?; + let full_refresh = args.value("full_refresh")?; + let inputs = args.value("build_inputs")?; + let inputs = if inputs.is_none() { + planner_model(py, "PlanEntryBuildInputs", &[])? + } else { + inputs + }; + let selected_keys = scope.getattr("selected_keys")?; + check_binding_diagnostics(&project, &adapter, &relations, &selected_keys)?; + let freshness_blocked = inputs.getattr("source_freshness_blocked_model_names")?; + let external_blocked = inputs.getattr("external_blocked_model_names")?; + let start_override = inputs.getattr("start_cursor_override")?; + let end_override = inputs.getattr("end_cursor_override")?; + let producers = native( + py, + "runtime_cursor_producer_names", + &[ + ("scope", scope.clone()), + ("source_map", relations.getattr("source_map")?), + ], + )?; + let functions_by_name = PyDict::new(py); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + functions_by_name.set_item(function.getattr("name")?, function)?; + } + let settings = project.getattr("settings")?; + let models_by_name = scope.getattr("models_by_name")?; + let context = planner_model( + py, + "ModelPlanContext", + &[ + ("model_locations", relations.getattr("model_locations")?), + ("models_by_name", models_by_name.clone()), + ("functions_by_name", functions_by_name.into_any()), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_read_map")?), + ( + "source_warehouse_columns", + relations.getattr("source_warehouse_columns")?, + ), + ( + "star_exclude_keyword", + relations.getattr("star_exclude_keyword")?, + ), + ("runtime_cursor_producer_names", producers), + ( + "future_cursor_config", + inputs.getattr("future_cursor_config")?, + ), + ( + "start_cursor_config", + inputs.getattr("start_cursor_config")?, + ), + ("invocation_time", inputs.getattr("invocation_time")?), + ], + )?; + let skip = planner_type(py, "PlanAction", "SKIP")?; + let microbatch = imported(py, PLANNER_TYPES, "IncrementalMode")?.getattr("MICROBATCH")?; + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + let name = key.getattr("name")?; + if !selected_keys.contains(&key)? || !models.contains(&name)? { + continue; + } + let model = models_by_name.call_method1("get", (&name,))?; + let resolved = models.get_item(&name)?; + if model.is_none() { + continue; + } + let overrides = model_cursor_overrides( + &model, + &cursor_overrides, + (start_override.clone(), end_override.clone()), + )?; + let effective_full_refresh = native( + py, + "resolve_model_full_refresh", + &[ + ("model", model.clone()), + ("cli_full_refresh", full_refresh.clone()), + ], + )? + .is_truthy()?; + let (mut entry, entry_warnings) = plan_model_from_change(&ModelPlanRequest { + model: model.clone(), + snapshot: snapshot.clone(), + adapter: adapter.clone(), + context: context.clone(), + sql_analysis_enabled: settings.getattr("sql_analysis")?, + full_refresh: effective_full_refresh, + overrides, + change_result: resolved.getattr("change")?, + external_resolver: project.getattr("external_sql_reference_resolver")?, + protect_full_refresh: !freshness_blocked.contains(&name)? + && !external_blocked.contains(&name)?, + })?; + let entry_name = entry.getattr("name")?; + if freshness_blocked.contains(&entry_name)? { + entry = replaced( + &entry, + &[ + ("action", skip.clone()), + ( + "reason", + planner_type(py, "PlanReason", "SOURCE_FRESHNESS_ERROR")?, + ), + ], + )?; + } + if external_blocked.contains(&entry_name)? { + entry = replaced( + &entry, + &[ + ("action", skip.clone()), + ( + "reason", + planner_type(py, "PlanReason", "EXTERNAL_UPSTREAM_FAILED")?, + ), + ], + )?; + } + if entry.getattr("incremental_mode")?.eq(µbatch)? { + let policy = entry.getattr("unaccounted_partition_policy")?; + let policy = if policy.is_truthy()? { + policy + } else { + settings.getattr("microbatch_unaccounted_partition_policy")? + }; + entry = replaced(&entry, &[("unaccounted_partition_policy", policy)])?; + if entry.getattr("action")?.ne(&skip)? { + let limited = native( + py, + "apply_configured_microbatch_limits", + &[("entry", entry), ("inputs", inputs.clone())], + )?; + entry = limited.get_item(0)?; + for warning in limited.get_item(1)?.try_iter()? { + warnings.push(warning?); + } + } + } + entries.push(entry); + for warning in entry_warnings.try_iter()? { + warnings.push(warning?); + } + } + if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries)? { + warnings.push(warning); + } + planner_model( + py, + "PlannerModelEntryResults", + &[ + ("entries", PyTuple::new(py, entries)?.into_any()), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + ) +} + +/// Build model plan entries from snapshot and per-model resolved actions. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_plan_entries<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_plan_entries", + kwargs, + ( + &[ + "project", + "adapter", + "scope", + "snapshot", + "relations", + "resolved_actions", + "cursor_overrides", + "full_refresh", + ], + &["build_inputs"], + ), + )?; + plan_entries(&args) +} + +/// Detect changes and build a plan entry and warnings for a single model. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn plan_model<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let args = KeywordArguments::read( + py, + "plan_model", + kwargs, + ( + &[ + "model", + "snapshot", + "adapter", + "context", + "sql_analysis_enabled", + "query_change_tracking", + "full_refresh", + "cursor_overrides", + ], + &["external_sql_reference_resolver"], + ), + )?; + let model = args.value("model")?; + let snapshot = args.value("snapshot")?; + let change_result = native( + py, + "detect_model_changes", + &[ + ("model", model.clone()), + ("snapshot", snapshot.clone()), + ("sql_analysis_enabled", args.value("sql_analysis_enabled")?), + ( + "query_change_tracking", + args.value("query_change_tracking")?, + ), + ("full_refresh", args.value("full_refresh")?), + ], + )?; + let overrides = args.value("cursor_overrides")?; + plan_model_from_change(&ModelPlanRequest { + model, + snapshot, + adapter: args.value("adapter")?, + context: args.value("context")?, + sql_analysis_enabled: args.value("sql_analysis_enabled")?, + full_refresh: args.flag("full_refresh")?, + overrides: ( + overrides.getattr("start_cursor_override")?, + overrides.getattr("end_cursor_override")?, + ), + change_result, + external_resolver: args.value("external_sql_reference_resolver")?, + protect_full_refresh: true, + }) +} + +/// Build a name-to-materialization-type lookup from planned model entries. +#[pyfunction] +#[pyo3(signature = (model_entries, /))] +fn build_model_materializations<'py>( + model_entries: &Bound<'py, PyAny>, +) -> PyResult> { + let lookup = PyDict::new(model_entries.py()); + for entry in model_entries.try_iter()? { + let entry = entry?; + lookup.set_item( + entry.getattr("name")?, + entry.getattr("materialization_type")?, + )?; + } + Ok(lookup) +} + +/// Check if any scope dependency is in the selected keys. +#[pyfunction] +#[pyo3(signature = (*, scope_deps, selected_keys))] +pub(crate) fn scope_overlaps( + scope_deps: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, +) -> PyResult { + for dependency in scope_deps.try_iter()? { + if selected_keys.contains(dependency?)? { + return Ok(true); + } + } + Ok(false) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_plan_entries, module)?)?; + module.add_function(wrap_pyfunction!(plan_model, module)?)?; + module.add_function(wrap_pyfunction!(build_model_materializations, module)?)?; + module.add_function(wrap_pyfunction!(scope_overlaps, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs new file mode 100644 index 0000000000..c1fe546924 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -0,0 +1,8 @@ +//! Register plan entry and plan output assembly entry points. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::model_entries::register(module)?; + super::relations::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs new file mode 100644 index 0000000000..fd974efa8a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -0,0 +1,245 @@ +//! Relation locations and source metadata for plan entry construction. + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +/// Relation keys needed to render selected audits without selecting those relations. +fn selected_audit_relation_keys<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let keys = PySet::empty(py)?; + for audit in project.getattr("audits")?.try_iter()? { + let audit = audit?; + if native( + py, + "audit_is_selected", + &[ + ("audit", audit.clone()), + ("selected_keys", selected_keys.clone()), + ], + )? + .is_truthy()? + { + for dependency in audit.getattr("scope_deps")?.try_iter()? { + keys.add(dependency?)?; + } + } + } + Ok(PyFrozenSet::new(py, keys.iter())?.into_any()) +} + +/// Reuse already-gathered source columns for the read map, or gather them once. +fn source_warehouse_columns<'py>( + (project, adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_read_map: &Bound<'py, PyAny>, + known_source_columns: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = project.py(); + if !known_source_columns.is_none() { + let known = PyDict::new(py); + for name in source_read_map.try_iter()? { + let name = name?; + if known_source_columns.contains(&name)? { + known.set_item(&name, known_source_columns.get_item(&name)?)?; + } + } + let names = PyFrozenSet::new(py, known.keys())?.into_any(); + return Ok((known.into_any(), names)); + } + let entries = PyTuple::new( + py, + source_read_map + .call_method0("values")? + .try_iter()? + .collect::>>()?, + )?; + let gathered = native( + py, + "gather_source_inspection", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("source_entries", entries.into_any()), + ], + )?; + Ok((gathered.get_item(0)?, gathered.get_item(1)?)) +} + +/// Resolve relation locations and source metadata for plan entry construction. +pub(crate) fn planner_relations_context<'py>( + args: &KeywordArguments<'py>, +) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let scope = args.value("scope")?; + let deferral = args.value("deferral")?; + let deferral = if deferral.is_none() { + planner_model(py, "DeferralInputs", &[])? + } else { + deferral + }; + let project_config = args.value("project_config")?; + let local_config = args.value("local_config")?; + let selected_keys = scope.getattr("selected_keys")?; + let mut model_locations = native( + py, + "build_model_locations", + &[("models", project.getattr("models")?)], + )?; + let mut seed_locations = native( + py, + "build_seed_locations", + &[("seeds", project.getattr("seeds")?)], + )?; + let function_locations = native( + py, + "build_function_locations", + &[("functions", project.getattr("functions")?)], + )?; + let deferred_locations = deferral.getattr("deferred_locations")?; + if !deferred_locations.is_none() { + let applied = native( + py, + "apply_deferred_locations", + &[ + ("model_locations", model_locations), + ("seed_locations", seed_locations), + ("deferred_locations", deferred_locations), + ("selected_keys", selected_keys.clone()), + ], + )?; + model_locations = applied.get_item(0)?; + seed_locations = applied.get_item(1)?; + } + let audit_keys = selected_audit_relation_keys(&project, &selected_keys)?; + let relation_keys = args.value("relation_keys")?; + let load_keys = if relation_keys.is_none() { + selected_keys.call_method1("__or__", (&audit_keys,))? + } else { + relation_keys + }; + let source_map = native( + py, + "build_source_load_map", + &[ + ("project", project.clone()), + ("selected_keys", load_keys), + ( + "upstream_deps", + if scope.getattr("all_keys")?.is_truthy()? { + scope.getattr("upstream_deps")? + } else { + none(py) + }, + ), + ], + )?; + let deferral_enabled = deferral.getattr("source_deferral_enabled")?.is_truthy()?; + let read_map = |map: &Bound<'py, PyAny>, names: Option>| { + if !deferral_enabled { + return Ok(map.clone()); + } + let mut pairs = vec![ + ("project", project.clone()), + ("source_map", map.clone()), + ("selected_keys", selected_keys.clone()), + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("defer_sources_to", deferral.getattr("defer_sources_to")?), + ]; + if let Some(names) = names { + pairs.push(("read_source_names", names)); + } + native(py, "build_source_read_map", &pairs) + }; + let source_read_map = read_map(&source_map, None)?; + let python_source_map = native( + py, + "with_declared_source_reads", + &[ + ("project", project.clone()), + ("source_map", source_map.clone()), + ("source_names", scope.getattr("python_read_source_names")?), + ], + )?; + let python_names = PyFrozenSet::new( + py, + python_source_map + .try_iter()? + .collect::>>()?, + )? + .into_any(); + let python_source_read_map = read_map(&python_source_map, Some(python_names))?; + let (columns, listed_source_names) = source_warehouse_columns( + (&project, &adapter, &connection), + &source_read_map, + &args.value("known_source_columns")?, + )?; + let semantic_columns = native( + py, + "get_semantic_source_columns", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection), + ("source_read_map", source_read_map.clone()), + ("selected_keys", selected_keys), + ("columns", columns), + ], + )?; + planner_model( + py, + "PlannerRelationsContext", + &[ + ("model_locations", model_locations), + ("seed_locations", seed_locations), + ("function_locations", function_locations), + ("source_map", source_map), + ("source_read_map", source_read_map), + ("python_source_read_map", python_source_read_map), + ("source_warehouse_columns", semantic_columns), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ("listed_source_names", listed_source_names), + ], + ) +} + +/// Resolve relation locations and source metadata for plan entry construction. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_planner_relations_context<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_planner_relations_context", + kwargs, + ( + &["project", "adapter", "connection", "scope"], + &[ + "deferral", + "project_config", + "local_config", + "known_source_columns", + "relation_keys", + ], + ), + )?; + planner_relations_context(&args) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_planner_relations_context, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs new file mode 100644 index 0000000000..f7e65b39a2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -0,0 +1,80 @@ +//! Python boundary values shared by plan entry and plan output assembly. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; + +pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; +pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(super) const PLANNER_CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; +pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; + +/// `sqlbuild._native.name(**pairs)`. +pub(super) fn native<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, NATIVE_MODULE, name)?, pairs) +} + +/// `sqlbuild.compiler.planner.models.name(**pairs)`. +pub(super) fn planner_model<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, PLANNER_MODELS, name)?, pairs) +} + +/// One member of a planner enumeration. +pub(super) fn planner_type<'py>( + py: Python<'py>, + class: &str, + member: &str, +) -> PyResult> { + imported(py, PLANNER_TYPES, class)?.getattr(member) +} + +/// A Python string. +pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { + PyString::new(py, text).into_any() +} + +/// A Python bool. +pub(super) fn boolean(py: Python<'_>, value: bool) -> PyResult> { + Ok(value.into_pyobject(py)?.to_owned().into_any()) +} + +/// `None`. +pub(super) fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +/// `dataclasses.replace(object, **fields)`. +pub(super) fn replaced<'py>( + object: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + imported(object.py(), "dataclasses", "replace")? + .call((object,), Some(&keywords(object.py(), fields)?)) +} + +/// `class(message, **fields)` from `module`, as a raised error. +pub(super) fn error_of( + py: Python<'_>, + (module, class): (&str, &str), + message: &str, + fields: &[(&str, Bound<'_, PyAny>)], +) -> PyErr { + match imported(py, module, class) + .and_then(|class| class.call((message,), Some(&keywords(py, fields)?))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index 08dd8db18a..c8ecc42e35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -7,7 +7,6 @@ pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exception pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; pub(crate) const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; -pub(crate) const PLAN_ENTRY_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_entry"; pub(crate) const PLAN_OUTPUT_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_output"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 4106139b51..e2f7805a21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_phases::constants::PLAN_ENTRY_MODULE; +use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -127,7 +127,7 @@ pub(crate) fn build_planner_entry_results<'py>( let config = runtime.getattr("project_config")?; let execution_scope = pruning.getattr("execution_scope")?; let relations = call_with( - &imported(py, PLAN_ENTRY_MODULE, "build_planner_relations_context")?, + &imported(py, NATIVE_MODULE, "build_planner_relations_context")?, &[ ("project", project.clone()), ("adapter", runtime.getattr("adapter")?), @@ -202,7 +202,7 @@ pub(crate) fn build_planner_entry_results<'py>( ], )?; let entries = call_with( - &imported(py, PLAN_ENTRY_MODULE, "build_plan_entries")?, + &imported(py, NATIVE_MODULE, "build_plan_entries")?, &[ ("project", project), ("adapter", runtime.getattr("adapter")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index d8cf02e822..c21f4b728a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,7 +1,7 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_phases::constants::PLAN_ENTRY_MODULE; +use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; @@ -98,7 +98,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( )?; fingerprints.call_method0("persist")?; let inspection_relations = call_with( - &imported(py, PLAN_ENTRY_MODULE, "build_planner_relations_context")?, + &imported(py, NATIVE_MODULE, "build_planner_relations_context")?, &[ ("project", project.clone()), ("adapter", adapter.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index cfffe8f312..0f20304fee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_identity, planner_migrations, planner_phases, planner_resolution, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, planner_warehouse, }; @@ -18,5 +18,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_scenario::registration::register(module)?; planner_strategy::registration::register(module)?; planner_resolution::registration::register(module)?; + planner_entries::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 212257f4dd..cce4a7c1d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -4,6 +4,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; +use crate::bindings::_helpers::planner_resolution::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; @@ -11,8 +14,6 @@ use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; const MODELS: &str = "sqlbuild.compiler.planner.models"; const CURSOR: &str = "sqlbuild._native"; -const POLICIES: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_policies"; -const FUTURE: &str = "sqlbuild.compiler.planner.main.execution.future_cursor_safety"; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { call_with( @@ -103,20 +104,8 @@ fn compute_model_cursor_bounds<'py>( } let context = args.value("context")?; let incremental_mode = config(&model, "incremental_mode")?; - let future_config = call_with( - &imported(py, POLICIES, "resolve_future_cursor_config")?, - &[ - ("model", model.clone()), - ("project_config", context.getattr("future_cursor_config")?), - ], - )?; - let start_config = call_with( - &imported(py, POLICIES, "resolve_start_cursor_config")?, - &[ - ("model", model.clone()), - ("project_config", context.getattr("start_cursor_config")?), - ], - )?; + let future_config = future_cursor_config(&model, &context.getattr("future_cursor_config")?)?; + let start_config = start_cursor_config(&model, &context.getattr("start_cursor_config")?)?; if incremental_mode.eq("microbatch")? { return unbounded(py); } @@ -270,7 +259,7 @@ fn compute_model_cursor_bounds<'py>( return Ok(bounds); } call_with( - &imported(py, FUTURE, "apply_future_cursor_safety")?, + &imported(py, NATIVE_MODULE, "resolve_future_cursor_safety")?, &[ ("bounds", bounds), ("cursor_type", cursor_type), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index a7d34b0743..5ef35e5179 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -21,7 +21,7 @@ fn type_cursor_overrides<'py>( py, "type_cursor_overrides", kwargs, - (&["cursor_overrides", "selected_models"], &[]), + (&["cursor_overrides"], &["selected_models"]), )?; let overrides = args.value("cursor_overrides")?; if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { @@ -29,7 +29,12 @@ fn type_cursor_overrides<'py>( } let get_config = imported(py, "sqlbuild._native", "get_config_str")?; let mut models: Vec<(String, String)> = Vec::new(); - for model in args.value("selected_models")?.try_iter()? { + let selected = if args.passed("selected_models")? { + args.value("selected_models")? + } else { + pyo3::types::PyTuple::empty(py).into_any() + }; + for model in selected.try_iter()? { let model = model?; let values = model.getattr("config")?.getattr("values")?; let config = |key: &str| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs new file mode 100644 index 0000000000..f0bd2a13dc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -0,0 +1,106 @@ +//! Model-level cursor safety policy overrides over project configuration. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; + +const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; +const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const POLICY_DISABLED: &str = "disabled"; + +/// The policy class, its distance field, and the model keys overriding distance and action. +struct PolicyFields { + class: &'static str, + distance: &'static str, + distance_key: &'static str, + action_key: &'static str, +} + +const FUTURE_POLICY: PolicyFields = PolicyFields { + class: "FutureCursorsConfig", + distance: "max_distance", + distance_key: "cursor_future_max_distance", + action_key: "cursor_future_action", +}; + +const START_POLICY: PolicyFields = PolicyFields { + class: "StartCursorsConfig", + distance: "max_ahead", + distance_key: "cursor_start_max_ahead", + action_key: "cursor_start_max_action", +}; + +fn resolve_policy<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, + fields: &PolicyFields, +) -> PyResult> { + let py = model.py(); + let class = imported(py, SPEC_MODELS, fields.class)?; + let configured = if project_config.is_truthy()? { + project_config.clone() + } else { + class.call0()? + }; + let values = model.getattr("config")?.getattr("values")?; + let raw_distance = values.call_method1("get", (fields.distance_key,))?; + let raw_action = values.call_method1("get", (fields.action_key,))?; + let mut distance = configured.getattr(fields.distance)?; + if raw_distance.is_instance_of::() { + distance = if raw_distance.eq(POLICY_DISABLED)? { + py.None().into_bound(py) + } else { + raw_distance + }; + } + let mut action = configured.getattr("action")?; + if raw_action.is_instance_of::() { + action = imported(py, SPEC_TYPES, "FutureCursorAction")?.call1((raw_action,))?; + } + if distance.is_none() { + return Ok(py.None().into_bound(py)); + } + call_with(&class, &[(fields.distance, distance), ("action", action)]) +} + +/// The future-end policy for one model, model configuration taking precedence. +pub(crate) fn future_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + resolve_policy(model, project_config, &FUTURE_POLICY) +} + +/// The automatic-start policy for one model, model configuration taking precedence. +pub(crate) fn start_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + resolve_policy(model, project_config, &START_POLICY) +} + +/// Resolve future-end policy with model config taking precedence. +#[pyfunction] +#[pyo3(signature = (*, model, project_config))] +fn resolve_future_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + future_cursor_config(model, project_config) +} + +/// Resolve automatic-start policy with model config taking precedence. +#[pyfunction] +#[pyo3(signature = (*, model, project_config))] +fn resolve_start_cursor_config<'py>( + model: &Bound<'py, PyAny>, + project_config: &Bound<'py, PyAny>, +) -> PyResult> { + start_cursor_config(model, project_config) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_future_cursor_config, module)?)?; + module.add_function(wrap_pyfunction!(resolve_start_cursor_config, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs new file mode 100644 index 0000000000..dcf00e724a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -0,0 +1,712 @@ +//! Plan-entry cursor windows: overrides, planner-owned bounds and microbatch ranges. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::{parse_value, sentinel_to_token}; +use crate::bindings::_helpers::cursor::values::{local_date_time, replaced}; +use crate::bindings::_helpers::planner_resolution::cursor_bounds::duration; +use crate::bindings::_helpers::planner_resolution::cursor_policies::{ + future_cursor_config, start_cursor_config, +}; +use crate::bindings::_helpers::planner_resolution::cursor_safety::{replace, utc_clock}; +use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use sqlbuild_cursor::algebra::main::compare::compare; +use sqlbuild_cursor::algebra::models::{Scalar, ScalarValue, Timestamp, ZoneOffset}; +use sqlbuild_cursor::planner::main::cursor_end_clamp::cursor_end_clamp; +use sqlbuild_cursor::planner::main::rolling_window::rolling_window; +use sqlbuild_cursor::planner::models::CursorEndClamp; +use std::cmp::Ordering; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; +const CURSOR_TYPES: &str = "sqlbuild.cursor_algebra.types"; +const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +const INCREMENTAL: &str = "incremental"; +const MICROBATCH: &str = "microbatch"; +const ROLLING_WINDOW: &str = "rolling_window"; +const TIMESTAMP: &str = "timestamp"; +const INTEGER: &str = "integer"; +const BOUNDED_BACKFILL: &str = "bounded"; +const FULL_BACKFILL: &str = "full"; +const WINDOW_REQUIRED: [&str; 7] = [ + "model", + "snapshot", + "backfill", + "full_refresh", + "start_cursor_override", + "end_cursor_override", + "runtime_owned_cursor_bounds", +]; +const WINDOW_OPTIONAL: [&str; 5] = [ + "invocation_time", + "future_cursor_config", + "start_cursor_config", + "replaces_relation", + "cursor_input_relations", +]; + +/// The planner inputs that decide one model's plan-time cursor window. +pub(crate) struct CursorWindowInputs<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) backfill: Bound<'py, PyAny>, + pub(crate) overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) runtime_owned: bool, + pub(crate) policies: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) invocation_time: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) replaces_relation: bool, + pub(crate) cursor_input_relations: Bound<'py, PyAny>, +} + +fn native<'py>( + py: Python<'py>, + name: &str, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, NATIVE_MODULE, name)?, pairs) +} + +/// One string-valued model configuration entry. +pub(crate) fn config_str<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + native( + model.py(), + "get_config_str", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +/// One cursor-bound model configuration entry as its token text. +pub(crate) fn config_bound<'py>( + model: &Bound<'py, PyAny>, + key: &str, +) -> PyResult> { + native( + model.py(), + "get_config_cursor_bound", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +fn boolean(py: Python<'_>, value: bool) -> PyResult> { + Ok(value.into_pyobject(py)?.to_owned().into_any()) +} + +/// A cursor token as the scalar it orders by: integers for integer cursors, else instants. +fn order_scalar<'py>( + session: &CursorSession<'py>, + token: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + let py = session.py(); + let kind = if cursor_type.eq(INTEGER)? { + INTEGER + } else { + TIMESTAMP + }; + session.scalar(&parse_value(token, &string(py, kind))?) +} + +fn order<'py>( + session: &CursorSession<'py>, + tokens: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + cursor_type: &Bound<'py, PyAny>, +) -> PyResult { + let left = order_scalar(session, tokens.0, cursor_type)?; + let right = order_scalar(session, tokens.1, cursor_type)?; + session.checked(compare(&left, &right, session)) +} + +fn token<'py>(py: Python<'py>, bound: &Bound<'py, PyAny>) -> PyResult> { + Ok(PyString::new(py, &sentinel_to_token(bound)?).into_any()) +} + +fn sentinel<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, CURSOR_TYPES, "BoundSentinel")?.getattr(name) +} + +/// `bounds` clamped so a concrete window never reaches the exclusive configured `cursor_end`. +fn clamp_cursor_end<'py>( + bounds: Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let cursor_end = config_bound(model, "cursor_end")?; + let cursor_type = config_str(model, "cursor_type")?; + if bounds.is_none() || cursor_end.is_none() { + return Ok(bounds); + } + let start = bounds.getattr("start")?; + let end = bounds.getattr("end")?; + if start.eq(sentinel(py, "START")?)? || end.eq(sentinel(py, "END")?)? { + return Ok(bounds); + } + let session = CursorSession::new(py)?; + let window = ( + order_scalar(&session, &token(py, &start)?, &cursor_type)?, + order_scalar(&session, &token(py, &end)?, &cursor_type)?, + ); + let limit = order_scalar(&session, &cursor_end, &cursor_type)?; + match session.checked(cursor_end_clamp((&window.0, &window.1), &limit, &session))? { + CursorEndClamp::Unchanged => Ok(bounds), + CursorEndClamp::Empty => replace( + &bounds, + &[("start", cursor_end.clone()), ("end", cursor_end)], + ), + CursorEndClamp::Capped => replace(&bounds, &[("end", cursor_end)]), + } +} + +fn timestamp_value<'py>(clock: &Bound<'py, PyAny>, scalar: &Scalar) -> PyResult> { + let ScalarValue::Timestamp(timestamp) = &scalar.value else { + return Ok(clock.py().None().into_bound(clock.py())); + }; + call_with( + &imported(clock.py(), CURSOR_MODELS, "TimestampValue")?, + &[("value", replaced(clock, ×tamp.local, timestamp.fold)?)], + ) +} + +/// The invocation-anchored rolling microbatch window, or `None` without a grain or lookback. +fn rolling_window_bounds<'py>( + model: &Bound<'py, PyAny>, + invocation_time: &Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let grain = config_str(model, "cursor_grain")?; + let lookback = native( + py, + "resolve_microbatch_lookback", + &[("model", model.clone()), ("effective_grain", grain.clone())], + )?; + let lookback = if lookback.is_truthy()? { + lookback + } else { + string(py, "") + }; + let span = duration(&lookback)?; + let (Some(span), false) = (span, grain.is_none()) else { + return Ok(py.None().into_bound(py)); + }; + let utc = imported(py, "datetime", "UTC")?; + let moment = if invocation_time.is_none() { + imported(py, "datetime", "datetime")?.call_method1("now", (&utc,))? + } else { + invocation_time.clone() + }; + let moment = if moment.getattr("tzinfo")?.is_none() { + call_with(&moment.getattr("replace")?, &[("tzinfo", utc)])? + } else { + utc_clock(&moment)? + }; + let session = CursorSession::new(py)?; + let Some(grain) = session.grain(&grain)? else { + return Ok(py.None().into_bound(py)); + }; + let clock = Scalar { + value: ScalarValue::Timestamp(Timestamp { + local: local_date_time(&moment)?, + fold: moment.getattr("fold")?.extract::()? != 0, + offset: ZoneOffset::Fixed(0), + template: None, + source_text: None, + }), + handle: None, + }; + let (start, end) = session.checked(rolling_window(&clock, &span, grain))?; + let mut bounds = call_with( + &imported(py, MODELS, "CursorBounds")?, + &[ + ("start", timestamp_value(&moment, &start)?), + ("end", timestamp_value(&moment, &end)?), + ], + )?; + let cursor_start = config_bound(model, "cursor_start")?; + let timestamp = string(py, TIMESTAMP); + if !cursor_start.is_none() + && order( + &session, + (&token(py, &bounds.getattr("start")?)?, &cursor_start), + ×tamp, + )? == Ordering::Less + { + bounds = replace(&bounds, &[("start", cursor_start)])?; + } + clamp_cursor_end(bounds, model) +} + +fn apply_future_safety<'py>( + bounds: Bound<'py, PyAny>, + typing: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + inputs: &CursorWindowInputs<'py>, + input_evidence: Bound<'py, PyAny>, +) -> PyResult> { + let py = bounds.py(); + if bounds.is_none() || bounds.getattr("end")?.eq(sentinel(py, "END")?)? { + return Ok(bounds); + } + native( + py, + "resolve_future_cursor_safety", + &[ + ("bounds", bounds), + ("cursor_type", typing.0.clone()), + ("cursor_grain", typing.1.clone()), + ("config", inputs.policies.0.clone()), + ("invocation_time", inputs.invocation_time.clone()), + ( + "has_complete_override", + boolean( + py, + !inputs.overrides.0.is_none() && !inputs.overrides.1.is_none(), + )?, + ), + ("input_evidence", input_evidence), + ], + ) +} + +fn bounded_override<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + native( + model.py(), + "resolve_bounded_cursor_override", + &[ + ("start_cursor_override", inputs.overrides.0.clone()), + ("end_cursor_override", inputs.overrides.1.clone()), + ("cursor_type", config_str(model, "cursor_type")?), + ("cursor_grain", config_str(model, "cursor_grain")?), + ], + ) +} + +fn backfill_duration<'py>(backfill: &Bound<'py, PyAny>) -> PyResult> { + if backfill.getattr("action")?.eq(BOUNDED_BACKFILL)? { + return backfill.getattr("duration"); + } + Ok(backfill.py().None().into_bound(backfill.py())) +} + +/// Cursor bounds and the maximum-start policy inputs, shared by both window computations. +struct BoundsRequest<'py> { + cursor_snapshot: Bound<'py, PyAny>, + typing: (Bound<'py, PyAny>, Bound<'py, PyAny>), + lookback: Bound<'py, PyAny>, + is_microbatch: bool, +} + +fn snapshot_bounds<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, + request: BoundsRequest<'py>, +) -> PyResult> { + let py = model.py(); + let policy = call_with( + &imported(py, MODELS, "MaximumStartPolicyInputs")?, + &[ + ("config", inputs.policies.1.clone()), + ("invocation_time", inputs.invocation_time.clone()), + ( + "incremental_strategy", + config_str(model, "incremental_strategy")?, + ), + ("incremental_mode", config_str(model, "incremental_mode")?), + ], + )?; + let bounds = native( + py, + "compute_cursor_bounds", + &[ + ("cursor_snapshot", request.cursor_snapshot.clone()), + ("cursor_type", request.typing.0.clone()), + ("cursor_start", config_bound(model, "cursor_start")?), + ("lookback", request.lookback), + ("backfill_duration", backfill_duration(&inputs.backfill)?), + ("start_cursor_override", inputs.overrides.0.clone()), + ("end_cursor_override", inputs.overrides.1.clone()), + ("is_microbatch", boolean(py, request.is_microbatch)?), + ("cursor_grain", request.typing.1.clone()), + ("maximum_start_policy", policy), + ], + )?; + let bounds = apply_future_safety( + bounds, + (&request.typing.0, &request.typing.1), + inputs, + request.cursor_snapshot.getattr("input_evidence")?, + )?; + clamp_cursor_end(bounds, model) +} + +fn model_cursor_snapshot<'py>( + model: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, +) -> PyResult> { + snapshot + .getattr("cursor_snapshots")? + .call_method1("get", (model.getattr("name")?,)) +} + +fn without_destination<'py>(cursor_snapshot: Bound<'py, PyAny>) -> PyResult> { + native( + cursor_snapshot.py(), + "without_destination_cursor", + &[("cursor_snapshot", cursor_snapshot)], + ) +} + +/// Compute cursor bounds for inclusion on the plan entry. +pub(crate) fn plan_cursor_bounds<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + let py = model.py(); + if !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let bounded = bounded_override(model, inputs)?; + if !bounded.is_none() { + return clamp_cursor_end(bounded, model); + } + if inputs.runtime_owned || inputs.full_refresh { + return Ok(py.None().into_bound(py)); + } + let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return Ok(cursor_snapshot); + } + if inputs.backfill.getattr("action")?.eq(FULL_BACKFILL)? { + cursor_snapshot = without_destination(cursor_snapshot)?; + } + snapshot_bounds( + model, + inputs, + BoundsRequest { + cursor_snapshot, + typing: ( + config_str(model, "cursor_type")?, + config_str(model, "cursor_grain")?, + ), + lookback: config_str(model, "lookback")?, + is_microbatch: config_str(model, "incremental_mode")?.eq(MICROBATCH)?, + }, + ) +} + +/// Compute the real overall cursor range for microbatch batch splitting. +pub(crate) fn microbatch_range<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult> { + let py = model.py(); + if !config_str(model, "materialized")?.eq(INCREMENTAL)? + || !config_str(model, "incremental_mode")?.eq(MICROBATCH)? + || config_str(model, "cursor")?.is_none() + { + return Ok(py.None().into_bound(py)); + } + let bounded = bounded_override(model, inputs)?; + if !bounded.is_none() { + return clamp_cursor_end(bounded, model); + } + let strategy = config_str(model, "microbatch_strategy")?; + if strategy.eq(ROLLING_WINDOW)? { + let rolling = rolling_window_bounds(model, &inputs.invocation_time)?; + if inputs.full_refresh && !rolling.is_none() { + let cursor_start = config_bound(model, "cursor_start")?; + if !cursor_start.is_none() { + return replace(&rolling, &[("start", cursor_start)]); + } + } + return Ok(rolling); + } + if inputs.runtime_owned { + return Ok(py.None().into_bound(py)); + } + let mut cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return Ok(cursor_snapshot); + } + if inputs.replaces_relation && !inputs.full_refresh { + cursor_snapshot = without_destination(cursor_snapshot)?; + } + let cursor_type = config_str(model, "cursor_type")?; + let mut grains: Vec> = Vec::new(); + for relation in inputs.cursor_input_relations.try_iter()? { + grains.push(relation?.getattr("cursor_grain")?); + } + let effective_grain = native( + py, + "resolve_effective_timestamp_grain", + &[ + ("cursor_type", cursor_type.clone()), + ("downstream_grain", config_str(model, "cursor_grain")?), + ("microbatch_strategy", strategy), + ("cursor_input_grains", PyTuple::new(py, grains)?.into_any()), + ], + )?; + if cursor_type.eq(TIMESTAMP)? && !effective_grain.is_none() { + cursor_snapshot = native( + py, + "normalize_cursor_snapshot_grain", + &[ + ("cursor_snapshot", cursor_snapshot), + ("cursor_type", cursor_type.clone()), + ("effective_grain", effective_grain.clone()), + ], + )?; + } + let lookback = native( + py, + "resolve_microbatch_lookback", + &[ + ("model", model.clone()), + ("effective_grain", effective_grain.clone()), + ], + )?; + snapshot_bounds( + model, + inputs, + BoundsRequest { + cursor_snapshot, + typing: (cursor_type, effective_grain), + lookback, + is_microbatch: false, + }, + ) +} + +/// Empty inputs that leave a planner-owned window empty, and whether others have rows. +pub(crate) fn empty_cursor_inputs<'py>( + model: &Bound<'py, PyAny>, + inputs: &CursorWindowInputs<'py>, +) -> PyResult<(Bound<'py, PyAny>, bool)> { + let py = model.py(); + let nothing = + || -> PyResult<(Bound<'py, PyAny>, bool)> { Ok((PyTuple::empty(py).into_any(), false)) }; + if inputs.full_refresh + || inputs.runtime_owned + || !config_str(model, "materialized")?.eq(INCREMENTAL)? + || config_str(model, "cursor")?.is_none() + || config_str(model, "microbatch_strategy")?.eq(ROLLING_WINDOW)? + || (!inputs.overrides.0.is_none() && !inputs.overrides.1.is_none()) + { + return nothing(); + } + let cursor_snapshot = model_cursor_snapshot(model, &inputs.snapshot)?; + if cursor_snapshot.is_none() { + return nothing(); + } + let decision = native( + py, + "cursor_inputs_without_rows", + &[("cursor_snapshot", cursor_snapshot)], + )?; + Ok(( + decision.getattr("input_names")?, + decision.getattr("waiting_on_empty_inputs")?.is_truthy()?, + )) +} + +/// The future-end and automatic-start policies effective for one model. +pub(crate) fn cursor_safety_configs<'py>( + model: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + Ok(( + future_cursor_config(model, &context.getattr("future_cursor_config")?)?, + start_cursor_config(model, &context.getattr("start_cursor_config")?)?, + )) +} + +fn compile_input_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, COMPILE_EXCEPTIONS, "CompileInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Resolve typed cursor overrides to generic start/end for one model. +pub(crate) fn model_cursor_overrides<'py>( + model: &Bound<'py, PyAny>, + cursor_overrides: &Bound<'py, PyAny>, + (start, end): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = model.py(); + let cursor_type = config_str(model, "cursor_type")?; + let (mut resolved_start, mut resolved_end) = (start, end); + let typed_fields = if cursor_overrides.is_none() { + None + } else if cursor_type.eq(TIMESTAMP)? { + Some(("start_ts", "end_ts")) + } else if cursor_type.eq(INTEGER)? { + Some(("start_int", "end_int")) + } else { + None + }; + if let Some((start_field, end_field)) = typed_fields { + let typed_start = cursor_overrides.getattr(start_field)?; + if !typed_start.is_none() { + resolved_start = token(py, &typed_start)?; + } + let typed_end = cursor_overrides.getattr(end_field)?; + if !typed_end.is_none() { + resolved_end = token(py, &typed_end)?; + } + } + let session = CursorSession::new(py)?; + let name = model.getattr("name")?.str()?; + let cursor_start = config_bound(model, "cursor_start")?; + let cursor_end = config_bound(model, "cursor_end")?; + if !resolved_start.is_none() + && !cursor_start.is_none() + && order(&session, (&resolved_start, &cursor_start), &cursor_type)? == Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit start {} is before cursor_start {}", + resolved_start.str()?, + cursor_start.str()? + ), + )); + } + if !cursor_end.is_none() { + for (value, label) in [(&resolved_start, "start"), (&resolved_end, "end")] { + if !value.is_none() + && order(&session, (value, &cursor_end), &cursor_type)? != Ordering::Less + { + return Err(compile_input_error( + py, + format!( + "model '{name}': explicit {label} {} is at or beyond exclusive cursor_end {}", + value.str()?, + cursor_end.str()? + ), + )); + } + } + } + Ok((resolved_start, resolved_end)) +} + +fn window_inputs<'py>(args: &KeywordArguments<'py>) -> PyResult> { + Ok(CursorWindowInputs { + snapshot: args.value("snapshot")?, + backfill: args.value("backfill")?, + overrides: ( + args.value("start_cursor_override")?, + args.value("end_cursor_override")?, + ), + runtime_owned: args.flag("runtime_owned_cursor_bounds")?, + policies: ( + args.value("future_cursor_config")?, + args.value("start_cursor_config")?, + ), + invocation_time: args.value("invocation_time")?, + full_refresh: args.flag("full_refresh")?, + replaces_relation: args.flag("replaces_relation")?, + cursor_input_relations: if args.passed("cursor_input_relations")? { + args.value("cursor_input_relations")? + } else { + PyTuple::empty(args.value("snapshot")?.py()).into_any() + }, + }) +} + +/// Compute cursor bounds for inclusion on the plan entry. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn compute_plan_cursor_bounds<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "compute_plan_cursor_bounds", + kwargs, + (&WINDOW_REQUIRED, &WINDOW_OPTIONAL), + )?; + plan_cursor_bounds(&args.value("model")?, &window_inputs(&args)?) +} + +/// Compute the real overall cursor range for microbatch batch splitting. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn compute_microbatch_range<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "compute_microbatch_range", + kwargs, + (&WINDOW_REQUIRED, &WINDOW_OPTIONAL), + )?; + microbatch_range(&args.value("model")?, &window_inputs(&args)?) +} + +/// Return empty inputs that leave a planner-owned window empty, and whether others have rows. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_empty_cursor_inputs<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyAny>, bool)> { + let args = KeywordArguments::read( + py, + "resolve_empty_cursor_inputs", + kwargs, + ( + &[ + "model", + "snapshot", + "full_refresh", + "runtime_owned_cursor_bounds", + "start_cursor_override", + "end_cursor_override", + ], + &[], + ), + )?; + empty_cursor_inputs(&args.value("model")?, &window_inputs(&args)?) +} + +/// Resolve typed cursor overrides to generic start/end for one model. +#[pyfunction] +#[pyo3(signature = (*, model, cursor_overrides, start_cursor_override, end_cursor_override))] +fn resolve_cursor_overrides<'py>( + model: &Bound<'py, PyAny>, + cursor_overrides: &Bound<'py, PyAny>, + start_cursor_override: Bound<'py, PyAny>, + end_cursor_override: Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + model_cursor_overrides( + model, + cursor_overrides, + (start_cursor_override, end_cursor_override), + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compute_plan_cursor_bounds, module)?)?; + module.add_function(wrap_pyfunction!(compute_microbatch_range, module)?)?; + module.add_function(wrap_pyfunction!(resolve_empty_cursor_inputs, module)?)?; + module.add_function(wrap_pyfunction!(resolve_cursor_overrides, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs new file mode 100644 index 0000000000..07ba97a0c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs @@ -0,0 +1,115 @@ +//! Prominent warnings for applied future-end and automatic-start cursor safety. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cursor::scalars::render_value; +use crate::bindings::_helpers::planner_resolution::values::string; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const MODELS: &str = "sqlbuild.compiler.planner.models"; +const TYPES: &str = "sqlbuild.compiler.planner.types"; +const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; +const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; + +fn rendered(evidence: &Bound<'_, PyAny>, field: &str) -> PyResult { + render_value(&evidence.getattr(field)?) +} + +/// Render a prominent warning for applied future cursor policy. +pub(crate) fn future_cap_message(bounds: &Bound<'_, PyAny>) -> PyResult> { + if bounds.is_none() { + return Ok(None); + } + let evidence = bounds.getattr("future_safety")?; + if evidence.is_none() { + return Ok(None); + } + Ok(Some(format!( + "FUTURE CURSOR CAPPED: discovered bounds ['{}', '{}'), applied bounds ['{}', '{}').", + rendered(&evidence, "discovered_start")?, + rendered(&evidence, "discovered_end")?, + rendered(&evidence, "applied_start")?, + rendered(&evidence, "applied_end")?, + ))) +} + +/// Render a warning that distinguishes start recovery from future-end capping. +pub(crate) fn maximum_start_message(bounds: &Bound<'_, PyAny>) -> PyResult> { + if bounds.is_none() { + return Ok(None); + } + let evidence = bounds.getattr("maximum_start_safety")?; + if evidence.is_none() { + return Ok(None); + } + let highest = evidence.getattr("highest_eligible_target_max")?; + let highest = if highest.is_none() { + "None".to_owned() + } else { + PyString::new(bounds.py(), &render_value(&highest)?) + .repr()? + .extract()? + }; + Ok(Some(format!( + "AUTOMATIC START CAPPED: physical target MAX '{}' exceeded horizon '{}'; highest \ + eligible target MAX={highest}, effective post-lookback start='{}', action={}, \ + input={}.{}.", + rendered(&evidence, "physical_target_max")?, + rendered(&evidence, "maximum_allowed_start")?, + rendered(&evidence, "effective_start")?, + evidence.getattr("action")?.getattr("value")?.str()?, + evidence.getattr("target_relation")?.str()?, + evidence.getattr("cursor_column")?.str()?, + ))) +} + +/// `warnings` followed by the future-end and automatic-start warnings `bounds` carries. +pub(crate) fn with_cursor_safety_warnings<'py>( + warnings: &Bound<'py, PyAny>, + model_name: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, +) -> PyResult> { + let py = warnings.py(); + let mut all: Vec> = warnings + .try_iter()? + .collect::>>>()?; + for (message, code) in [ + (future_cap_message(bounds)?, FUTURE_CAPPED_CODE), + (maximum_start_message(bounds)?, START_CAPPED_CODE), + ] { + let Some(message) = message else { continue }; + all.push(call_with( + &imported(py, MODELS, "PlanWarning")?, + &[ + ("model_name", model_name.clone()), + ( + "severity", + imported(py, TYPES, "WarningSeverity")?.getattr("WARNING")?, + ), + ("code", string(py, code)), + ("message", string(py, &message)), + ], + )?); + } + Ok(PyTuple::new(py, all)?.into_any()) +} + +/// Render a prominent warning for applied future cursor policy. +#[pyfunction] +#[pyo3(signature = (bounds, /))] +fn future_cursor_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { + future_cap_message(bounds) +} + +/// Render a warning that distinguishes start recovery from future-end capping. +#[pyfunction] +#[pyo3(signature = (bounds, /))] +fn maximum_start_cap_warning(bounds: &Bound<'_, PyAny>) -> PyResult> { + maximum_start_message(bounds) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(future_cursor_cap_warning, module)?)?; + module.add_function(wrap_pyfunction!(maximum_start_cap_warning, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index b90f4dc22a..6fa15a9dae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -4,7 +4,10 @@ mod constants; mod cursor_bounds; mod cursor_model; mod cursor_overrides; +pub(crate) mod cursor_policies; +pub(crate) mod cursor_ranges; mod cursor_safety; +pub(crate) mod cursor_warnings; mod cursor_windows; mod future_safety; mod locations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index fe8a260dbd..f6ec9df817 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -3,13 +3,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::planner_resolution::values::{NATIVE_MODULE, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const CONFIG_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.config"; -const INTRINSICS_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics"; const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; const ERROR_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -213,8 +212,7 @@ fn resolve_model_sql<'py>( &imported(py, NATIVE_MODULE, "resolve_cursor_input_roles")?, &[("model", model.clone())], )?; - let inclusive = - imported(py, CONFIG_MODULE, "get_config_append_cursor_inclusive")?.call1((&model,))?; + let inclusive = append_cursor_inclusive(&model)?; let mut sql = resolution.resolve( model.getattr("query_sql")?, &[ @@ -241,20 +239,64 @@ fn resolve_model_sql<'py>( }; return Err(unresolved_bounds_error(py, &name, cause.as_deref())?); } - sql = call_with( - &imported(py, INTRINSICS_MODULE, "render_cursor_intrinsic_bounds")?, - &[ - ("sql", sql), - ("bounds", bounds), - ("cursor_type", cursor_type), - ("adapter", resolution.adapter.clone()), - ], - )?; + sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; Ok(sql) } +/// Effective append cursor lower-bound inclusivity. +fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py = model.py(); + let strategy = call_with( + &imported(py, NATIVE_MODULE, "get_config_str")?, + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(py, "incremental_strategy")), + ], + )?; + let raw = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("append_cursor_inclusive",))?; + let inclusive = + !strategy.eq("append")? || !raw.is_instance_of::() || raw.is_truthy()?; + Ok(inclusive.into_pyobject(py)?.to_owned().into_any()) +} + +/// `__cursor_start()` / `__cursor_end()` replaced with the adapter's typed bound literals. +fn intrinsic_bounds<'py>( + sql: &Bound<'py, PyAny>, + bounds: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = sql.py(); + let literal = |field: &str| -> PyResult> { + call_with( + &adapter.getattr("render_cursor_bound_literal")?, + &[ + ( + "value", + PyString::new(py, &sentinel_to_token(&bounds.getattr(field)?)?).into_any(), + ), + ("cursor_type", cursor_type.clone()), + ], + ) + }; + let start_sql = literal("start")?; + let end_sql = literal("end")?; + call_with( + &imported(py, NATIVE_MODULE, "resolve_cursor_intrinsics")?, + &[ + ("sql", sql.clone()), + ("start_sql", start_sql), + ("end_sql", end_sql), + ], + )? + .get_item(0) +} + fn unresolved_bounds_error(py: Python<'_>, name: &str, cause: Option<&str>) -> PyResult { let error = imported(py, ERROR_MODULE, "PlannerInputError")?; let kwargs = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 6626407126..94a90504d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -6,6 +6,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::cursor_bounds::register(module)?; super::cursor_model::register(module)?; super::cursor_overrides::register(module)?; + super::cursor_policies::register(module)?; + super::cursor_ranges::register(module)?; + super::cursor_warnings::register(module)?; super::future_safety::register(module)?; super::maximum_safety::register(module)?; super::cursor_windows::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 1cf94c2c32..22748240c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -74,11 +74,7 @@ pub(super) fn function_entry<'py>( ) -> PyResult> { let py = function.py(); let body = call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.resolve.resolve", - "resolve_function_sql", - )?, + &imported(py, "sqlbuild._native", "resolve_function_sql")?, &[ ("adapter", context.adapter.clone()), ("function", function.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 88eb774c88..e82d4e2fda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -127,11 +127,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( let model = imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; let model_context = model_context(&project, &context, (&models, &locations, &columns))?; let result = call_with( - &imported( - py, - "sqlbuild.compiler.planner._helpers.output.plan_entry", - "plan_model", - )?, + &imported(py, "sqlbuild._native", "plan_model")?, &[ ("model", model), ("snapshot", snapshot.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 21f6f9a2d8..c44191bdae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -33,19 +33,12 @@ pub(crate) const LOG_DEBUG_EVENT: (&str, &str) = ( "sqlbuild.diagnostics.main.log_debug_event", "log_debug_event", ); -pub(crate) const CURSOR_RESOLVE_MODULE: &str = "sqlbuild.compiler.planner._helpers.resolve.cursor"; -pub(crate) const CURSOR_POLICIES: (&str, &str) = ( - "sqlbuild.compiler.planner._helpers.resolve.cursor_policies", - "resolve_start_cursor_config", -); -pub(crate) const MAXIMUM_START: (&str, &str) = ( - "sqlbuild.compiler.planner._helpers.resolve.maximum_start", - "maximum_allowed_start", -); -pub(crate) const RENDER_SOURCE_READ: (&str, &str) = ( - "sqlbuild.compiler.planner._helpers.resolve.sources", - "render_source_read_relation", -); +pub(crate) const CURSOR_RESOLVE_MODULE: &str = "sqlbuild._native"; +pub(crate) const CURSOR_POLICIES: (&str, &str) = + ("sqlbuild._native", "resolve_start_cursor_config"); +pub(crate) const MAXIMUM_START: (&str, &str) = ("sqlbuild._native", "maximum_allowed_start"); +pub(crate) const RENDER_SOURCE_READ: (&str, &str) = + ("sqlbuild._native", "render_source_read_relation"); pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ( "sqlbuild.compiler.discovery.main._yaml_entry_line", "yaml_entry_line", diff --git a/python-boundary.toml b/python-boundary.toml index 924fcc11ea..a70b835915 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 21360 +remaining_lines = 19798 [[stays]] path = "executor" diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index e528e91df7..085a2d85f9 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -37,7 +37,6 @@ ALLOWED_METADATA_LOOP_PATHS: frozenset[str] = frozenset( { "src/sqlbuild/adapter/relations/main/relation_lookup.py", - "src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py", "src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py", "src/sqlbuild/executor/janitor/_helpers/plan.py", "src/sqlbuild/executor/pipeline/_helpers/testing.py", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e3ca62302f..7a150b3625 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -251,6 +251,7 @@ from sqlbuild.compiler.planner.models import ( PlannerChangeResults, PlannerEntryResults, PlannerIdentityContext, + PlannerModelEntryResults, PlannerOverrides, PlannerPolicies, PlannerRelationsContext, @@ -4819,7 +4820,7 @@ def cursor_sentinel_to_token(*, sentinel: Bound) -> str: ... def cursor_sentinel_from_token(*, token: str) -> BoundSentinel | None: ... def cursor_compare(*, left: CursorScalar, right: CursorScalar) -> int: ... def type_cursor_overrides( - *, cursor_overrides: CursorOverrides | None, selected_models: Iterable[CompiledModelView] + *, cursor_overrides: CursorOverrides | None, selected_models: Iterable[CompiledModelView] = () ) -> CursorOverrides | None: ... def maximum_allowed_start( *, discovered_value: str, cursor_grain: str | None, invocation_time: datetime, max_ahead: str @@ -4836,6 +4837,90 @@ def apply_maximum_start_policy( policy: MaximumStartPolicyInputs, has_start_override: bool, ) -> CursorBounds: ... +def resolve_future_cursor_config( + *, model: CompiledModelView, project_config: FutureCursorsConfig | None +) -> FutureCursorsConfig | None: ... +def resolve_start_cursor_config( + *, model: CompiledModelView, project_config: StartCursorsConfig | None +) -> StartCursorsConfig | None: ... +def future_cursor_cap_warning(bounds: CursorBounds | None, /) -> str | None: ... +def maximum_start_cap_warning(bounds: CursorBounds | None, /) -> str | None: ... +def compute_plan_cursor_bounds( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + backfill: BackfillResult, + full_refresh: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, + runtime_owned_cursor_bounds: bool, + invocation_time: datetime | None = None, + future_cursor_config: FutureCursorsConfig | None = None, + start_cursor_config: StartCursorsConfig | None = None, + replaces_relation: bool = False, + cursor_input_relations: tuple[CursorInputRelation, ...] = (), +) -> CursorBounds | None: ... +def compute_microbatch_range( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + backfill: BackfillResult, + full_refresh: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, + runtime_owned_cursor_bounds: bool, + invocation_time: datetime | None = None, + future_cursor_config: FutureCursorsConfig | None = None, + start_cursor_config: StartCursorsConfig | None = None, + replaces_relation: bool = False, + cursor_input_relations: tuple[CursorInputRelation, ...] = (), +) -> CursorBounds | None: ... +def resolve_empty_cursor_inputs( + *, + model: CompiledModelView, + snapshot: WarehouseSnapshot, + full_refresh: bool, + runtime_owned_cursor_bounds: bool, + start_cursor_override: str | None, + end_cursor_override: str | None, +) -> tuple[tuple[str, ...], bool]: ... +def resolve_cursor_overrides( + *, + model: CompiledModelView, + cursor_overrides: CursorOverrides | None, + start_cursor_override: str | None, + end_cursor_override: str | None, +) -> tuple[str | None, str | None]: ... +def build_planner_relations_context( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + scope: PlannerScope, + deferral: DeferralInputs | None = None, + project_config: ProjectConfig | None = None, + local_config: LocalConfig | None = None, + known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, + relation_keys: frozenset[CompiledObjectKey] | None = None, +) -> PlannerRelationsContext: ... +def build_plan_entries( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + relations: PlannerRelationsContext, + resolved_actions: PlannerResolvedActions, + cursor_overrides: CursorOverrides | None, + full_refresh: bool, + build_inputs: PlanEntryBuildInputs | None = None, +) -> PlannerModelEntryResults: ... +def build_model_materializations( + model_entries: tuple[ModelPlanEntry, ...], / +) -> dict[str, str]: ... +def scope_overlaps( + *, scope_deps: tuple[CompiledObjectKey, ...], selected_keys: frozenset[CompiledObjectKey] +) -> bool: ... def resolve_future_cursor_safety( *, bounds: CursorBounds, diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index 06b82aa8e6..17b17d23e7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -4,6 +4,7 @@ from sqlbuild._native import ( build_compiled_function_fingerprint_sql, + build_planner_relations_context, build_seed_identity, build_selected_audit_entries, extract_seed_columns, @@ -14,7 +15,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner._helpers.output.plan_output import ( build_selected_test_entries, plan_function, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py deleted file mode 100644 index 6dac3632fa..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ /dev/null @@ -1,1101 +0,0 @@ -"""Per-model plan entry construction helpers.""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from datetime import UTC, datetime -from decimal import Decimal -from typing import Any - -from sqlbuild._native import ( - apply_configured_microbatch_limits, - assemble_model_plan_entry, - audit_is_selected, - build_cursor_input_relations, - build_source_load_map, - build_source_read_map, - check_full_rebuild_allowed, - describe_full_rebuild_cause, - detect_model_changes, - gather_source_inspection, - get_config_cursor_bound, - get_config_str, - get_materialization_type, - get_semantic_source_columns, - microbatch_guidance, - replaces_incremental_relation, - resolve_bounded_cursor_override, - resolve_microbatch_lookback, - resolve_model_full_refresh, - resolve_model_plan_action, - runtime_cursor_producer_names, - source_binding_diagnostics, - validate_capped_watermark_inputs, - validate_source_cursor_input_columns, - with_declared_source_reads, -) -from sqlbuild._native import cursor_render as render -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - compute_cursor_bounds, - cursor_inputs_without_rows, - normalize_cursor_snapshot_grain, - resolve_effective_timestamp_grain, - without_destination_cursor, -) -from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import ( - resolve_future_cursor_config, - resolve_start_cursor_config, -) -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - apply_deferred_locations, - build_function_locations, - build_model_locations, - build_seed_locations, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import ( - resolve_model_hook_entries, - resolve_model_sql, -) -from sqlbuild.compiler.planner.constants import MICROBATCH_CONCURRENCY_MESSAGE -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning -from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - CursorBounds, - CursorInputEvidence, - CursorInputRelation, - CursorOverridePair, - CursorOverrides, - DeferralInputs, - Duration, - EmptyCursorInputDecision, - MaximumStartPolicyInputs, - ModelCursorSnapshot, - ModelPlanContext, - ModelPlanEntry, - PlanEntryBuildInputs, - PlannerModelEntryResults, - PlannerRelationsContext, - PlannerResolvedActions, - PlannerScope, - PlanWarning, - ResolvedModelAction, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - IncrementalMode, - MaterializationType, - MicrobatchStrategy, - PlanAction, - PlanReason, - WarningSeverity, -) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.cursor_algebra.models import TimestampValue -from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.models import ( - FutureCursorsConfig, - LocalConfig, - ProjectConfig, - SourceEntry, - StartCursorsConfig, -) - - -@dataclass(frozen=True) -class _MicrobatchRangeInputs: - snapshot: WarehouseSnapshot - backfill: BackfillResult - start_cursor_override: str | None - end_cursor_override: str | None - runtime_owned_cursor_bounds: bool - cursor_input_relations: tuple[CursorInputRelation, ...] - future_cursor_config: FutureCursorsConfig | None - start_cursor_config: StartCursorsConfig | None - invocation_time: datetime | None - full_refresh: bool - replaces_relation: bool = False - - -def build_planner_relations_context( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - scope: PlannerScope, - deferral: DeferralInputs | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, - known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerRelationsContext: - """Resolve relation locations and source metadata for plan entry construction.""" - - effective_deferral: DeferralInputs = deferral if deferral is not None else DeferralInputs() - model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) - seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - if effective_deferral.deferred_locations is not None: - model_locations, seed_locations = apply_deferred_locations( - model_locations=model_locations, - seed_locations=seed_locations, - deferred_locations=effective_deferral.deferred_locations, - selected_keys=scope.selected_keys, - ) - audit_relation_keys: frozenset[CompiledObjectKey] = _selected_audit_relation_keys( - project=project, - selected_keys=scope.selected_keys, - ) - source_map: dict[str, SourceEntry] = build_source_load_map( - project=project, - selected_keys=( - relation_keys - if relation_keys is not None - else scope.selected_keys | audit_relation_keys - ), - upstream_deps=scope.upstream_deps if scope.all_keys else None, - ) - source_read_map: dict[str, SourceEntry] = ( - build_source_read_map( - project=project, - source_map=source_map, - selected_keys=scope.selected_keys, - project_config=project_config, - local_config=local_config, - defer_sources_to=effective_deferral.defer_sources_to, - ) - if effective_deferral.source_deferral_enabled - else source_map - ) - python_source_map: dict[str, SourceEntry] = with_declared_source_reads( - project=project, source_map=source_map, source_names=scope.python_read_source_names - ) - python_source_read_map: dict[str, SourceEntry] = ( - build_source_read_map( - project=project, - source_map=python_source_map, - selected_keys=scope.selected_keys, - project_config=project_config, - local_config=local_config, - defer_sources_to=effective_deferral.defer_sources_to, - read_source_names=frozenset(python_source_map), - ) - if effective_deferral.source_deferral_enabled - else python_source_map - ) - source_columns: dict[str, tuple[ColumnInfo, ...]] - listed_source_names: frozenset[str] - source_columns, listed_source_names = _resolve_source_warehouse_columns( - project=project, - adapter=adapter, - connection=connection, - source_read_map=source_read_map, - known_source_columns=known_source_columns, - ) - return PlannerRelationsContext( - model_locations=model_locations, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_read_map=source_read_map, - python_source_read_map=python_source_read_map, - source_warehouse_columns=get_semantic_source_columns( - project=project, - adapter=adapter, - connection=connection, - source_read_map=source_read_map, - selected_keys=scope.selected_keys, - columns=source_columns, - ), - star_exclude_keyword=adapter.star_exclude_keyword(), - listed_source_names=listed_source_names, - ) - - -def _resolve_source_warehouse_columns( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - source_read_map: dict[str, SourceEntry], - known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None, -) -> tuple[dict[str, tuple[ColumnInfo, ...]], frozenset[str]]: - """Reuse already-gathered source columns for the read map or gather them once.""" - - if known_source_columns is not None: - known: dict[str, tuple[ColumnInfo, ...]] = { - name: known_source_columns[name] - for name in source_read_map - if name in known_source_columns - } - return known, frozenset(known) - return gather_source_inspection( - project=project, - adapter=adapter, - connection=connection, - source_entries=tuple(source_read_map.values()), - ) - - -def plan_model( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - adapter: BaseAdapter, - context: ModelPlanContext, - sql_analysis_enabled: bool, - query_change_tracking: bool, - full_refresh: bool, - cursor_overrides: CursorOverridePair, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: - """Detect changes and build a plan entry and warnings for a single model.""" - - change_result: ChangeDetectionResult = detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=sql_analysis_enabled, - query_change_tracking=query_change_tracking, - full_refresh=full_refresh, - ) - - return plan_model_from_change( - model=model, - snapshot=snapshot, - adapter=adapter, - context=context, - sql_analysis_enabled=sql_analysis_enabled, - full_refresh=full_refresh, - cursor_overrides=cursor_overrides, - change_result=change_result, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - - -def build_plan_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - resolved_actions: PlannerResolvedActions, - cursor_overrides: CursorOverrides | None, - full_refresh: bool, - build_inputs: PlanEntryBuildInputs | None = None, -) -> PlannerModelEntryResults: - """Build model plan entries from snapshot and per-model resolved actions.""" - - inputs: PlanEntryBuildInputs = ( - build_inputs if build_inputs is not None else PlanEntryBuildInputs() - ) - binding_diagnostics: tuple[CompilerDiagnostic, ...] = ( - *( - diagnostic - for diagnostic in project.diagnostics - if diagnostic.is_error and diagnostic.resource_type != CompiledResourceType.SQL_TEST - ), - *source_binding_diagnostics( - project=project, - columns=relations.source_warehouse_columns, - profile=adapter.expression_inference_profile(), - selected_keys=scope.selected_keys, - ), - ) - binding_diagnostics = tuple( - diagnostic for diagnostic in binding_diagnostics if diagnostic.is_error - ) - if binding_diagnostics: - raise PlannerInputError( - "\n".join( - f"error[{diagnostic.code}]: {diagnostic.message}\n" - f" --> {diagnostic.path}:{diagnostic.line or 1}:{diagnostic.column or 1}" - for diagnostic in binding_diagnostics - ), - code=binding_diagnostics[0].code, - ) - source_freshness_blocked_model_names: frozenset[str] = ( - inputs.source_freshness_blocked_model_names - ) - external_blocked_model_names: frozenset[str] = inputs.external_blocked_model_names - start_cursor_override: str | None = inputs.start_cursor_override - end_cursor_override: str | None = inputs.end_cursor_override - entries: list[ModelPlanEntry] = [] - warnings: list[PlanWarning] = [] - runtime_producer_names: frozenset[str] = runtime_cursor_producer_names( - scope=scope, - source_map=relations.source_map, - ) - key: CompiledObjectKey - for key in scope.execution_order: - if key not in scope.selected_keys or key.name not in resolved_actions.models: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - resolved: ResolvedModelAction = resolved_actions.models[key.name] - if model is None: - continue - resolved_start: str | None - resolved_end: str | None - resolved_start, resolved_end = resolve_cursor_overrides( - model=model, - cursor_overrides=cursor_overrides, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - ) - entry: ModelPlanEntry - entry_warnings: tuple[PlanWarning, ...] - effective_full_refresh: bool = resolve_model_full_refresh( - model=model, - cli_full_refresh=full_refresh, - ) - entry, entry_warnings = plan_model_from_change( - model=model, - snapshot=snapshot, - adapter=adapter, - context=ModelPlanContext( - model_locations=relations.model_locations, - models_by_name=scope.models_by_name, - functions_by_name={function.name: function for function in project.functions}, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_read_map, - source_warehouse_columns=relations.source_warehouse_columns, - star_exclude_keyword=relations.star_exclude_keyword, - runtime_cursor_producer_names=runtime_producer_names, - future_cursor_config=inputs.future_cursor_config, - start_cursor_config=inputs.start_cursor_config, - invocation_time=inputs.invocation_time, - ), - sql_analysis_enabled=project.settings.sql_analysis, - full_refresh=effective_full_refresh, - cursor_overrides=CursorOverridePair( - start_cursor_override=resolved_start, - end_cursor_override=resolved_end, - ), - change_result=resolved.change, - external_sql_reference_resolver=project.external_sql_reference_resolver, - protect_full_refresh=key.name not in source_freshness_blocked_model_names - and key.name not in external_blocked_model_names, - ) - if entry.name in source_freshness_blocked_model_names: - entry = replace( - entry, - action=PlanAction.SKIP, - reason=PlanReason.SOURCE_FRESHNESS_ERROR, - ) - if entry.name in external_blocked_model_names: - entry = replace( - entry, - action=PlanAction.SKIP, - reason=PlanReason.EXTERNAL_UPSTREAM_FAILED, - ) - if entry.incremental_mode == IncrementalMode.MICROBATCH: - entry = replace( - entry, - unaccounted_partition_policy=( - entry.unaccounted_partition_policy - or project.settings.microbatch_unaccounted_partition_policy - ), - ) - if entry.action != PlanAction.SKIP: - entry, limit_warnings = apply_configured_microbatch_limits( - entry=entry, inputs=inputs - ) - warnings.extend(limit_warnings) - entries.append(entry) - warnings.extend(entry_warnings) - configured_concurrent_entries: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in entries - if entry.batch_concurrency > 1 and entry.action != PlanAction.SKIP - ) - if configured_concurrent_entries and not project.settings.microbatch_concurrency: - names: str = ", ".join(entry.name for entry in configured_concurrent_entries) - note, guidance = microbatch_guidance() - raise CompileInputError(f"{MICROBATCH_CONCURRENCY_MESSAGE}{names}; {note}", help=guidance) - if configured_concurrent_entries and not adapter.supports_concurrent_microbatch_dml(): - raise CompileInputError( - f"adapter '{adapter.adapter_name}' does not support concurrent microbatch " - "delete/insert DML; set batch_concurrency = 1" - ) - concurrent_entries: tuple[ModelPlanEntry, ...] = tuple( - entry for entry in configured_concurrent_entries if entry.action != PlanAction.CREATE_TABLE - ) - if concurrent_entries and project.settings.concurrency > 1: - count: int = len(concurrent_entries) - names = ", ".join(entry.name for entry in concurrent_entries) - warnings.append( - PlanWarning( - model_name=None, - severity=WarningSeverity.WARNING, - code="MICROBATCH_CONCURRENCY_COMPLEXITY", - message=( - f"Concurrent microbatch execution is enabled for {count} selected " - f"model{'s' if count != 1 else ''}: {names}. Concurrent batches may complete " - "out of order and rely on persistent microbatch history for gap " - "recovery and fingerprint provenance. State deletion or corruption " - "can require synthetic reconciliation and weaken version guarantees. " - "Keep batch_concurrency at 1 unless the runtime benefit justifies " - "this complexity; see the microbatch concurrency and recovery " - "documentation." - ), - ) - ) - return PlannerModelEntryResults(entries=tuple(entries), warnings=tuple(warnings)) - - -def plan_model_from_change( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - adapter: BaseAdapter, - context: ModelPlanContext, - sql_analysis_enabled: bool, - full_refresh: bool, - cursor_overrides: CursorOverridePair, - change_result: ChangeDetectionResult, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - protect_full_refresh: bool = True, -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: - """Build a model plan entry from a resolved change result.""" - - start_cursor_override: str | None = cursor_overrides.start_cursor_override - end_cursor_override: str | None = cursor_overrides.end_cursor_override - backfill: BackfillResult = ( - BackfillResult(action=BackfillAction.FULL) if full_refresh else change_result.backfill - ) - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - validate_source_cursor_input_columns( - model=model, - cursor_column=cursor_column, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - source_map=context.source_map, - source_warehouse_columns=context.source_warehouse_columns, - ) - - action: PlanAction - reason: PlanReason - action, reason = resolve_model_plan_action( - model=model, - change_result=change_result, - full_refresh=full_refresh, - ) - materialization_type: MaterializationType = get_materialization_type(model) - replaces_relation: bool = replaces_incremental_relation( - materialization_type=materialization_type, action=action - ) - full_rebuild_cause: str | None = ( - describe_full_rebuild_cause(change_result=change_result, full_refresh=full_refresh) - if replaces_relation - else None - ) - if full_rebuild_cause is not None and protect_full_refresh and not full_refresh: - check_full_rebuild_allowed( - model=model, change_result=change_result, full_rebuild_cause=full_rebuild_cause - ) - - resolved_sql: str = resolve_model_sql( - adapter=adapter, - model=model, - snapshot=snapshot, - context=context, - backfill=backfill, - full_refresh=full_refresh, - cursor_overrides=cursor_overrides, - external_sql_reference_resolver=external_sql_reference_resolver, - replaces_relation=replaces_relation, - full_rebuild_cause=full_rebuild_cause, - ) - - pre_hooks, post_hooks = resolve_model_hook_entries( - model=model, - adapter=adapter, - context=context, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - effective_future_cursor_config, effective_start_cursor_config = _resolve_cursor_safety_configs( - model=model, context=context - ) - validate_capped_watermark_inputs( - model=model, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - ) - cursor_input_relations: tuple[CursorInputRelation, ...] = build_cursor_input_relations( - model=model, - adapter=adapter, - model_locations=context.model_locations, - models_by_name=context.models_by_name, - functions_by_name=context.functions_by_name, - seed_locations=context.seed_locations, - source_map=context.source_map, - cursor_column=cursor_column, - runtime_cursor_producer_names=context.runtime_cursor_producer_names, - ) - runtime_owned_cursor_bounds: bool = any( - relation.is_runtime_owned for relation in cursor_input_relations - ) and not (start_cursor_override is not None and end_cursor_override is not None) - cursor_bounds: CursorBounds | None = _compute_plan_cursor_bounds( - model=model, - snapshot=snapshot, - backfill=backfill, - full_refresh=full_refresh, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - future_cursor_config=effective_future_cursor_config, - start_cursor_config=effective_start_cursor_config, - invocation_time=context.invocation_time, - ) - - empty_cursor_inputs: tuple[str, ...] - waiting_on_empty_inputs: bool - empty_cursor_inputs, waiting_on_empty_inputs = _resolve_empty_cursor_inputs( - model=model, - snapshot=snapshot, - full_refresh=full_refresh, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - cursor_overrides=cursor_overrides, - ) - - microbatch_range: CursorBounds | None = _compute_microbatch_range( - model=model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=backfill, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - cursor_input_relations=cursor_input_relations, - future_cursor_config=effective_future_cursor_config, - start_cursor_config=effective_start_cursor_config, - invocation_time=context.invocation_time, - full_refresh=full_refresh, - replaces_relation=replaces_relation, - ), - ) - - entry: ModelPlanEntry - warnings: tuple[PlanWarning, ...] - entry, warnings = assemble_model_plan_entry( - model=model, - adapter=adapter, - snapshot=snapshot, - change_result=change_result, - action=action, - reason=reason, - materialization_type=materialization_type, - resolved_sql=resolved_sql, - sql_analysis_enabled=sql_analysis_enabled, - pre_hooks=pre_hooks, - post_hooks=post_hooks, - cursor_column=cursor_column, - cursor_input_relations=cursor_input_relations, - runtime_owned_cursor_bounds=runtime_owned_cursor_bounds, - cursor_bounds=cursor_bounds, - microbatch_range=microbatch_range, - empty_cursor_inputs=empty_cursor_inputs, - waiting_on_empty_inputs=waiting_on_empty_inputs, - backfill=backfill, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - future_cursor_config=effective_future_cursor_config, - start_cursor_config=effective_start_cursor_config, - invocation_time=context.invocation_time, - ) - return entry, _append_cursor_safety_warnings( - warnings=warnings, - model_name=model.name, - bounds=microbatch_range or cursor_bounds, - ) - - -def resolve_cursor_overrides( - *, - model: CompiledModelView, - cursor_overrides: CursorOverrides | None, - start_cursor_override: str | None, - end_cursor_override: str | None, -) -> tuple[str | None, str | None]: - """Resolve typed cursor overrides to generic start/end for one model.""" - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - resolved_start: str | None = start_cursor_override - resolved_end: str | None = end_cursor_override - - if cursor_overrides is not None and cursor_type == CursorType.TIMESTAMP: - if cursor_overrides.start_ts is not None: - resolved_start = render(value=cursor_overrides.start_ts) - if cursor_overrides.end_ts is not None: - resolved_end = render(value=cursor_overrides.end_ts) - elif cursor_overrides is not None and cursor_type == CursorType.INTEGER: - if cursor_overrides.start_int is not None: - resolved_start = render(value=cursor_overrides.start_int) - if cursor_overrides.end_int is not None: - resolved_end = render(value=cursor_overrides.end_int) - - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - cursor_end: str | None = get_config_cursor_bound(values=model.config.values, key="cursor_end") - if ( - resolved_start is not None - and cursor_start is not None - and _cursor_order_key(value=resolved_start, cursor_type=cursor_type) - < _cursor_order_key(value=cursor_start, cursor_type=cursor_type) - ): - raise CompileInputError( - f"model '{model.name}': explicit start {resolved_start} is before cursor_start " - f"{cursor_start}" - ) - if cursor_end is not None: - if resolved_start is not None and _cursor_order_key( - value=resolved_start, cursor_type=cursor_type - ) >= _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - raise CompileInputError( - f"model '{model.name}': explicit start {resolved_start} is at or beyond " - f"exclusive cursor_end {cursor_end}" - ) - if resolved_end is not None and _cursor_order_key( - value=resolved_end, cursor_type=cursor_type - ) >= _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - raise CompileInputError( - f"model '{model.name}': explicit end {resolved_end} is at or beyond " - f"exclusive cursor_end {cursor_end}" - ) - return resolved_start, resolved_end - - -def _resolve_cursor_safety_configs( - *, model: CompiledModelView, context: ModelPlanContext -) -> tuple[FutureCursorsConfig | None, StartCursorsConfig | None]: - return ( - resolve_future_cursor_config(model=model, project_config=context.future_cursor_config), - resolve_start_cursor_config(model=model, project_config=context.start_cursor_config), - ) - - -def _compute_microbatch_range( - *, model: CompiledModelView, inputs: _MicrobatchRangeInputs -) -> CursorBounds | None: - """Compute the real overall cursor range for microbatch batch splitting.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - if materialized != MaterializationType.INCREMENTAL: - return None - incremental_mode: str | None = get_config_str( - values=model.config.values, key="incremental_mode" - ) - if incremental_mode != IncrementalMode.MICROBATCH: - return None - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - if cursor_column is None: - return None - bounded_override: CursorBounds | None = resolve_bounded_cursor_override( - start_cursor_override=inputs.start_cursor_override, - end_cursor_override=inputs.end_cursor_override, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - ) - if bounded_override is not None: - return _clamp_cursor_end(bounds=bounded_override, model=model) - if ( - get_config_str(values=model.config.values, key="microbatch_strategy") - == MicrobatchStrategy.ROLLING_WINDOW - ): - rolling_bounds: CursorBounds | None = _rolling_window_bounds( - model=model, invocation_time=inputs.invocation_time - ) - if inputs.full_refresh and rolling_bounds is not None: - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - if cursor_start is not None: - rolling_bounds = replace(rolling_bounds, start=cursor_start) - return rolling_bounds - if inputs.runtime_owned_cursor_bounds: - return None - cursor_snapshot: ModelCursorSnapshot | None = inputs.snapshot.cursor_snapshots.get(model.name) - if cursor_snapshot is None: - return None - if inputs.replaces_relation and not inputs.full_refresh: - cursor_snapshot = without_destination_cursor(cursor_snapshot=cursor_snapshot) - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - downstream_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - effective_grain: str | None = resolve_effective_timestamp_grain( - cursor_type=cursor_type, - downstream_grain=downstream_grain, - microbatch_strategy=get_config_str(values=model.config.values, key="microbatch_strategy"), - cursor_input_grains=tuple( - relation.cursor_grain for relation in inputs.cursor_input_relations - ), - ) - if cursor_type == CursorType.TIMESTAMP and effective_grain is not None: - cursor_snapshot = normalize_cursor_snapshot_grain( - cursor_snapshot=cursor_snapshot, - cursor_type=cursor_type, - effective_grain=effective_grain, - ) - - lookback: str | None = resolve_microbatch_lookback(model=model, effective_grain=effective_grain) - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - backfill_duration: str | None = None - if inputs.backfill.action == BackfillAction.BOUNDED: - backfill_duration = inputs.backfill.duration - - bounds: CursorBounds | None = compute_cursor_bounds( - cursor_snapshot=cursor_snapshot, - cursor_type=cursor_type, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - start_cursor_override=inputs.start_cursor_override, - end_cursor_override=inputs.end_cursor_override, - cursor_grain=effective_grain, - is_microbatch=False, - maximum_start_policy=MaximumStartPolicyInputs( - config=inputs.start_cursor_config, - invocation_time=inputs.invocation_time, - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=incremental_mode, - ), - ) - return _clamp_cursor_end( - bounds=_apply_future_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=effective_grain, - start_cursor_override=inputs.start_cursor_override, - end_cursor_override=inputs.end_cursor_override, - future_cursor_config=inputs.future_cursor_config, - invocation_time=inputs.invocation_time, - input_evidence=cursor_snapshot.input_evidence, - ), - model=model, - ) - - -def _compute_plan_cursor_bounds( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - backfill: BackfillResult, - full_refresh: bool, - start_cursor_override: str | None, - end_cursor_override: str | None, - runtime_owned_cursor_bounds: bool, - future_cursor_config: FutureCursorsConfig | None = None, - start_cursor_config: StartCursorsConfig | None = None, - invocation_time: datetime | None = None, -) -> CursorBounds | None: - """Compute cursor bounds for inclusion on the plan entry.""" - - materialized: str | None = get_config_str(values=model.config.values, key="materialized") - cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") - if materialized != MaterializationType.INCREMENTAL or cursor_column is None: - return None - bounded_override: CursorBounds | None = resolve_bounded_cursor_override( - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - cursor_grain=get_config_str(values=model.config.values, key="cursor_grain"), - ) - if bounded_override is not None: - return _clamp_cursor_end(bounds=bounded_override, model=model) - if runtime_owned_cursor_bounds: - return None - if full_refresh: - return None - - cursor_snapshot: ModelCursorSnapshot | None = snapshot.cursor_snapshots.get(model.name) - if cursor_snapshot is None: - return None - if backfill.action == BackfillAction.FULL: - cursor_snapshot = without_destination_cursor(cursor_snapshot=cursor_snapshot) - - lookback: str | None = get_config_str(values=model.config.values, key="lookback") - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - incremental_mode: str | None = get_config_str( - values=model.config.values, key="incremental_mode" - ) - is_microbatch: bool = incremental_mode == IncrementalMode.MICROBATCH - - backfill_duration: str | None = None - if backfill.action == BackfillAction.BOUNDED: - backfill_duration = backfill.duration - - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - cursor_grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - bounds: CursorBounds | None = compute_cursor_bounds( - cursor_snapshot=cursor_snapshot, - cursor_type=cursor_type, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - is_microbatch=is_microbatch, - cursor_grain=cursor_grain, - maximum_start_policy=MaximumStartPolicyInputs( - config=start_cursor_config, - invocation_time=invocation_time, - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=incremental_mode, - ), - ) - return _clamp_cursor_end( - bounds=_apply_future_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - start_cursor_override=start_cursor_override, - end_cursor_override=end_cursor_override, - future_cursor_config=future_cursor_config, - invocation_time=invocation_time, - input_evidence=cursor_snapshot.input_evidence, - ), - model=model, - ) - - -def _resolve_empty_cursor_inputs( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - full_refresh: bool, - runtime_owned_cursor_bounds: bool, - cursor_overrides: CursorOverridePair, -) -> tuple[tuple[str, ...], bool]: - """Return empty inputs that leave a planner-owned window empty, and whether others have rows.""" - - if ( - full_refresh - or runtime_owned_cursor_bounds - or get_config_str(values=model.config.values, key="materialized") - != MaterializationType.INCREMENTAL - or get_config_str(values=model.config.values, key="cursor") is None - or get_config_str(values=model.config.values, key="microbatch_strategy") - == MicrobatchStrategy.ROLLING_WINDOW - or ( - cursor_overrides.start_cursor_override is not None - and cursor_overrides.end_cursor_override is not None - ) - ): - return (), False - cursor_snapshot: ModelCursorSnapshot | None = snapshot.cursor_snapshots.get(model.name) - if cursor_snapshot is None: - return (), False - decision: EmptyCursorInputDecision = cursor_inputs_without_rows(cursor_snapshot=cursor_snapshot) - return decision.input_names, decision.waiting_on_empty_inputs - - -def _apply_future_safety( - *, - bounds: CursorBounds | None, - cursor_type: str | None, - cursor_grain: str | None, - start_cursor_override: str | None, - end_cursor_override: str | None, - future_cursor_config: FutureCursorsConfig | None, - invocation_time: datetime | None, - input_evidence: tuple[CursorInputEvidence, ...], -) -> CursorBounds | None: - if bounds is None: - return None - if bounds.end == BoundSentinel.END: - return bounds - return apply_future_cursor_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - config=future_cursor_config, - invocation_time=invocation_time, - has_complete_override=( - start_cursor_override is not None and end_cursor_override is not None - ), - input_evidence=input_evidence, - ) - - -def _append_future_cap_warning( - *, warnings: tuple[PlanWarning, ...], model_name: str, bounds: CursorBounds | None -) -> tuple[PlanWarning, ...]: - message: str | None = future_cursor_cap_warning(bounds) - if message is None: - return warnings - return ( - *warnings, - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - code="FUTURE_CURSOR_CAPPED", - message=message, - ), - ) - - -def _append_cursor_safety_warnings( - *, warnings: tuple[PlanWarning, ...], model_name: str, bounds: CursorBounds | None -) -> tuple[PlanWarning, ...]: - with_future: tuple[PlanWarning, ...] = _append_future_cap_warning( - warnings=warnings, model_name=model_name, bounds=bounds - ) - return _append_maximum_start_warning(warnings=with_future, model_name=model_name, bounds=bounds) - - -def _append_maximum_start_warning( - *, warnings: tuple[PlanWarning, ...], model_name: str, bounds: CursorBounds | None -) -> tuple[PlanWarning, ...]: - message: str | None = maximum_start_cap_warning(bounds) - if message is None: - return warnings - return ( - *warnings, - PlanWarning( - model_name=model_name, - severity=WarningSeverity.WARNING, - code="AUTOMATIC_START_CAPPED", - message=message, - ), - ) - - -def scope_overlaps( - *, - scope_deps: tuple[CompiledObjectKey, ...], - selected_keys: frozenset[CompiledObjectKey], -) -> bool: - """Check if any scope dependency is in the selected keys.""" - - dep: CompiledObjectKey - for dep in scope_deps: - if dep in selected_keys: - return True - return False - - -def _selected_audit_relation_keys( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], -) -> frozenset[CompiledObjectKey]: - """Return relation keys needed to render selected audits without selecting those relations.""" - - relation_keys: set[CompiledObjectKey] = set() - audit: CompiledAuditView - for audit in project.audits: - if audit_is_selected(audit=audit, selected_keys=selected_keys): - relation_keys.update(audit.scope_deps) - return frozenset(relation_keys) - - -def build_model_materializations( - model_entries: tuple[ModelPlanEntry, ...], -) -> dict[str, str]: - """Build a name-to-materialization-type lookup from planned model entries.""" - - return {entry.name: entry.materialization_type for entry in model_entries} - - -def _rolling_window_bounds( - *, model: CompiledModelView, invocation_time: datetime | None -) -> CursorBounds | None: - grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") - lookback: str | None = resolve_microbatch_lookback(model=model, effective_grain=grain) - duration: Duration | None = Duration.parse(lookback or "") - if grain is None or duration is None: - return None - moment: datetime = invocation_time or datetime.now(tz=UTC) - if moment.tzinfo is None: - moment = moment.replace(tzinfo=UTC) - else: - moment = moment.astimezone(UTC) - anchor: datetime = _floor_utc_grain(moment=moment, grain=grain) - bounds: CursorBounds = CursorBounds( - start=TimestampValue(value=duration.subtract_from(anchor)), - end=TimestampValue(value=anchor), - ) - cursor_start: str | None = get_config_cursor_bound( - values=model.config.values, key="cursor_start" - ) - if cursor_start is not None and _cursor_order_key( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=CursorType.TIMESTAMP - ) < _cursor_order_key(value=cursor_start, cursor_type=CursorType.TIMESTAMP): - bounds = replace(bounds, start=cursor_start) - return _clamp_cursor_end(bounds=bounds, model=model) - - -def _floor_utc_grain(*, moment: datetime, grain: str) -> datetime: - if grain == CursorGrain.YEAR: - return moment.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0) - if grain == CursorGrain.MONTH: - return moment.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - if grain == CursorGrain.DAY: - return moment.replace(hour=0, minute=0, second=0, microsecond=0) - if grain == CursorGrain.HOUR: - return moment.replace(minute=0, second=0, microsecond=0) - if grain == CursorGrain.MINUTE: - return moment.replace(second=0, microsecond=0) - return moment.replace(microsecond=0) - - -def _clamp_cursor_end( - *, bounds: CursorBounds | None, model: CompiledModelView -) -> CursorBounds | None: - cursor_end: str | None = get_config_cursor_bound(values=model.config.values, key="cursor_end") - cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") - if bounds is None or cursor_end is None: - return bounds - if bounds.start == BoundSentinel.START or bounds.end == BoundSentinel.END: - return bounds - if _cursor_order_key( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type - ) >= _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - return replace(bounds, start=cursor_end, end=cursor_end) - if _cursor_order_key( - value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type - ) > _cursor_order_key(value=cursor_end, cursor_type=cursor_type): - return replace(bounds, end=cursor_end) - return bounds - - -def _cursor_order_key(*, value: str, cursor_type: str | None) -> Decimal: - if cursor_type == CursorType.INTEGER: - return Decimal(value) - parsed: datetime = datetime.fromisoformat(value) - if parsed.tzinfo is not None: - parsed = parsed.astimezone(UTC).replace(tzinfo=None) - return Decimal(str(parsed.replace(tzinfo=UTC).timestamp())) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index 850550e264..026081c888 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -3,11 +3,14 @@ from __future__ import annotations from sqlbuild._native import ( + build_model_materializations, build_relation_fixture_context, build_selected_audit_entries, build_source_load_entries, extract_seed_columns, plan_sql_tests, + resolve_function_sql, + scope_overlaps, upstream_loader_dependency_names, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -27,11 +30,6 @@ PythonHookEntry, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - build_model_materializations, - scope_overlaps, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_function_sql from sqlbuild.compiler.planner.exceptions import ( PlannerInputError, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/__init__.py b/src/sqlbuild/compiler/planner/_helpers/resolve/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py b/src/sqlbuild/compiler/planner/_helpers/resolve/config.py deleted file mode 100644 index 26698e7dcc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Model config extraction helpers for resolve modules.""" - -from __future__ import annotations - -from sqlbuild._native import get_config_str -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.types import IncrementalStrategy - - -def get_config_append_cursor_inclusive(model: CompiledModelView) -> bool: - """Return effective append cursor lower-bound inclusivity.""" - - strategy: str | None = get_config_str(values=model.config.values, key="incremental_strategy") - if strategy != IncrementalStrategy.APPEND: - return True - raw: object | None = model.config.values.get("append_cursor_inclusive") - if isinstance(raw, bool): - return raw - return True diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py deleted file mode 100644 index cfa1fa0d30..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Native cursor override typing and cursor-window exports.""" - -from sqlbuild._native import ( - apply_typed_cursor_replay_policy as apply_typed_cursor_replay_policy, -) -from sqlbuild._native import ( - compute_cursor_bounds as compute_cursor_bounds, -) -from sqlbuild._native import ( - cursor_inputs_without_rows as cursor_inputs_without_rows, -) -from sqlbuild._native import ( - cursor_window_override_flags as cursor_window_override_flags, -) -from sqlbuild._native import ( - decide_empty_cursor_inputs as decide_empty_cursor_inputs, -) -from sqlbuild._native import ( - empty_cursor_window as empty_cursor_window, -) -from sqlbuild._native import ( - empty_input_rebuild_refusal as empty_input_rebuild_refusal, -) -from sqlbuild._native import ( - normalize_cursor_snapshot_grain as normalize_cursor_snapshot_grain, -) -from sqlbuild._native import ( - resolve_effective_timestamp_grain as resolve_effective_timestamp_grain, -) -from sqlbuild._native import ( - type_cursor_overrides as type_cursor_overrides, -) -from sqlbuild._native import ( - unreadable_cursor_inputs_error as unreadable_cursor_inputs_error, -) -from sqlbuild._native import ( - without_destination_cursor as without_destination_cursor, -) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py deleted file mode 100644 index b819708b61..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Render cursor intrinsics with one interval's adapter-typed bound literals.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import resolve_cursor_intrinsics -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import CursorBounds - - -def render_cursor_intrinsic_bounds( - *, sql: str, bounds: CursorBounds, cursor_type: str | None, adapter: BaseAdapter -) -> str: - """Replace `__cursor_start()` / `__cursor_end()` with the adapter's typed bound literals.""" - - rendered, _ = resolve_cursor_intrinsics( - sql=sql, - start_sql=adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type - ), - end_sql=adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type - ), - ) - return rendered diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py deleted file mode 100644 index f1885743da..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Resolve model-level cursor policy overrides.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.spec.contracts.constants import CURSOR_POLICY_DISABLED -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction - - -def resolve_start_cursor_config( - *, model: CompiledModelView, project_config: StartCursorsConfig | None -) -> StartCursorsConfig | None: - """Resolve automatic-start policy with model config taking precedence.""" - - configured: StartCursorsConfig = project_config or StartCursorsConfig() - raw_distance: object | None = model.config.values.get("cursor_start_max_ahead") - raw_action: object | None = model.config.values.get("cursor_start_max_action") - max_ahead: str | None = configured.max_ahead - if isinstance(raw_distance, str): - max_ahead = None if raw_distance == CURSOR_POLICY_DISABLED else raw_distance - action: FutureCursorAction = configured.action - if isinstance(raw_action, str): - action = FutureCursorAction(raw_action) - return StartCursorsConfig(max_ahead=max_ahead, action=action) if max_ahead is not None else None - - -def resolve_future_cursor_config( - *, model: CompiledModelView, project_config: FutureCursorsConfig | None -) -> FutureCursorsConfig | None: - """Resolve future-end policy with model config taking precedence.""" - - configured: FutureCursorsConfig = project_config or FutureCursorsConfig() - raw_distance: object | None = model.config.values.get("cursor_future_max_distance") - raw_action: object | None = model.config.values.get("cursor_future_action") - max_distance: str | None = configured.max_distance - if isinstance(raw_distance, str): - max_distance = None if raw_distance == CURSOR_POLICY_DISABLED else raw_distance - action: FutureCursorAction = configured.action - if isinstance(raw_action, str): - action = FutureCursorAction(raw_action) - return ( - FutureCursorsConfig(max_distance=max_distance, action=action) - if max_distance is not None - else None - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py b/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py deleted file mode 100644 index ac87304474..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/future_cursor_safety.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Native future cursor safety policy.""" - -from sqlbuild._native import resolve_future_cursor_safety as resolve_future_cursor_safety diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py b/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py deleted file mode 100644 index 267593051d..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/maximum_start.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Native maximum automatic cursor-start eligibility policy.""" - -from sqlbuild._native import apply_maximum_start_policy as apply_maximum_start_policy -from sqlbuild._native import maximum_allowed_start as maximum_allowed_start diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py deleted file mode 100644 index 8ad3e3c0e5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Native reference resolution and generic executable-SQL callback support.""" - -from __future__ import annotations - -import re -from collections.abc import Callable, Iterator - -import sqlbuild._native as _native -from sqlbuild._native import apply_deferred_locations as apply_deferred_locations -from sqlbuild._native import build_function_locations as build_function_locations -from sqlbuild._native import build_model_locations as build_model_locations -from sqlbuild._native import build_seed_locations as build_seed_locations -from sqlbuild._native import resolve_dbt_ref_references as resolve_dbt_ref_references -from sqlbuild._native import resolve_ref_references as resolve_ref_references -from sqlbuild._native import resolve_table_function_references as resolve_table_function_references -from sqlbuild._native import resolve_udf_references as resolve_udf_references -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -_REFERENCE_SCAN_CONTEXT: str = "SQL reference resolution" - - -def replace_executable_matches( - *, - sql: str, - pattern: re.Pattern[str], - replace: Callable[[re.Match[str]], str], - lexical_syntax: SqlLexicalSyntax, -) -> str: - """Replace pattern matches in executable SQL, leaving quoted text and comments untouched.""" - - parts: list[str] = [] - last_index: int = 0 - for match in _iter_code_matches( - sql=sql, - pattern=pattern, - scanner=_native.SqlCodeScanner(sql, lexical_syntax.native_mapping, _REFERENCE_SCAN_CONTEXT), - ): - parts.append(sql[last_index : match.start()]) - parts.append(replace(match)) - last_index = match.end() - parts.append(sql[last_index:]) - return "".join(parts) - - -def _iter_code_matches( - *, sql: str, pattern: re.Pattern[str], scanner: _native.SqlCodeScanner -) -> Iterator[re.Match[str]]: - index: int = 0 - while index < len(sql): - non_code_end: int | None = scanner.non_code_end(index) - if non_code_end is not None: - index = non_code_end - continue - match: re.Match[str] | None = pattern.match(sql, index) - if match is None: - index += 1 - continue - yield match - index = match.end() diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py deleted file mode 100644 index 2802c8ba56..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Native model, function and hook SQL resolution exports.""" - -from sqlbuild._native import resolve_function_sql as resolve_function_sql -from sqlbuild._native import resolve_model_hook_entries as resolve_model_hook_entries -from sqlbuild._native import resolve_model_sql as resolve_model_sql diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py b/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py deleted file mode 100644 index b285a5aa70..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/sources.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Native source resolution with adapter rendering callbacks.""" - -from sqlbuild._native import render_source_read_relation as render_source_read_relation -from sqlbuild._native import resolve_source_references as resolve_source_references diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index 18cbcfb135..782aa273c7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -8,9 +8,14 @@ build_compiled_function_fingerprint_sql, build_downstream_deps, build_execution_upstream_deps, + build_function_locations, + build_model_locations, + build_seed_locations, gather_source_columns, get_materialization_type, + resolve_function_sql, resolve_graph_selection, + resolve_model_sql, topologically_order_keys, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -24,15 +29,6 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - build_function_locations, - build_model_locations, - build_seed_locations, -) -from sqlbuild.compiler.planner._helpers.resolve.resolve import ( - resolve_function_sql, - resolve_model_sql, -) from sqlbuild.compiler.planner.models import ( BackfillResult, CloneSourcePlanEntry, diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/__init__.py b/src/sqlbuild/compiler/planner/main/cursor_window/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py b/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py deleted file mode 100644 index 68493db0bd..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public typing of untyped `--start-cursor`/`--end-cursor` values.""" - -from __future__ import annotations - -from collections.abc import Iterable - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - type_cursor_overrides, -) -from sqlbuild.compiler.planner.models import CursorOverrides - - -def typed_cursor_overrides( - *, - cursor_overrides: CursorOverrides | None, - selected_models: Iterable[CompiledModelView] = (), -) -> CursorOverrides | None: - """Read `--start-cursor`/`--end-cursor` as the one cursor type of the selected models.""" - - return type_cursor_overrides(cursor_overrides=cursor_overrides, selected_models=selected_models) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py b/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py deleted file mode 100644 index 132826c442..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/cursor_window_override_flags.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public naming of the CLI flags that set a cursor window explicitly.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - cursor_window_override_flags as _cursor_window_override_flags, -) - - -def cursor_window_override_flags(*, cursor_type: str | None) -> str: - """Name the CLI flags that set a cursor window explicitly for this cursor type.""" - - return _cursor_window_override_flags(cursor_type=cursor_type) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py deleted file mode 100644 index 543483c310..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_input_decision.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public decision on whether empty cursor inputs leave a model with no window.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - decide_empty_cursor_inputs as _decide_empty_cursor_inputs, -) -from sqlbuild.compiler.planner.models import EmptyCursorInputDecision - - -def decide_empty_cursor_inputs( - *, - empty_input_names: tuple[str, ...], - waits_for_every_input: bool, - inputs_with_rows: bool, - has_window_start: bool, -) -> EmptyCursorInputDecision: - """Decide whether empty cursor inputs leave no window, shared by planning and runtime.""" - - return _decide_empty_cursor_inputs( - empty_input_names=empty_input_names, - waits_for_every_input=waits_for_every_input, - inputs_with_rows=inputs_with_rows, - has_window_start=has_window_start, - ) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py deleted file mode 100644 index 4cc7058cf2..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/empty_cursor_window.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public zero-width cursor window entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - empty_cursor_window as _empty_cursor_window, -) -from sqlbuild.compiler.planner.models import CursorBounds - - -def empty_cursor_window(*, cursor_type: str | None, cursor_start: str | None) -> CursorBounds: - """Return a zero-width cursor window, which selects no rows from any filtered input.""" - - return _empty_cursor_window(cursor_type=cursor_type, cursor_start=cursor_start) diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py b/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py deleted file mode 100644 index fae1a78827..0000000000 --- a/src/sqlbuild/compiler/planner/main/cursor_window/empty_input_rebuild_refusal.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public wording for refusing to rebuild an existing table from empty cursor inputs.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - empty_input_rebuild_refusal as _empty_input_rebuild_refusal, -) - - -def empty_input_rebuild_refusal( - *, model_name: str, input_names: tuple[str, ...], cursor_type: str | None -) -> tuple[str, str]: - """Return the message and help for refusing to rebuild an existing table from empty inputs.""" - - return _empty_input_rebuild_refusal( - model_name=model_name, input_names=input_names, cursor_type=cursor_type - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py b/src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py deleted file mode 100644 index b239ea88bc..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/cursor_replay_policy.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Public typed cursor replay policy entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - apply_typed_cursor_replay_policy as _apply_typed_cursor_replay_policy, -) -from sqlbuild.compiler.planner.models import Duration -from sqlbuild.cursor_algebra.types import CursorScalar - - -def apply_typed_cursor_replay_policy( - *, - start: CursorScalar, - end: CursorScalar, - cursor_start: CursorScalar | str | None, - cursor_type: str | None, - lookback: Duration | str | None, - backfill_duration: Duration | str | None, - has_start_override: bool, -) -> CursorScalar: - """Apply replay policy without crossing a string serialization boundary.""" - - return _apply_typed_cursor_replay_policy( - start=start, - end=end, - cursor_start=cursor_start, - cursor_type=cursor_type, - lookback=lookback, - backfill_duration=backfill_duration, - has_start_override=has_start_override, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py index 637548b5ff..1df559aa82 100644 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ b/src/sqlbuild/compiler/planner/main/execution/execution.py @@ -20,6 +20,7 @@ reconcile_execution_changes, resolve_model_actions, resolve_planner_scopes, + type_cursor_overrides, with_plan_metadata, with_plan_warnings, with_storage_policies, @@ -31,9 +32,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - type_cursor_overrides, -) from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( BackgroundSqlTestPlanning, ) diff --git a/src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py b/src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py deleted file mode 100644 index a4458d654d..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/future_cursor_safety.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Public future cursor safety entry point.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.planner._helpers.resolve.future_cursor_safety import ( - resolve_future_cursor_safety, -) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputEvidence, -) -from sqlbuild.spec.contracts.models import FutureCursorsConfig - - -def apply_future_cursor_safety( - *, - bounds: CursorBounds, - cursor_type: str | None, - cursor_grain: str | None, - config: FutureCursorsConfig | None, - invocation_time: datetime | None, - has_complete_override: bool, - input_evidence: tuple[CursorInputEvidence, ...] = (), -) -> CursorBounds: - """Apply configured future safety to effective cursor bounds.""" - - return resolve_future_cursor_safety( - bounds=bounds, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - config=config, - invocation_time=invocation_time, - has_complete_override=has_complete_override, - input_evidence=input_evidence, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py b/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py deleted file mode 100644 index 6bd19aadcb..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/future_cursor_warning.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public future cursor warning rendering entry point.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.models import CursorBounds, FutureCursorSafetyEvidence - - -def future_cursor_cap_warning(bounds: CursorBounds | None) -> str | None: - """Render a prominent warning for applied future cursor policy.""" - - if bounds is None or bounds.future_safety is None: - return None - evidence: FutureCursorSafetyEvidence = bounds.future_safety - return ( - "FUTURE CURSOR CAPPED: discovered bounds " - f"['{render(value=evidence.discovered_start)}', " - f"'{render(value=evidence.discovered_end)}'), applied bounds " - f"['{render(value=evidence.applied_start)}', '{render(value=evidence.applied_end)}')." - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py b/src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py deleted file mode 100644 index 8507b0f775..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/maximum_start_safety.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Public maximum automatic-start safety entry points.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.resolve.maximum_start import ( - apply_maximum_start_policy as _apply_maximum_start_policy, -) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - Duration, - MaximumStartPolicyInputs, - ModelCursorSnapshot, -) -from sqlbuild.cursor_algebra.types import CursorScalar - - -def apply_maximum_start_policy( - *, - bounds: CursorBounds, - snapshot: ModelCursorSnapshot, - cursor_type: str | None, - cursor_grain: str | None, - cursor_start: CursorScalar | str | None, - lookback: Duration | str | None, - backfill_duration: Duration | str | None, - policy: MaximumStartPolicyInputs, - has_start_override: bool, -) -> CursorBounds: - """Apply maximum automatic-start safety to cursor bounds.""" - - return _apply_maximum_start_policy( - bounds=bounds, - snapshot=snapshot, - cursor_type=cursor_type, - cursor_grain=cursor_grain, - cursor_start=cursor_start, - lookback=lookback, - backfill_duration=backfill_duration, - policy=policy, - has_start_override=has_start_override, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py b/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py deleted file mode 100644 index 061ec067fd..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/maximum_start_warning.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public maximum automatic-start warning rendering entry point.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.models import CursorBounds, MaximumStartSafetyEvidence -from sqlbuild.cursor_algebra.types import CursorScalar - - -def maximum_start_cap_warning(bounds: CursorBounds | None) -> str | None: - """Render a warning that distinguishes start recovery from future-end capping.""" - - if bounds is None or bounds.maximum_start_safety is None: - return None - evidence: MaximumStartSafetyEvidence = bounds.maximum_start_safety - return ( - "AUTOMATIC START CAPPED: physical target MAX " - f"'{render(value=evidence.physical_target_max)}' exceeded horizon " - f"'{render(value=evidence.maximum_allowed_start)}'; highest eligible target MAX=" - f"{_render_optional_repr(evidence.highest_eligible_target_max)}, " - "effective post-lookback start=" - f"'{render(value=evidence.effective_start)}', action={evidence.action.value}, " - f"input={evidence.target_relation}.{evidence.cursor_column}." - ) - - -def _render_optional_repr(value: CursorScalar | None) -> str: - if value is None: - return "None" - return repr(render(value=value)) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index c7061b769f..9b0d288808 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import future_cursor_cap_warning, maximum_start_cap_warning from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -17,8 +18,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.compiler.planner.types import IncrementalStrategy, OnSchemaChange from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 8d0eba4ad8..2fdb534834 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -15,7 +15,10 @@ aligned_timestamp_bounds, compute_query_hash, cursor_bound_display, + empty_cursor_window, + future_cursor_cap_warning, inclusive_cursor_end, + maximum_start_cap_warning, microbatch_limit_warning, resolve_effective_microbatch_batch_size, ) @@ -42,9 +45,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning -from sqlbuild.compiler.planner.main.execution.maximum_start_warning import maximum_start_cap_warning from sqlbuild.compiler.planner.models import ( CursorBounds, Duration, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index 79b37bd434..d8eb5b41a5 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -4,12 +4,12 @@ from dataclasses import replace -from sqlbuild._native import replaces_incremental_relation -from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_window import empty_cursor_window -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import ( +from sqlbuild._native import ( + empty_cursor_window, future_cursor_cap_warning, + replaces_incremental_relation, ) +from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index e611499f86..edd3b81662 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -8,6 +8,14 @@ from functools import partial from typing import Any +from sqlbuild._native import ( + apply_maximum_start_policy, + apply_typed_cursor_replay_policy, + cursor_window_override_flags, + decide_empty_cursor_inputs, + empty_input_rebuild_refusal, + resolve_cursor_intrinsics, +) from sqlbuild._native import cursor_clamp as clamp from sqlbuild._native import cursor_compare as compare from sqlbuild._native import ( @@ -22,24 +30,10 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import resolve_cursor_intrinsics +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.main.cursor_window.cursor_window_override_flags import ( - cursor_window_override_flags, -) -from sqlbuild.compiler.planner.main.cursor_window.empty_cursor_input_decision import ( - decide_empty_cursor_inputs, -) -from sqlbuild.compiler.planner.main.cursor_window.empty_input_rebuild_refusal import ( - empty_input_rebuild_refusal, -) -from sqlbuild.compiler.planner.main.execution.cursor_replay_policy import ( - apply_typed_cursor_replay_policy, -) -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.main.execution.maximum_start_safety import apply_maximum_start_policy from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputEvidence, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 65e2308310..e5e227e68a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -6,7 +6,12 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + resolve_model_sql, +) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -22,7 +27,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.models import ( BackfillResult, CursorOverridePair, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py index 13cd341c29..1b1ca53e1b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner._helpers.resolve.sources import resolve_source_references from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterOverrideTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index e89b3c5ddc..b587e45dff 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds from sqlbuild.compiler.planner.models import ( BackfillResult, CursorBounds, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index f67198973a..bf30c933fc 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -16,6 +16,7 @@ expand_sql_macros, infer_native_query_columns, project_lineage_views, + resolve_ref_references, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -63,7 +64,6 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 0ad9a64bbe..8e08025218 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -4,13 +4,13 @@ import pytest -from sqlbuild._native import watermark_type_is_compatible -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileModelConfig -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( +from sqlbuild._native import ( compute_cursor_bounds, resolve_effective_timestamp_grain, + watermark_type_is_compatible, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 6b0e9aed81..6272cc9b6d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import assemble_project +from sqlbuild._native import assemble_project, scope_overlaps from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -20,7 +20,6 @@ CompileInputsView, TypedSqlValueRenderer, ) -from sqlbuild.compiler.planner._helpers.output.plan_entry import scope_overlaps from sqlbuild.sql_values.models import SqlLogicalType, SqlValue from sqlbuild.sql_values.types import SqlValueKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 3a41b522c7..2aac608caf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -7,13 +7,10 @@ import pytest +from sqlbuild._native import apply_typed_cursor_replay_policy, compute_cursor_bounds from sqlbuild._native import cursor_observed_partition as observed_partition from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner._helpers.resolve.cursor import ( - apply_typed_cursor_replay_policy, - compute_cursor_bounds, -) from sqlbuild.compiler.planner.constants import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 1ac4c873f5..4a6a6ee82c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -2,9 +2,9 @@ import pytest +from sqlbuild._native import resolve_model_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( BackfillResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py index 3bef326593..19522990df 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.resolve.cursor import type_cursor_overrides +from sqlbuild._native import type_cursor_overrides from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index c71184bd0b..14311bc6e5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -6,10 +6,10 @@ import pytest +from sqlbuild._native import resolve_model_hook_entries from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_hook_entries from sqlbuild.compiler.planner.models import ModelPlanContext from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 95886abb32..e790e6ecb7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -4,7 +4,15 @@ import pytest -from sqlbuild._native import DbtCompileReferenceResolver, build_dbt_manifest_index +from sqlbuild._native import ( + DbtCompileReferenceResolver, + apply_deferred_locations, + build_dbt_manifest_index, + resolve_dbt_ref_references, + resolve_ref_references, + resolve_table_function_references, + resolve_udf_references, +) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -12,13 +20,6 @@ CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.resolve.refs import ( - apply_deferred_locations, - resolve_dbt_ref_references, - resolve_ref_references, - resolve_table_function_references, - resolve_udf_references, -) from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.integrations.dbt.models import DbtManifestIndex from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py index f77d96815e..62a2c40c5c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -11,9 +12,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner._helpers.resolve.sources import ( - resolve_source_references, -) from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py index 41cd712f21..7ad94b1446 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import resolve_cursor_overrides from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner._helpers.output.plan_entry import resolve_cursor_overrides from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorOverrideResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py index 55d33f98f9..351f03d67d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorStartBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index d163308c0a..1616717d3f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import compute_microbatch_range, compute_plan_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -15,11 +16,6 @@ CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - _compute_microbatch_range, - _compute_plan_cursor_bounds, - _MicrobatchRangeInputs, -) from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.compiler.planner.models import ( BackfillResult, @@ -86,7 +82,7 @@ def test_given_planner_owned_microbatch_with_cursor_end_when_planning_then_senti ) backfill: BackfillResult = BackfillResult(action=BackfillAction.FORWARD_ONLY) - plan_bounds: CursorBounds | None = _compute_plan_cursor_bounds( + plan_bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=snapshot, backfill=backfill, @@ -95,22 +91,20 @@ def test_given_planner_owned_microbatch_with_cursor_end_when_planning_then_senti end_cursor_override=None, runtime_owned_cursor_bounds=False, ) - microbatch_range: CursorBounds | None = _compute_microbatch_range( + microbatch_range: CursorBounds | None = compute_microbatch_range( model=model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=backfill, - start_cursor_override=None, - end_cursor_override=None, - runtime_owned_cursor_bounds=False, - cursor_input_relations=( - CursorInputRelation(relation="main.raw_orders", cursor_column="cursor_value"), - ), - future_cursor_config=None, - start_cursor_config=None, - invocation_time=None, - full_refresh=False, + snapshot=snapshot, + backfill=backfill, + start_cursor_override=None, + end_cursor_override=None, + runtime_owned_cursor_bounds=False, + cursor_input_relations=( + CursorInputRelation(relation="main.raw_orders", cursor_column="cursor_value"), ), + future_cursor_config=None, + start_cursor_config=None, + invocation_time=None, + full_refresh=False, ) assert plan_bounds == CursorBounds( @@ -161,7 +155,7 @@ def test_given_explicit_cursor_overrides_without_snapshot_when_planning_then_use ), ) - bounds: CursorBounds | None = _compute_plan_cursor_bounds( + bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=WarehouseSnapshot(), backfill=BackfillResult(action=BackfillAction.BOUNDED, duration="7d"), @@ -257,7 +251,7 @@ def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_r } ), ) - plan_bounds: CursorBounds | None = _compute_plan_cursor_bounds( + plan_bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=snapshot, backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), @@ -266,20 +260,18 @@ def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_r end_cursor_override=test_case.end_override, runtime_owned_cursor_bounds=True, ) - microbatch_range: CursorBounds | None = _compute_microbatch_range( + microbatch_range: CursorBounds | None = compute_microbatch_range( model=microbatch_model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - start_cursor_override=test_case.start_override, - end_cursor_override=test_case.end_override, - runtime_owned_cursor_bounds=True, - cursor_input_relations=(), - future_cursor_config=None, - start_cursor_config=None, - invocation_time=None, - full_refresh=False, - ), + snapshot=snapshot, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + start_cursor_override=test_case.start_override, + end_cursor_override=test_case.end_override, + runtime_owned_cursor_bounds=True, + cursor_input_relations=(), + future_cursor_config=None, + start_cursor_config=None, + invocation_time=None, + full_refresh=False, ) assert plan_bounds == test_case.expected_bounds @@ -355,28 +347,26 @@ def test_given_planner_owned_model_backed_input_when_grain_is_coarser_then_range } ) - microbatch_range: CursorBounds | None = _compute_microbatch_range( + microbatch_range: CursorBounds | None = compute_microbatch_range( model=model, - inputs=_MicrobatchRangeInputs( - snapshot=snapshot, - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - start_cursor_override=None, - end_cursor_override=None, - runtime_owned_cursor_bounds=False, - cursor_input_relations=( - CursorInputRelation( - relation="main.daily_rollup", - cursor_column="activity_day", - cursor_grain=test_case.upstream_grain, - is_model_backed=True, - is_runtime_produced=False, - ), + snapshot=snapshot, + backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + start_cursor_override=None, + end_cursor_override=None, + runtime_owned_cursor_bounds=False, + cursor_input_relations=( + CursorInputRelation( + relation="main.daily_rollup", + cursor_column="activity_day", + cursor_grain=test_case.upstream_grain, + is_model_backed=True, + is_runtime_produced=False, ), - future_cursor_config=None, - start_cursor_config=None, - invocation_time=None, - full_refresh=False, ), + future_cursor_config=None, + start_cursor_config=None, + invocation_time=None, + full_refresh=False, ) assert microbatch_range == test_case.expected_bounds @@ -421,7 +411,7 @@ def test_given_snapshot_cursor_grain_when_planning_then_dml_bound_matches_query_ ), ) - bounds: CursorBounds | None = _compute_plan_cursor_bounds( + bounds: CursorBounds | None = compute_plan_cursor_bounds( model=model, snapshot=WarehouseSnapshot( cursor_snapshots={ diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index ca6736d1da..486e3bba2e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( build_cursor_input_relations, + build_planner_relations_context, gather_source_columns, runtime_cursor_producer_names, validate_source_cursor_input_columns, @@ -25,7 +26,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.output.plan_entry import build_planner_relations_context from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorInputRelation, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py index 12f9cc2855..62d897ec76 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py @@ -5,9 +5,9 @@ import pytest from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import future_cursor_cap_warning +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety -from sqlbuild.compiler.planner.main.execution.future_cursor_warning import future_cursor_cap_warning from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.spec.contracts.models import FutureCursorsConfig diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py index 34b2183d86..6c10df7d30 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py @@ -5,10 +5,10 @@ import pytest +from sqlbuild._native import compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner._helpers.resolve.cursor import compute_cursor_bounds +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError -from sqlbuild.compiler.planner.main.execution.future_cursor_safety import apply_future_cursor_safety from sqlbuild.compiler.planner.models import ( CursorBounds, MaximumStartPolicyInputs, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index cc01478c10..9c27ea3d76 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -10,11 +10,11 @@ from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import normalize_cursor_snapshot_grain from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.resolve.cursor import normalize_cursor_snapshot_grain from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError from sqlbuild.compiler.planner.models import ( CursorBounds, From 8e00d6c5fc18f5fb5161de33be3fd4e39ccf9ecc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:09:47 +0100 Subject: [PATCH 02/33] refactor: host plan output and static command plans natively Port PlanOutput assembly, SQL test selection and planning, function entries, static audit/test/seed/relation command plans and direct seed fingerprint reads to native code, deleting the Python plan_output, command_planning and main/commands modules. --- .../_helpers/audit_command/command.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 2 +- .../compile_driver/selection_phases.rs | 4 +- .../_helpers/planner_entries/command_plans.rs | 470 ++++++++++ .../bindings/_helpers/planner_entries/mod.rs | 2 + .../_helpers/planner_entries/plan_output.rs | 844 ++++++++++++++++++ .../_helpers/planner_entries/registration.rs | 2 + .../_helpers/planner_entries/values.rs | 1 + .../_helpers/planner_phases/constants.rs | 1 - .../planner_phases/output_assembly.rs | 4 +- .../planner_registration/registration.rs | 5 +- .../bindings/_helpers/seed_command/command.rs | 4 +- .../_helpers/sql_test_command/test_command.rs | 2 +- src/sqlbuild/_native.pyi | 94 ++ .../_helpers/command_planning/__init__.py | 1 - .../_helpers/command_planning/seed_state.py | 87 -- .../_helpers/command_planning/static.py | 257 ------ .../planner/_helpers/output/plan_output.py | 498 ----------- .../classes/background_sql_test_planning.py | 27 +- .../planner/main/commands/__init__.py | 1 - .../compiler/planner/main/commands/_audit.py | 20 - .../planner/main/commands/_relation_plan.py | 15 - .../planner/main/commands/_relations.py | 33 - .../compiler/planner/main/commands/_scope.py | 17 - .../compiler/planner/main/commands/_seed.py | 20 - .../planner/main/commands/_seed_state.py | 16 - .../planner/main/commands/_sql_test.py | 25 - .../planner/_helpers/test_fixture_planning.py | 2 +- .../sql_test_assembly/test_plan_test.py | 6 +- .../_helpers/test_fixture_planning_work.py | 5 +- .../test_background_sql_test_planning.py | 10 +- .../main/test_static_command_planning.py | 12 +- 32 files changed, 1444 insertions(+), 1045 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/command_planning/static.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/output/plan_output.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_audit.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_relation_plan.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_relations.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_scope.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_seed.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_seed_state.py delete mode 100644 src/sqlbuild/compiler/planner/main/commands/_sql_test.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index c6c24751b6..5a4d0ed19c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -19,7 +19,7 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; +const AUDIT_PLAN_MODULE: &str = "sqlbuild._native"; const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const PROJECTION_MODULE: &str = "sqlbuild.executor.auditing.main._current_audit_result_projection"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index b40ea8a85c..aec20540f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -31,7 +31,7 @@ const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; +const RELATION_PLAN_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 5234f47d92..8167475805 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -13,8 +13,8 @@ const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const SPLIT_SELECTORS_MODULE: &str = "sqlbuild._native"; const PLANNER_SCOPES_MODULE: &str = "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; -const STATIC_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._scope"; -const STATIC_RELATIONS_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relations"; +const STATIC_SCOPE_MODULE: &str = "sqlbuild._native"; +const STATIC_RELATIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; /// Whether a selection names nothing to select or exclude. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs new file mode 100644 index 0000000000..2086ff2412 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -0,0 +1,470 @@ +//! Static plans for commands that do not build models, and direct seed identity reads. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::plan_output::{ + function_entry, selected_test_entries, test_selected, +}; +use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_entries::values::{ + boolean, native, none, planner_model, planner_type, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use std::collections::{BTreeMap, BTreeSet}; + +const FINGERPRINT_CONSTANTS: &str = "sqlbuild.compiler.fingerprints.constants"; + +/// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. +#[derive(Default)] +struct StaticPlanParts<'py> { + execution_order: Option>, + fields: Vec<(&'static str, Bound<'py, PyAny>)>, +} + +fn base_plan_output<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + parts: StaticPlanParts<'py>, +) -> PyResult> { + let py = project.py(); + let selected_keys = scope.getattr("selected_keys")?; + let execution_order = match parts.execution_order { + Some(order) => order, + None => { + let mut order: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if selected_keys.contains(&key)? { + order.push(key); + } + } + PyTuple::new(py, order)?.into_any() + } + }; + let mut fields = vec![("execution_order", execution_order)]; + fields.extend(parts.fields); + fields.extend([ + ("selected_keys", selected_keys), + ("upstream_deps", scope.getattr("upstream_deps")?), + ("downstream_deps", scope.getattr("downstream_deps")?), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_map")?), + ("source_read_map", relations.getattr("source_read_map")?), + ( + "python_source_read_map", + relations.getattr("python_source_read_map")?, + ), + ("hook_functions", project.getattr("hook_functions")?), + ( + "enforce_explicit_references", + project.getattr("enforce_explicit_references")?, + ), + ]); + planner_model(py, "PlanOutput", &fields) +} + +/// Resolve command selection through the canonical build selector phase. +#[pyfunction] +#[pyo3(signature = (*, project, selection, auto_load_sources = false))] +fn resolve_static_command_scope<'py>( + project: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + auto_load_sources: bool, +) -> PyResult> { + let py = project.py(); + native( + py, + "resolve_planner_scopes", + &[ + ("project", project.clone()), + ("selection", selection.clone()), + ( + "policies", + planner_model( + py, + "PlannerPolicies", + &[("auto_load_sources", boolean(py, auto_load_sources)?)], + )?, + ), + ], + )? + .getattr("selected_scope") +} + +/// Resolve canonical locations and source routing without warehouse inventory. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_static_relation_context<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let options = kwargs.unwrap_or_else(|| PyDict::new(py)); + KeywordArguments::read( + py, + "resolve_static_relation_context", + Some(options.clone()), + ( + &["project", "adapter", "scope"], + &[ + "deferral", + "project_config", + "local_config", + "relation_keys", + ], + ), + )?; + options.set_item("connection", none(py))?; + options.set_item("known_source_columns", PyDict::new(py))?; + let args = KeywordArguments::read( + py, + "resolve_static_relation_context", + Some(options), + ( + &["project", "adapter", "scope", "connection"], + &[ + "deferral", + "project_config", + "local_config", + "relation_keys", + "known_source_columns", + ], + ), + )?; + planner_relations_context(&args) +} + +/// Project selected audits from canonical static state. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, scope, relations))] +fn build_audit_command_plan<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let materializations = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + materializations.set_item( + model.getattr("name")?, + native(py, "get_materialization_type", &[("model", model.clone())])? + .getattr("value")?, + )?; + } + let audits = native( + py, + "build_selected_audit_entries", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?; + base_plan_output( + project, + scope, + relations, + StaticPlanParts { + execution_order: None, + fields: vec![( + "audit_entries", + PyTuple::new(py, audits.try_iter()?.collect::>>()?)?.into_any(), + )], + }, + ) +} + +/// Project selected SQL tests from canonical static state. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_test_command_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_test_command_plan", + kwargs, + ( + &["project", "adapter", "scope", "relations"], + &["case_name"], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let scope = args.value("scope")?; + let relations = args.value("relations")?; + let selected_keys = scope.getattr("selected_keys")?; + let selection = scope.getattr("sql_test_selection")?; + let selected = selected_test_entries( + (&project, &adapter), + &selected_keys, + &selection, + &args.value("case_name")?, + )?; + let (entries, warnings) = (selected.entries, selected.warnings); + let mut case_names: BTreeSet = BTreeSet::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + let test = test?; + let case_name = test.getattr("case_name")?; + if !case_name.is_none() && test_selected(&test, &selected_keys, &selection)? { + case_names.insert(case_name.extract()?); + } + } + let mut test_keys: Vec> = Vec::new(); + let mut function_keys: Vec> = Vec::new(); + for entry in &entries { + test_keys.push(entry.getattr("key")?); + for key in entry.getattr("function_deps")?.try_iter()? { + function_keys.push(key?); + } + } + let function_keys = PyFrozenSet::new(py, function_keys)?; + let mut functions: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if !function_keys.contains(function.getattr("key")?)? { + continue; + } + let fingerprint_sql = native( + py, + "build_compiled_function_fingerprint_sql", + &[("function", function.clone())], + )?; + functions.push(function_entry( + &function, + &adapter, + &relations, + ( + fingerprint_sql, + none(py), + planner_type(py, "PlanReason", "NO_CHANGE")?, + ), + )?); + } + let scoped_keys = selected_keys.call_method1("__or__", (PyFrozenSet::new(py, test_keys)?,))?; + let mut order: Vec> = Vec::new(); + for key in scope.getattr("execution_order")?.try_iter()? { + let key = key?; + if scoped_keys.contains(&key)? { + order.push(key); + } + } + base_plan_output( + &project, + &scope, + &relations, + StaticPlanParts { + execution_order: Some(PyTuple::new(py, order)?.into_any()), + fields: vec![ + ("test_entries", PyTuple::new(py, entries)?.into_any()), + ( + "available_test_case_names", + PyTuple::new(py, case_names)?.into_any(), + ), + ("function_entries", PyTuple::new(py, functions)?.into_any()), + ("warnings", PyTuple::new(py, warnings)?.into_any()), + ], + }, + ) +} + +/// Project selected direct seed work and canonical identities. +#[pyfunction] +#[pyo3(signature = (*, project, scope, relations, fingerprints = None))] +fn build_seed_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + fingerprints: Option>, +) -> PyResult> { + let py = project.py(); + let existing = match fingerprints { + Some(fingerprints) if fingerprints.is_truthy()? => fingerprints, + _ => PyDict::new(py).into_any(), + }; + let selected_keys = scope.getattr("selected_keys")?; + let mut entries: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if !selected_keys.contains(seed.getattr("key")?)? { + continue; + } + let identity = native(py, "build_seed_identity", &[("seed", seed.clone())])?; + let (version_hash, metadata_json) = (identity.get_item(0)?, identity.get_item(1)?); + let previous = existing.call_method1("get", (seed.getattr("name")?,))?; + let reason = if previous.is_none() { + "FIRST_RUN" + } else if previous.getattr("version_hash")?.ne(&version_hash)? { + "CONFIG_CHANGED" + } else { + "NO_CHANGE" + }; + entries.push(planner_model( + py, + "SeedPlanEntry", + &[ + ("key", seed.getattr("key")?), + ("name", seed.getattr("name")?), + ("destination", seed.getattr("destination")?), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ( + "columns", + native(py, "extract_seed_columns", &[("seed", seed.clone())])?, + ), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ("fingerprint_definition", metadata_json.clone()), + ("fingerprint_version_hash", version_hash), + ("fingerprint_metadata_json", metadata_json), + ("reason", planner_type(py, "PlanReason", reason)?), + ], + )?); + } + base_plan_output( + project, + scope, + relations, + StaticPlanParts { + execution_order: None, + fields: vec![("seed_entries", PyTuple::new(py, entries)?.into_any())], + }, + ) +} + +/// Project static relation data for Python checks. +#[pyfunction] +#[pyo3(signature = (*, project, scope, relations))] +fn build_relation_command_plan<'py>( + project: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, +) -> PyResult> { + base_plan_output(project, scope, relations, StaticPlanParts::default()) +} + +/// A schema target's sort key: the database with `None` first as an empty name, then the schema. +type Target = (Option, String); + +/// Read only fingerprint state required by selected direct seeds. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, seeds))] +fn read_selected_seed_fingerprints<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + seeds: &Bound<'py, PyAny>, +) -> PyResult> { + let py = adapter.py(); + let table_name: String = + imported(py, FINGERPRINT_CONSTANTS, "FINGERPRINT_TABLE_NAME")?.extract()?; + let seed_node_type = imported(py, FINGERPRINT_CONSTANTS, "NODE_TYPE_SEED")?; + let mut names_by_target: BTreeMap<(String, String), (Target, BTreeSet)> = + BTreeMap::new(); + for seed in seeds.try_iter()? { + let seed = seed?; + let destination = seed.getattr("destination")?; + let schema = destination.getattr("schema")?; + if schema.is_none() { + continue; + } + let database: Option = destination.getattr("database")?.extract()?; + let schema: String = schema.extract()?; + names_by_target + .entry((database.clone().unwrap_or_default(), schema.clone())) + .or_insert_with(|| ((database, schema), BTreeSet::new())) + .1 + .insert(seed.getattr("name")?.extract()?); + } + let fingerprints = PyDict::new(py); + let mut relations_by_target: Vec> = Vec::new(); + for ((database, schema), _) in names_by_target.values() { + let names = PyTuple::new(py, [table_name.as_str()])?; + let schemas = PyTuple::new(py, [schema.as_str()])?; + let kwargs = PyDict::new(py); + kwargs.set_item("connection", connection)?; + kwargs.set_item("database", database)?; + kwargs.set_item("schemas", schemas)?; + kwargs.set_item("names", names)?; + relations_by_target.push(adapter.getattr("list_relations")?.call((), Some(&kwargs))?); + } + for (((database, schema), names), relations) in + names_by_target.values().zip(relations_by_target) + { + let mut table_exists = false; + for relation in relations.try_iter()? { + let name: String = relation?.getattr("name")?.extract()?; + if name.to_lowercase() == table_name.to_lowercase() { + table_exists = true; + break; + } + } + let node_names = PyTuple::new(py, names.iter())?; + let latest = native( + py, + "read_latest_fingerprints", + &[ + ("connection", connection.clone()), + ("execute", adapter.getattr("execute")?), + ("table_exists", boolean(py, table_exists)?), + ( + "database", + database + .as_deref() + .map_or_else(|| none(py), |name| PyString::new(py, name).into_any()), + ), + ("schema", PyString::new(py, schema).into_any()), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "render_read_latest_sql", + adapter.getattr("render_read_latest_fingerprints_sql")?, + ), + ("node_names", node_names.into_any()), + ], + )?; + let items = PyList::new( + py, + latest + .getattr("fingerprints")? + .call_method0("items")? + .try_iter()? + .collect::>>()?, + )?; + for item in items.iter() { + let (name, fingerprint) = (item.get_item(0)?, item.get_item(1)?); + if fingerprint.getattr("node_type")?.eq(&seed_node_type)? + && names.contains(&name.extract::()?) + { + fingerprints.set_item(name, fingerprint)?; + } + } + } + Ok(fingerprints) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_static_command_scope, module)?)?; + module.add_function(wrap_pyfunction!(resolve_static_relation_context, module)?)?; + module.add_function(wrap_pyfunction!(build_audit_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_test_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_seed_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(build_relation_command_plan, module)?)?; + module.add_function(wrap_pyfunction!(read_selected_seed_fingerprints, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 9950582c5a..82b0c1a87c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -1,6 +1,8 @@ //! Plan entry and plan output assembly over native planner resolution. +mod command_plans; mod model_entries; +mod plan_output; pub(crate) mod registration; mod relations; mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs new file mode 100644 index 0000000000..1fdd330837 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -0,0 +1,844 @@ +//! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; +use crate::bindings::_helpers::planner_entries::values::{ + DISCOVERY_MODELS, PLANNER_EXCEPTIONS, error_of, native, none, planner_model, planner_type, + string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const LOADER_CONSUMER: &str = "loader"; +const HOOK_CONSUMER: &str = "hook"; +const MATERIALIZATION_CONSUMER: &str = "custom materialization"; + +fn tuple_of<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// `mapping.get(key)` for an optional mapping. +fn optional_get<'py>( + mapping: &Bound<'py, PyAny>, + key: &Bound<'py, PyAny>, +) -> PyResult> { + if mapping.is_truthy()? { + return mapping.call_method1("get", (key,)); + } + Ok(none(mapping.py())) +} + +fn optional_get_or<'py>( + mapping: &Bound<'py, PyAny>, + key: &Bound<'py, PyAny>, + default: &str, +) -> PyResult> { + let value = optional_get(mapping, key)?; + if value.is_none() { + return Ok(string(mapping.py(), default)); + } + Ok(value) +} + +fn by_name<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let lookup = PyDict::new(items.py()); + for item in items.try_iter()? { + let item = item?; + lookup.set_item(item.getattr("name")?, item)?; + } + Ok(lookup) +} + +/// The plan reason for one selected seed. +fn seed_plan_reason<'py>( + seed_name: &Bound<'py, PyAny>, + expected_version_hash: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + seed_plan_reasons: &Bound<'py, PyAny>, +) -> PyResult> { + let py = seed_name.py(); + let reason = optional_get(seed_plan_reasons, seed_name)?; + if !reason.is_none() { + return Ok(reason); + } + if !snapshot + .getattr("existing_relations")? + .contains(seed_name)? + { + return planner_type(py, "PlanReason", "FIRST_RUN"); + } + let fingerprint = snapshot + .getattr("fingerprints")? + .getattr("seeds")? + .call_method1("get", (seed_name,))?; + if fingerprint.is_none() { + return planner_type(py, "PlanReason", "FIRST_RUN"); + } + if !expected_version_hash.is_none() + && fingerprint + .getattr("version_hash")? + .ne(expected_version_hash)? + { + return planner_type(py, "PlanReason", "CONFIG_CHANGED"); + } + planner_type(py, "PlanReason", "NO_CHANGE") +} + +fn seed_entries<'py>( + project: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + extras: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = project.py(); + let version_hashes = extras.getattr("seed_version_hashes")?; + let metadata_jsons = extras.getattr("seed_metadata_jsons")?; + let plan_reasons = extras.getattr("seed_plan_reasons")?; + let mut entries: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if !selected_keys.contains(seed.getattr("key")?)? { + continue; + } + let name = seed.getattr("name")?; + entries.push(planner_model( + py, + "SeedPlanEntry", + &[ + ("key", seed.getattr("key")?), + ("name", name.clone()), + ("destination", seed.getattr("destination")?), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ( + "columns", + native(py, "extract_seed_columns", &[("seed", seed.clone())])?, + ), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ( + "fingerprint_definition", + optional_get_or(&metadata_jsons, &name, "")?, + ), + ( + "fingerprint_version_hash", + optional_get_or(&version_hashes, &name, "")?, + ), + ( + "fingerprint_metadata_json", + optional_get_or(&metadata_jsons, &name, "{}")?, + ), + ( + "reason", + seed_plan_reason( + &name, + &optional_get(&version_hashes, &name)?, + snapshot, + &plan_reasons, + )?, + ), + ], + )?); + } + Ok(entries) +} + +fn provider_usages<'py>( + provider_usages: &Bound<'py, PyAny>, + consumer_kind: &str, + consumer_name: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = consumer_name.py(); + let mut usages: Vec> = Vec::new(); + for usage in provider_usages.try_iter()? { + let usage = usage?; + usages.push(planner_model( + py, + "PlanProviderUsage", + &[ + ("provider_name", usage.getattr("provider_name")?), + ("consumer_kind", string(py, consumer_kind)), + ("consumer_name", consumer_name.clone()), + ("parameter_name", usage.getattr("parameter_name")?), + ( + "annotation_class_name", + usage.getattr("annotation_class_name")?, + ), + ("annotation_module", usage.getattr("annotation_module")?), + ], + )?); + } + Ok(usages) +} + +fn hook_provider_usages<'py>( + entry: &Bound<'py, PyAny>, + hook_by_name: &Bound<'py, PyDict>, +) -> PyResult>> { + let py = entry.py(); + let mut usages: Vec> = Vec::new(); + let python_hook = imported(py, DISCOVERY_MODELS, "PythonHookEntry")?; + let mut hooks: Vec> = Vec::new(); + for field in ["pre_hooks", "post_hooks"] { + let hook_entry = entry.getattr(field)?; + if hook_entry.is_none() { + continue; + } + if hook_entry.is_instance_of::() || hook_entry.is_instance_of::() { + for hook in hook_entry.try_iter()? { + hooks.push(hook?); + } + } else { + hooks.push(hook_entry); + } + } + for hook in hooks { + if !hook.is_instance(&python_hook)? { + continue; + } + let Some(discovered) = hook_by_name.get_item(hook.getattr("name")?)? else { + continue; + }; + usages.extend(provider_usages( + &discovered.getattr("provider_usages")?, + HOOK_CONSUMER, + &discovered.getattr("name")?, + )?); + } + Ok(usages) +} + +/// Provider usages of the planned loaders, hooks and custom materializations, in plan order. +fn plan_provider_usages<'py>( + project: &Bound<'py, PyAny>, + model_entries: &Bound<'py, PyAny>, + source_load_entries: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut usages: Vec> = Vec::new(); + let loader_by_name = by_name(&project.getattr("loader_functions")?)?; + for entry in source_load_entries.try_iter()? { + let Some(loader) = loader_by_name.get_item(entry?.getattr("loader")?)? else { + continue; + }; + usages.extend(provider_usages( + &loader.getattr("provider_usages")?, + LOADER_CONSUMER, + &loader.getattr("name")?, + )?); + } + let hook_by_name = by_name(&project.getattr("hook_functions")?)?; + let materialization_by_name = by_name(&project.getattr("materialization_files")?)?; + for entry in model_entries.try_iter()? { + let entry = entry?; + usages.extend(hook_provider_usages(&entry, &hook_by_name)?); + let custom = entry.getattr("custom_materialization_name")?; + if custom.is_none() { + continue; + } + let Some(materialization) = materialization_by_name.get_item(custom)? else { + continue; + }; + usages.extend(provider_usages( + &materialization.getattr("provider_usages")?, + MATERIALIZATION_CONSUMER, + &materialization.getattr("name")?, + )?); + } + tuple_of(py, usages) +} + +/// Fail when a selected loader depends on an unselected intermediate loader with no target. +fn require_intermediate_loader_targets<'py>( + project: &Bound<'py, PyAny>, + snapshot: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyAny>, + source_load_entries: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py = project.py(); + let loader_functions = project.getattr("loader_functions")?; + let loader_by_name = by_name(&loader_functions)?; + let selected_names = PyFrozenSet::new( + py, + source_load_entries + .try_iter()? + .map(|entry| entry?.getattr("name")) + .collect::>>()?, + )?; + let existing = snapshot.getattr("existing_relations")?; + for entry in source_load_entries.try_iter()? { + let source_entry = source_map.call_method1("get", (entry?.getattr("name")?,))?; + if source_entry.is_none() || source_entry.getattr("loader")?.is_none() { + continue; + } + let Some(loader_function) = loader_by_name.get_item(source_entry.getattr("loader")?)? + else { + continue; + }; + let dependencies = native( + py, + "upstream_loader_dependency_names", + &[ + ("loader_function", loader_function), + ("loader_functions", loader_functions.clone()), + ], + )?; + for dependency_name in dependencies.try_iter()? { + let dependency_name = dependency_name?; + let dependency = source_map.call_method1("get", (&dependency_name,))?; + if dependency.is_none() || selected_names.contains(dependency.getattr("name")?)? { + continue; + } + let table = dependency.getattr("table")?; + let target = if table.is_truthy()? { + table + } else { + dependency.getattr("name")? + }; + if existing.contains(&target)? { + continue; + } + let source_name = source_entry.getattr("name")?.str()?; + return Err(error_of( + py, + (PLANNER_EXCEPTIONS, "PlannerInputError"), + &format!( + "Source '{source_name}' requires intermediate loader '{}', but its target \ + relation '{}' does not exist; use +source:{source_name} to refresh upstream \ + ingress dependencies", + dependency_name.str()?, + target.str()?, + ), + &[], + )); + } + } + Ok(()) +} + +/// Build one canonical function entry from resolved relation inputs. +pub(super) fn function_entry<'py>( + function: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + relations: &Bound<'py, PyAny>, + (fingerprint_query_sql, previous_query_sql, reason): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ), +) -> PyResult> { + let py = function.py(); + let body_sql = native( + py, + "resolve_function_sql", + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", relations.getattr("source_read_map")?), + ( + "source_warehouse_columns", + relations.getattr("source_warehouse_columns")?, + ), + ( + "star_exclude_keyword", + relations.getattr("star_exclude_keyword")?, + ), + ], + )?; + let field = |name: &str| function.getattr(name); + planner_model( + py, + "FunctionPlanEntry", + &[ + ("key", field("key")?), + ("name", field("name")?), + ("relative_path", field("relative_path")?), + ("destination", field("destination")?), + ("arguments", field("arguments")?), + ("returns", field("returns")?), + ("body_sql", body_sql), + ("fingerprint_query_sql", fingerprint_query_sql), + ("fingerprint_destination", field("fingerprint_destination")?), + ("return_columns", field("return_columns")?), + ("language", field("language")?), + ("source_file_path", field("source_file_path")?), + ("runtime_version", field("runtime_version")?), + ("entry_point", field("entry_point")?), + ("packages", field("packages")?), + ("previous_query_sql", previous_query_sql), + ("reason", reason), + ], + ) +} + +fn function_entries<'py>( + (project, adapter, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = project.py(); + let functions = changes.getattr("functions")?; + let previous = snapshot.getattr("fingerprints")?.getattr("functions")?; + let mut entries: Vec> = Vec::new(); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if !selected_keys.contains(function.getattr("key")?)? { + continue; + } + let name = function.getattr("name")?; + let change = functions.get_item(&name)?; + let previous_query_sql = if previous.contains(&name)? { + previous.get_item(&name)?.getattr("definition")? + } else { + none(py) + }; + entries.push(function_entry( + &function, + adapter, + relations, + ( + change.getattr("fingerprint_sql")?, + previous_query_sql, + change.getattr("reason")?, + ), + )?); + } + Ok(entries) +} + +/// Whether a test is named by the selection or covers a selected resource, and not excluded. +pub(super) fn test_selected( + test: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, + selection: &Bound<'_, PyAny>, +) -> PyResult { + let parent = test.getattr("parent_name")?; + let name = if parent.is_truthy()? { + parent + } else { + test.getattr("name")? + }; + if selection.getattr("excluded_names")?.contains(&name)? { + return Ok(false); + } + Ok(selection.getattr("names")?.contains(&name)? + || scope_overlaps(&test.getattr("scope_deps")?, selected_keys)?) +} + +/// Planned SQL test entries and the warnings planning them produced. +pub(super) struct SelectedTests<'py> { + pub(super) entries: Vec>, + pub(super) warnings: Vec>, +} + +/// Selected SQL test entries and their warnings, failing on invalid fixtures. +pub(super) fn selected_test_entries<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + selected_keys: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, + case_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut selected: Vec> = Vec::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + let test = test?; + if !test_selected(&test, selected_keys, selection)? { + continue; + } + if !case_name.is_none() && test.getattr("case_name")?.ne(case_name)? { + continue; + } + selected.push(test); + } + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); + if selected.is_empty() { + return Ok(SelectedTests { entries, warnings }); + } + let analysis = project.getattr("settings")?.getattr("sql_analysis")?; + let fixture_context = if analysis.is_truthy()? { + native( + py, + "build_relation_fixture_context", + &[("project", project.clone())], + )? + } else { + none(py) + }; + let results = native( + py, + "plan_sql_tests", + &[ + ("tests", tuple_of(py, selected)?), + ("project", project.clone()), + ("adapter", adapter.clone()), + ("sql_analysis_enabled", analysis), + ( + "validate_fixtures", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("fixture_planning_context", fixture_context), + ], + )?; + let mut diagnostics: Vec = Vec::new(); + for result in results.try_iter()? { + let result = result?; + let entry = result.getattr("entry")?; + if entry.is_none() { + for diagnostic in result.getattr("fixture_diagnostics")?.try_iter()? { + diagnostics.push(diagnostic?.str()?.to_string()); + } + continue; + } + entries.push(entry); + for warning in result.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + } + if !diagnostics.is_empty() { + return Err(error_of( + py, + (PLANNER_EXCEPTIONS, "PlannerInputError"), + &format!("Invalid SQL test fixtures:\n- {}", diagnostics.join("\n- ")), + &[], + )); + } + Ok(SelectedTests { entries, warnings }) +} + +/// SQL tests selected by name or overlapping one selection, planned into a reusable result. +fn planned_sql_tests<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let selected = selected_test_entries((project, adapter), selected_keys, selection, &none(py))?; + planner_model( + py, + "PlannedSqlTests", + &[ + ("selected_keys", selected_keys.clone()), + ("sql_test_selection", selection.clone()), + ("entries", tuple_of(py, selected.entries)?), + ("warnings", tuple_of(py, selected.warnings)?), + ], + ) +} + +fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let adapter = args.value("adapter")?; + let scope = args.value("scope")?; + let snapshot = args.value("snapshot")?; + let relations = args.value("relations")?; + let model_entry_results = args.value("model_entry_results")?; + let extras = args.value("extras")?; + let extras = if extras.is_none() { + planner_model(py, "PlanOutputExtras", &[])? + } else { + extras + }; + let selected_keys = scope.getattr("selected_keys")?; + let execution_order = scope.getattr("execution_order")?; + let source_map = relations.getattr("source_map")?; + let seeds = seed_entries(&project, &snapshot, &selected_keys, &extras)?; + let source_load_entries = native( + py, + "build_source_load_entries", + &[ + ("execution_order", execution_order.clone()), + ("selected_keys", selected_keys.clone()), + ("source_map", source_map.clone()), + ("is_reload", args.value("reload_sources")?), + ], + )?; + require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; + let functions = function_entries( + (&project, &adapter, &snapshot), + &relations, + &args.value("changes")?, + &selected_keys, + )?; + let model_entries = model_entry_results.getattr("entries")?; + let materializations = PyDict::new(py); + for entry in model_entries.try_iter()? { + let entry = entry?; + materializations.set_item( + entry.getattr("name")?, + entry.getattr("materialization_type")?, + )?; + } + let audit_entries = native( + py, + "build_selected_audit_entries", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("scope", scope.clone()), + ("relations", relations.clone()), + ("model_materializations", materializations.into_any()), + ], + )?; + let selection = scope.getattr("sql_test_selection")?; + let mut planned = extras.getattr("planned_sql_tests")?; + if planned.is_none() + || planned.getattr("selected_keys")?.ne(&selected_keys)? + || planned.getattr("sql_test_selection")?.ne(&selection)? + { + planned = planned_sql_tests(&project, &adapter, &selected_keys, &selection)?; + } + let test_entries = planned.getattr("entries")?; + let mut test_keys: Vec> = Vec::new(); + for entry in test_entries.try_iter()? { + test_keys.push(entry?.getattr("key")?); + } + let scoped_keys = selected_keys.call_method1("__or__", (PyFrozenSet::new(py, test_keys)?,))?; + let mut scoped_order: Vec> = Vec::new(); + for key in execution_order.try_iter()? { + let key = key?; + if scoped_keys.contains(&key)? { + scoped_order.push(key); + } + } + let mut warnings: Vec> = Vec::new(); + for warning in model_entry_results.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + for warning in planned.getattr("warnings")?.try_iter()? { + warnings.push(warning?); + } + planner_model( + py, + "PlanOutput", + &[ + ("execution_order", tuple_of(py, scoped_order)?), + ("model_entries", model_entries.clone()), + ("seed_entries", tuple_of(py, seeds)?), + ( + "source_load_entries", + PyTuple::new( + py, + source_load_entries + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ("function_entries", tuple_of(py, functions)?), + ( + "audit_entries", + PyTuple::new(py, audit_entries.try_iter()?.collect::>>()?)? + .into_any(), + ), + ( + "test_entries", + PyTuple::new(py, test_entries.try_iter()?.collect::>>()?)? + .into_any(), + ), + ("selected_keys", selected_keys), + ("warnings", tuple_of(py, warnings)?), + ("upstream_deps", scope.getattr("upstream_deps")?), + ("downstream_deps", scope.getattr("downstream_deps")?), + ("model_locations", relations.getattr("model_locations")?), + ("seed_locations", relations.getattr("seed_locations")?), + ( + "function_locations", + relations.getattr("function_locations")?, + ), + ("source_map", source_map), + ("source_read_map", relations.getattr("source_read_map")?), + ( + "python_source_read_map", + relations.getattr("python_source_read_map")?, + ), + ("hook_functions", project.getattr("hook_functions")?), + ( + "enforce_explicit_references", + project.getattr("enforce_explicit_references")?, + ), + ( + "provider_usages", + plan_provider_usages(&project, &model_entries, &source_load_entries)?, + ), + ( + "python_identity_fingerprints", + snapshot.getattr("fingerprints")?.getattr("python_nodes")?, + ), + ], + ) +} + +/// Assemble the plan output from planned entries and resolved relation inputs. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_plan_output<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_plan_output", + kwargs, + ( + &[ + "project", + "adapter", + "scope", + "snapshot", + "relations", + "changes", + "model_entry_results", + "reload_sources", + ], + &["extras"], + ), + )?; + plan_output(&args) +} + +/// Build one canonical function entry from resolved relation inputs. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn plan_function<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "plan_function", + kwargs, + ( + &["function", "adapter", "relations", "fingerprint_query_sql"], + &["previous_query_sql", "reason"], + ), + )?; + let reason = if args.passed("reason")? { + args.value("reason")? + } else { + planner_type(py, "PlanReason", "NO_CHANGE")? + }; + function_entry( + &args.value("function")?, + &args.value("adapter")?, + &args.value("relations")?, + ( + args.value("fingerprint_query_sql")?, + args.value("previous_query_sql")?, + reason, + ), + ) +} + +/// Plan SQL tests selected by name or overlapping one selection into a reusable result. +#[pyfunction] +#[pyo3(signature = (*, project, adapter, selected_keys, sql_test_selection))] +fn plan_selected_sql_tests<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + sql_test_selection: &Bound<'py, PyAny>, +) -> PyResult> { + planned_sql_tests(project, adapter, selected_keys, sql_test_selection) +} + +/// Plan selected SQL tests into `future`, which re-raises any failure on the joining thread. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn plan_selected_sql_tests_into<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<()> { + let args = KeywordArguments::read( + py, + "plan_selected_sql_tests_into", + kwargs, + ( + &[ + "future", + "project", + "adapter", + "selected_keys", + "sql_test_selection", + ], + &[], + ), + )?; + let future = args.value("future")?; + match native( + py, + "plan_selected_sql_tests", + &[ + ("project", args.value("project")?), + ("adapter", args.value("adapter")?), + ("selected_keys", args.value("selected_keys")?), + ("sql_test_selection", args.value("sql_test_selection")?), + ], + ) { + Ok(planned) => future.call_method1("set_result", (planned,))?, + Err(error) => future.call_method1("set_exception", (error.value(future.py()),))?, + }; + Ok(()) +} + +/// Whether a test is named by the selection or covers a selected resource, and not excluded. +#[pyfunction] +#[pyo3(signature = (*, test, selected_keys, sql_test_selection))] +fn sql_test_is_selected( + test: &Bound<'_, PyAny>, + selected_keys: &Bound<'_, PyAny>, + sql_test_selection: &Bound<'_, PyAny>, +) -> PyResult { + test_selected(test, selected_keys, sql_test_selection) +} + +/// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_selected_test_entries<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { + let args = KeywordArguments::read( + py, + "build_selected_test_entries", + kwargs, + ( + &["project", "adapter", "selected_keys", "sql_test_selection"], + &["case_name"], + ), + )?; + let selected = selected_test_entries( + (&args.value("project")?, &args.value("adapter")?), + &args.value("selected_keys")?, + &args.value("sql_test_selection")?, + &args.value("case_name")?, + )?; + Ok(( + PyList::new(py, selected.entries)?, + PyList::new(py, selected.warnings)?, + )) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_plan_output, module)?)?; + module.add_function(wrap_pyfunction!(plan_function, module)?)?; + module.add_function(wrap_pyfunction!(plan_selected_sql_tests, module)?)?; + module.add_function(wrap_pyfunction!(plan_selected_sql_tests_into, module)?)?; + module.add_function(wrap_pyfunction!(sql_test_is_selected, module)?)?; + module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index c1fe546924..d1fe76c103 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -3,6 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::command_plans::register(module)?; super::model_entries::register(module)?; + super::plan_output::register(module)?; super::relations::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index f7e65b39a2..0cbbac663b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -12,6 +12,7 @@ pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exception pub(super) const PLANNER_CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +pub(super) const DISCOVERY_MODELS: &str = "sqlbuild.compiler.discovery.models"; /// `sqlbuild._native.name(**pairs)`. pub(super) fn native<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index c8ecc42e35..f80cc80940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -7,7 +7,6 @@ pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exception pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; pub(crate) const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; -pub(crate) const PLAN_OUTPUT_MODULE: &str = "sqlbuild.compiler.planner._helpers.output.plan_output"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 99fb843502..ab65288a43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_MODULE; +use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::objects::{ model_key_names, native, planner_model, replace, sorted_names, strings, text, tuple, }; @@ -92,7 +92,7 @@ pub(crate) fn assemble_base_plan_output<'py>( ], )?; let plan_output = call_with( - &imported(py, PLAN_OUTPUT_MODULE, "build_plan_output")?, + &imported(py, NATIVE_MODULE, "build_plan_output")?, &[ ("project", runtime.getattr("project")?), ("adapter", runtime.getattr("adapter")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index 0f20304fee..ab366575e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,8 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, planner_resolution, - planner_scenario, planner_scope, planner_selection, planner_strategy, planner_warehouse, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, + planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, + planner_warehouse, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 729601ba64..97aa00d23c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -22,8 +22,8 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; -const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; +const SEED_PLAN_MODULE: &str = "sqlbuild._native"; +const SEED_STATE_MODULE: &str = "sqlbuild._native"; const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 018dbf9435..fcc0b9a8e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -36,7 +36,7 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; -const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; +const TEST_PLAN_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7a150b3625..a348b73457 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,6 +1,7 @@ """Private SQLBuild native engine bindings.""" from collections.abc import Callable, Iterable, Mapping, Sequence, Set +from concurrent.futures import Future from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime @@ -224,6 +225,7 @@ from sqlbuild.compiler.planner.models import ( DirectModelVersionIdentities, EmptyCursorInputDecision, FixtureRelationMetadata, + FunctionPlanEntry, FutureCursorSafetyEvidence, MaximumStartPolicyInputs, MaximumStartSafetyEvidence, @@ -263,6 +265,7 @@ from sqlbuild.compiler.planner.models import ( PlannerSelection, PlannerWarehouseState, PlanOutput, + PlanOutputExtras, PlanWarning, QueryShape, ReferenceRename, @@ -4837,6 +4840,97 @@ def apply_maximum_start_policy( policy: MaximumStartPolicyInputs, has_start_override: bool, ) -> CursorBounds: ... +def build_plan_output( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + relations: PlannerRelationsContext, + changes: PlannerChangeResults, + model_entry_results: PlannerModelEntryResults, + reload_sources: bool, + extras: PlanOutputExtras | None = None, +) -> PlanOutput: ... +def plan_function( + *, + function: CompiledFunctionView, + adapter: BaseAdapter, + relations: PlannerRelationsContext, + fingerprint_query_sql: str, + previous_query_sql: str | None = None, + reason: PlanReason = ..., +) -> FunctionPlanEntry: ... +def plan_selected_sql_tests( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, +) -> PlannedSqlTests: ... +def plan_selected_sql_tests_into( + *, + future: Future[PlannedSqlTests], + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, +) -> None: ... +def sql_test_is_selected( + *, + test: CompiledSqlTest, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, +) -> bool: ... +def build_selected_test_entries( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, + case_name: str | None = None, +) -> tuple[list[SqlTestPlanEntry], list[PlanWarning]]: ... +def resolve_static_command_scope( + *, project: CompiledProjectView, selection: PlannerSelection, auto_load_sources: bool = False +) -> PlannerScope: ... +def resolve_static_relation_context( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + deferral: DeferralInputs | None = None, + project_config: ProjectConfig | None = None, + local_config: LocalConfig | None = None, + relation_keys: frozenset[CompiledObjectKey] | None = None, +) -> PlannerRelationsContext: ... +def build_audit_command_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + relations: PlannerRelationsContext, +) -> PlanOutput: ... +def build_test_command_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + scope: PlannerScope, + relations: PlannerRelationsContext, + case_name: str | None = None, +) -> PlanOutput: ... +def build_seed_command_plan( + *, + project: CompiledProjectView, + scope: PlannerScope, + relations: PlannerRelationsContext, + fingerprints: dict[str, Fingerprint] | None = None, +) -> PlanOutput: ... +def build_relation_command_plan( + *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext +) -> PlanOutput: ... +def read_selected_seed_fingerprints( + *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] +) -> dict[str, Fingerprint]: ... def resolve_future_cursor_config( *, model: CompiledModelView, project_config: FutureCursorsConfig | None ) -> FutureCursorsConfig | None: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py deleted file mode 100644 index 8dc4d94372..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Internal focused-command planning phases.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py deleted file mode 100644 index 650547e835..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Targeted warehouse identity state for direct seed commands.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import read_latest_fingerprints -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet - - -def read_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] -) -> dict[str, Fingerprint]: - """Read only fingerprint state required to classify selected direct seeds.""" - - names_by_target: dict[tuple[str | None, str], set[str]] = {} - for seed in seeds: - if seed.destination.schema is None: - continue - names_by_target.setdefault((seed.destination.database, seed.destination.schema), set()).add( - seed.name - ) - - targets: tuple[tuple[str | None, str], ...] = tuple( - sorted(names_by_target, key=lambda target: ((target[0] or ""), target[1])) - ) - relations_by_target: dict[tuple[str | None, str], tuple[RelationInfo, ...]] = ( - _gather_fingerprint_relations( - adapter=adapter, - connection=connection, - targets=targets, - ) - ) - fingerprints: dict[str, Fingerprint] = {} - for database, schema in targets: - names: set[str] = names_by_target[(database, schema)] - relations: tuple[RelationInfo, ...] = relations_by_target[(database, schema)] - table_exists: bool = any( - relation.name.lower() == FINGERPRINT_TABLE_NAME.lower() for relation in relations - ) - fingerprint_set: FingerprintSet = read_latest_fingerprints( - connection=connection, - execute=adapter.execute, - table_exists=table_exists, - database=database, - schema=schema, - render_qualified_name=adapter.render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_fingerprints_sql, - node_names=tuple(sorted(names)), - ) - fingerprints.update( - { - name: fingerprint - for name, fingerprint in fingerprint_set.fingerprints.items() - if fingerprint.node_type == NODE_TYPE_SEED and name in names - } - ) - return fingerprints - - -def _gather_fingerprint_relations( - *, - adapter: BaseAdapter, - connection: Any, - targets: tuple[tuple[str | None, str], ...], -) -> dict[tuple[str | None, str], tuple[RelationInfo, ...]]: - if not targets: - return {} - database, schema = targets[0] - relations: tuple[RelationInfo, ...] = adapter.list_relations( - connection=connection, - database=database, - schemas=(schema,), - names=(FINGERPRINT_TABLE_NAME,), - ) - return { - (database, schema): relations, - **_gather_fingerprint_relations( - adapter=adapter, - connection=connection, - targets=targets[1:], - ), - } diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py deleted file mode 100644 index 17b17d23e7..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Canonical static planning phases for commands that do not build models.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_compiled_function_fingerprint_sql, - build_planner_relations_context, - build_seed_identity, - build_selected_audit_entries, - extract_seed_columns, - get_materialization_type, - resolve_planner_scopes, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.output.plan_output import ( - build_selected_test_entries, - plan_function, - sql_test_is_selected, -) -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - DeferralInputs, - FunctionPlanEntry, - PlannerPolicies, - PlannerRelationsContext, - PlannerScope, - PlannerSelection, - PlanOutput, - PlanWarning, - SeedPlanEntry, - SqlTestPlanEntry, -) -from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_static_command_scope_impl( - *, - project: CompiledProjectView, - selection: PlannerSelection, - auto_load_sources: bool = False, -) -> PlannerScope: - """Resolve command selection through the same canonical selector phase as builds.""" - - return resolve_planner_scopes( - project=project, - selection=selection, - policies=PlannerPolicies(auto_load_sources=auto_load_sources), - ).selected_scope - - -def resolve_static_relation_context_impl( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - deferral: DeferralInputs | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerRelationsContext: - """Resolve canonical locations and source routing without warehouse inventory.""" - - return build_planner_relations_context( - project=project, - adapter=adapter, - connection=None, - scope=scope, - deferral=deferral, - project_config=project_config, - local_config=local_config, - known_source_columns={}, - relation_keys=relation_keys, - ) - - -def build_audit_command_plan_impl( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, -) -> PlanOutput: - """Project the selected audits from static compile and relation state.""" - - model_materializations: dict[str, str] = { - model.name: get_materialization_type(model).value for model in project.models - } - return _base_plan_output( - project=project, - scope=scope, - relations=relations, - audit_entries=tuple( - build_selected_audit_entries( - project=project, - adapter=adapter, - scope=scope, - relations=relations, - model_materializations=model_materializations, - ) - ), - ) - - -def build_test_command_plan_impl( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, - case_name: str | None = None, -) -> PlanOutput: - """Project selected SQL tests from static compile state.""" - - test_entries, warnings = build_selected_test_entries( - project=project, - adapter=adapter, - selected_keys=scope.selected_keys, - sql_test_selection=scope.sql_test_selection, - case_name=case_name, - ) - available_test_case_names: tuple[str, ...] = tuple( - sorted( - { - test.case_name - for test in project.sql_tests - if test.case_name is not None - and sql_test_is_selected( - test=test, - selected_keys=scope.selected_keys, - sql_test_selection=scope.sql_test_selection, - ) - } - ) - ) - test_keys: frozenset[CompiledObjectKey] = frozenset(entry.key for entry in test_entries) - function_keys_set: set[CompiledObjectKey] = set() - for test_entry in test_entries: - function_keys_set.update(test_entry.function_deps) - function_keys: frozenset[CompiledObjectKey] = frozenset(function_keys_set) - function_entries: tuple[FunctionPlanEntry, ...] = tuple( - plan_function( - function=function, - adapter=adapter, - relations=relations, - fingerprint_query_sql=build_compiled_function_fingerprint_sql(function), - ) - for function in project.functions - if function.key in function_keys - ) - return _base_plan_output( - project=project, - scope=scope, - relations=relations, - execution_order=tuple( - key for key in scope.execution_order if key in scope.selected_keys | test_keys - ), - test_entries=tuple(test_entries), - available_test_case_names=available_test_case_names, - function_entries=function_entries, - warnings=tuple(warnings), - ) - - -def build_seed_command_plan_impl( - *, - project: CompiledProjectView, - scope: PlannerScope, - relations: PlannerRelationsContext, - fingerprints: dict[str, Fingerprint] | None = None, -) -> PlanOutput: - """Project selected direct seed work and canonical identities.""" - - existing: dict[str, Fingerprint] = fingerprints or {} - entries: list[SeedPlanEntry] = [] - for seed in project.seeds: - if seed.key not in scope.selected_keys: - continue - version_hash, metadata_json = build_seed_identity(seed) - previous: Fingerprint | None = existing.get(seed.name) - reason: PlanReason = ( - PlanReason.FIRST_RUN - if previous is None - else PlanReason.CONFIG_CHANGED - if previous.version_hash != version_hash - else PlanReason.NO_CHANGE - ) - entries.append( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=extract_seed_columns(seed), - csv_settings=seed.schema_entry.csv_settings, - fingerprint_definition=metadata_json, - fingerprint_version_hash=version_hash, - fingerprint_metadata_json=metadata_json, - reason=reason, - ) - ) - return _base_plan_output( - project=project, - scope=scope, - relations=relations, - seed_entries=tuple(entries), - ) - - -def build_relation_command_plan_impl( - *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext -) -> PlanOutput: - """Project only static relation data for Python checks and scenarios.""" - - return _base_plan_output(project=project, scope=scope, relations=relations) - - -def _base_plan_output( - *, - project: CompiledProjectView, - scope: PlannerScope, - relations: PlannerRelationsContext, - execution_order: tuple[CompiledObjectKey, ...] | None = None, - seed_entries: tuple[SeedPlanEntry, ...] = (), - function_entries: tuple[FunctionPlanEntry, ...] = (), - audit_entries: tuple[AuditPlanEntry, ...] = (), - test_entries: tuple[SqlTestPlanEntry, ...] = (), - available_test_case_names: tuple[str, ...] = (), - warnings: tuple[PlanWarning, ...] = (), -) -> PlanOutput: - return PlanOutput( - execution_order=( - execution_order - if execution_order is not None - else tuple(key for key in scope.execution_order if key in scope.selected_keys) - ), - seed_entries=seed_entries, - function_entries=function_entries, - audit_entries=audit_entries, - test_entries=test_entries, - available_test_case_names=available_test_case_names, - selected_keys=scope.selected_keys, - warnings=warnings, - upstream_deps=scope.upstream_deps, - downstream_deps=scope.downstream_deps, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_map, - source_read_map=relations.source_read_map, - python_source_read_map=relations.python_source_read_map, - hook_functions=project.hook_functions, - enforce_explicit_references=project.enforce_explicit_references, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py deleted file mode 100644 index 026081c888..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ /dev/null @@ -1,498 +0,0 @@ -"""PlanOutput assembly helpers.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_model_materializations, - build_relation_fixture_context, - build_selected_audit_entries, - build_source_load_entries, - extract_seed_columns, - plan_sql_tests, - resolve_function_sql, - scope_overlaps, - upstream_loader_dependency_names, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledProjectView, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredProviderUsage, - PythonHookEntry, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.exceptions import ( - PlannerInputError, -) -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - FunctionChangeResult, - FunctionPlanEntry, - ModelPlanEntry, - PlannedSqlTests, - PlannerChangeResults, - PlannerModelEntryResults, - PlannerRelationsContext, - PlannerScope, - PlanOutput, - PlanOutputExtras, - PlanProviderUsage, - PlanWarning, - RelationFixturePlanningContext, - SeedPlanEntry, - SourceLoadPlanEntry, - SqlTestPlanEntry, - SqlTestPlanResult, - SqlTestSelection, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_plan_output( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - changes: PlannerChangeResults, - model_entry_results: PlannerModelEntryResults, - reload_sources: bool, - extras: PlanOutputExtras | None = None, -) -> PlanOutput: - resolved_extras: PlanOutputExtras = extras if extras is not None else PlanOutputExtras() - seed_version_hashes: dict[str, str] | None = resolved_extras.seed_version_hashes - seed_metadata_jsons: dict[str, str] | None = resolved_extras.seed_metadata_jsons - seed_plan_reasons: dict[str, PlanReason] | None = resolved_extras.seed_plan_reasons - seed_entries: list[SeedPlanEntry] = [ - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=extract_seed_columns(seed), - csv_settings=seed.schema_entry.csv_settings, - fingerprint_definition=(seed_metadata_jsons or {}).get(seed.name, ""), - fingerprint_version_hash=(seed_version_hashes or {}).get(seed.name, ""), - fingerprint_metadata_json=(seed_metadata_jsons or {}).get(seed.name, "{}"), - reason=_resolve_seed_plan_reason( - seed_name=seed.name, - expected_version_hash=(seed_version_hashes or {}).get(seed.name), - snapshot=snapshot, - seed_plan_reasons=seed_plan_reasons, - ), - ) - for seed in project.seeds - if seed.key in scope.selected_keys - ] - source_load_entries: tuple[SourceLoadPlanEntry, ...] = build_source_load_entries( - execution_order=scope.execution_order, - selected_keys=scope.selected_keys, - source_map=relations.source_map, - is_reload=reload_sources, - ) - _validate_skipped_intermediate_loader_targets( - project=project, - snapshot=snapshot, - relations=relations, - source_load_entries=source_load_entries, - ) - function_entries: list[FunctionPlanEntry] = _build_function_entries( - project=project, - adapter=adapter, - snapshot=snapshot, - relations=relations, - changes=changes, - selected_keys=scope.selected_keys, - ) - model_materializations: dict[str, str] = build_model_materializations( - model_entry_results.entries - ) - audit_entries: list[AuditPlanEntry] = build_selected_audit_entries( - project=project, - adapter=adapter, - scope=scope, - relations=relations, - model_materializations=model_materializations, - ) - planned_tests: PlannedSqlTests | None = resolved_extras.planned_sql_tests - if ( - planned_tests is None - or planned_tests.selected_keys != scope.selected_keys - or planned_tests.sql_test_selection != scope.sql_test_selection - ): - planned_tests = plan_selected_sql_tests( - project=project, - adapter=adapter, - selected_keys=scope.selected_keys, - sql_test_selection=scope.sql_test_selection, - ) - test_entries: list[SqlTestPlanEntry] = list(planned_tests.entries) - test_warnings: list[PlanWarning] = list(planned_tests.warnings) - selected_test_keys: frozenset[CompiledObjectKey] = frozenset( - entry.key for entry in test_entries - ) - scoped_keys: frozenset[CompiledObjectKey] = scope.selected_keys | selected_test_keys - scoped_order: tuple[CompiledObjectKey, ...] = tuple( - k for k in scope.execution_order if k in scoped_keys - ) - return PlanOutput( - execution_order=scoped_order, - model_entries=model_entry_results.entries, - seed_entries=tuple(seed_entries), - source_load_entries=tuple(source_load_entries), - function_entries=tuple(function_entries), - audit_entries=tuple(audit_entries), - test_entries=tuple(test_entries), - selected_keys=scope.selected_keys, - warnings=(*model_entry_results.warnings, *test_warnings), - upstream_deps=scope.upstream_deps, - downstream_deps=scope.downstream_deps, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_map, - source_read_map=relations.source_read_map, - python_source_read_map=relations.python_source_read_map, - hook_functions=project.hook_functions, - enforce_explicit_references=project.enforce_explicit_references, - provider_usages=_build_provider_usages( - project=project, - model_entries=model_entry_results.entries, - source_load_entries=source_load_entries, - ), - python_identity_fingerprints=snapshot.fingerprints.python_nodes, - ) - - -def _resolve_seed_plan_reason( - *, - seed_name: str, - expected_version_hash: str | None, - snapshot: WarehouseSnapshot, - seed_plan_reasons: dict[str, PlanReason] | None = None, -) -> PlanReason: - reason: PlanReason | None = (seed_plan_reasons or {}).get(seed_name) - if reason is not None: - return reason - if seed_name not in snapshot.existing_relations: - return PlanReason.FIRST_RUN - fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) - if fingerprint is None: - return PlanReason.FIRST_RUN - if expected_version_hash is not None and fingerprint.version_hash != expected_version_hash: - return PlanReason.CONFIG_CHANGED - return PlanReason.NO_CHANGE - - -def _build_provider_usages( - *, - project: CompiledProjectView, - model_entries: tuple[ModelPlanEntry, ...], - source_load_entries: tuple[SourceLoadPlanEntry, ...], -) -> tuple[PlanProviderUsage, ...]: - usages: list[PlanProviderUsage] = [] - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - source_load_entry: SourceLoadPlanEntry - for source_load_entry in source_load_entries: - loader: DiscoveredLoaderFunction | None = loader_by_name.get(source_load_entry.loader) - if loader is None: - continue - usages.extend( - _to_plan_provider_usages( - provider_usages=loader.provider_usages, - consumer_kind="loader", - consumer_name=loader.name, - ) - ) - - hook_by_name: dict[str, DiscoveredHookFunction] = { - hook.name: hook for hook in project.hook_functions - } - materialization_by_name: dict[str, DiscoveredMaterializationFile] = { - materialization.name: materialization for materialization in project.materialization_files - } - model_entry: ModelPlanEntry - for model_entry in model_entries: - usages.extend( - _hook_provider_usages( - hook_entries=(model_entry.pre_hooks, model_entry.post_hooks), - hook_by_name=hook_by_name, - ) - ) - if model_entry.custom_materialization_name is None: - continue - materialization: DiscoveredMaterializationFile | None = materialization_by_name.get( - model_entry.custom_materialization_name - ) - if materialization is None: - continue - usages.extend( - _to_plan_provider_usages( - provider_usages=materialization.provider_usages, - consumer_kind="custom materialization", - consumer_name=materialization.name, - ) - ) - return tuple(usages) - - -def _hook_provider_usages( - *, - hook_entries: tuple[object, ...], - hook_by_name: dict[str, DiscoveredHookFunction], -) -> tuple[PlanProviderUsage, ...]: - usages: list[PlanProviderUsage] = [] - hooks: list[object] = [] - hook_entry: object - for hook_entry in hook_entries: - if hook_entry is None: - continue - if isinstance(hook_entry, list | tuple): - hooks.extend(hook_entry) - else: - hooks.append(hook_entry) - hook: object - for hook in hooks: - if not isinstance(hook, PythonHookEntry): - continue - discovered_hook: DiscoveredHookFunction | None = hook_by_name.get(hook.name) - if discovered_hook is None: - continue - usages.extend( - _to_plan_provider_usages( - provider_usages=discovered_hook.provider_usages, - consumer_kind="hook", - consumer_name=discovered_hook.name, - ) - ) - return tuple(usages) - - -def _to_plan_provider_usages( - *, - provider_usages: tuple[DiscoveredProviderUsage, ...], - consumer_kind: str, - consumer_name: str, -) -> tuple[PlanProviderUsage, ...]: - return tuple( - PlanProviderUsage( - provider_name=usage.provider_name, - consumer_kind=consumer_kind, - consumer_name=consumer_name, - parameter_name=usage.parameter_name, - annotation_class_name=usage.annotation_class_name, - annotation_module=usage.annotation_module, - ) - for usage in provider_usages - ) - - -def _validate_skipped_intermediate_loader_targets( - *, - project: CompiledProjectView, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - source_load_entries: tuple[SourceLoadPlanEntry, ...], -) -> None: - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - selected_load_names: frozenset[str] = frozenset(entry.name for entry in source_load_entries) - entry: SourceLoadPlanEntry - for entry in source_load_entries: - source_entry: SourceEntry | None = relations.source_map.get(entry.name) - if source_entry is None or source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(source_entry.loader) - if loader_function is None: - continue - dependency_loader_name: str - for dependency_loader_name in upstream_loader_dependency_names( - loader_function=loader_function, - loader_functions=project.loader_functions, - ): - dependency_source: SourceEntry | None = relations.source_map.get(dependency_loader_name) - if dependency_source is None or dependency_source.name in selected_load_names: - continue - dependency_target: str = dependency_source.table or dependency_source.name - if dependency_target in snapshot.existing_relations: - continue - raise PlannerInputError( - f"Source '{source_entry.name}' requires intermediate loader " - f"'{dependency_loader_name}', but its target relation " - f"'{dependency_target}' does not exist; use +source:{source_entry.name} " - "to refresh upstream ingress dependencies" - ) - - -def _build_function_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - changes: PlannerChangeResults, - selected_keys: frozenset[CompiledObjectKey], -) -> list[FunctionPlanEntry]: - entries: list[FunctionPlanEntry] = [] - function: CompiledFunctionView - for function in project.functions: - if function.key not in selected_keys: - continue - function_change: FunctionChangeResult = changes.functions[function.name] - entries.append( - plan_function( - function=function, - adapter=adapter, - relations=relations, - fingerprint_query_sql=function_change.fingerprint_sql, - previous_query_sql=( - snapshot.fingerprints.functions[function.name].definition - if function.name in snapshot.fingerprints.functions - else None - ), - reason=function_change.reason, - ) - ) - return entries - - -def plan_function( - *, - function: CompiledFunctionView, - adapter: BaseAdapter, - relations: PlannerRelationsContext, - fingerprint_query_sql: str, - previous_query_sql: str | None = None, - reason: PlanReason = PlanReason.NO_CHANGE, -) -> FunctionPlanEntry: - """Build one canonical function entry from resolved relation inputs.""" - - return FunctionPlanEntry( - key=function.key, - name=function.name, - relative_path=function.relative_path, - destination=function.destination, - arguments=function.arguments, - returns=function.returns, - body_sql=resolve_function_sql( - adapter=adapter, - function=function, - model_locations=relations.model_locations, - seed_locations=relations.seed_locations, - function_locations=relations.function_locations, - source_map=relations.source_read_map, - source_warehouse_columns=relations.source_warehouse_columns, - star_exclude_keyword=relations.star_exclude_keyword, - ), - fingerprint_query_sql=fingerprint_query_sql, - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - previous_query_sql=previous_query_sql, - reason=reason, - ) - - -def plan_selected_sql_tests( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, -) -> PlannedSqlTests: - """Plan SQL tests selected by name or overlapping one selection into a reusable result.""" - - entries: list[SqlTestPlanEntry] - warnings: list[PlanWarning] - entries, warnings = build_selected_test_entries( - project=project, - adapter=adapter, - selected_keys=selected_keys, - sql_test_selection=sql_test_selection, - ) - return PlannedSqlTests( - selected_keys=selected_keys, - sql_test_selection=sql_test_selection, - entries=tuple(entries), - warnings=tuple(warnings), - ) - - -def sql_test_is_selected( - *, - test: CompiledSqlTest, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, -) -> bool: - """Whether a test is named by the selection or covers a selected resource, and not excluded.""" - - name: str = test.parent_name or test.name - if name in sql_test_selection.excluded_names: - return False - return name in sql_test_selection.names or scope_overlaps( - scope_deps=test.scope_deps, selected_keys=selected_keys - ) - - -def build_selected_test_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - case_name: str | None = None, -) -> tuple[list[SqlTestPlanEntry], list[PlanWarning]]: - entries: list[SqlTestPlanEntry] = [] - warnings: list[PlanWarning] = [] - fixture_diagnostics: list[str] = [] - selected_tests: list[CompiledSqlTest] = [] - sql_test: CompiledSqlTest - for sql_test in project.sql_tests: - if not sql_test_is_selected( - test=sql_test, selected_keys=selected_keys, sql_test_selection=sql_test_selection - ): - continue - if case_name is not None and sql_test.case_name != case_name: - continue - selected_tests.append(sql_test) - if not selected_tests: - return entries, warnings - fixture_planning_context: RelationFixturePlanningContext | None = ( - build_relation_fixture_context(project=project) if project.settings.sql_analysis else None - ) - result: SqlTestPlanResult - for result in plan_sql_tests( - tests=tuple(selected_tests), - project=project, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - validate_fixtures=True, - fixture_planning_context=fixture_planning_context, - ): - if result.entry is None: - fixture_diagnostics.extend(result.fixture_diagnostics) - continue - entries.append(result.entry) - warnings.extend(result.warnings) - if fixture_diagnostics: - raise PlannerInputError("Invalid SQL test fixtures:\n- " + "\n- ".join(fixture_diagnostics)) - return entries, warnings diff --git a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py b/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py index 661972273d..a368305335 100644 --- a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py +++ b/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py @@ -6,10 +6,10 @@ from concurrent.futures import Future from types import TracebackType +from sqlbuild._native import plan_selected_sql_tests_into from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.output.plan_output import plan_selected_sql_tests from sqlbuild.compiler.planner.models import PlannedSqlTests, SqlTestSelection @@ -30,7 +30,7 @@ def __init__( future: Future[PlannedSqlTests] = Future() self._future = future threading.Thread( - target=_plan_into, + target=plan_selected_sql_tests_into, kwargs={ "future": future, "project": project, @@ -63,26 +63,3 @@ def result(self) -> PlannedSqlTests: selected_keys=self._selected_keys, sql_test_selection=self._sql_test_selection ) return self._future.result() - - -def _plan_into( - *, - future: Future[PlannedSqlTests], - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, -) -> None: - """Run planning on a daemon thread so a failed plan never waits for it at exit.""" - - try: - future.set_result( - plan_selected_sql_tests( - project=project, - adapter=adapter, - selected_keys=selected_keys, - sql_test_selection=sql_test_selection, - ) - ) - except BaseException as error: # noqa: BLE001 - re-raised on the joining thread - future.set_exception(error) diff --git a/src/sqlbuild/compiler/planner/main/commands/__init__.py b/src/sqlbuild/compiler/planner/main/commands/__init__.py deleted file mode 100644 index 2d27e6e963..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Focused command planning entrypoints.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/_audit.py b/src/sqlbuild/compiler/planner/main/commands/_audit.py deleted file mode 100644 index 79e4b27482..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_audit.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public focused audit-plan entrypoint.""" - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import build_audit_command_plan_impl -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_audit_command_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, -) -> PlanOutput: - """Project selected audits from canonical static state.""" - - return build_audit_command_plan_impl( - project=project, adapter=adapter, scope=scope, relations=relations - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py b/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py deleted file mode 100644 index d622979f2c..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_relation_plan.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public relation-only plan entrypoint.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import ( - build_relation_command_plan_impl, -) -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_relation_command_plan( - *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext -) -> PlanOutput: - """Project static relation data for Python checks.""" - - return build_relation_command_plan_impl(project=project, scope=scope, relations=relations) diff --git a/src/sqlbuild/compiler/planner/main/commands/_relations.py b/src/sqlbuild/compiler/planner/main/commands/_relations.py deleted file mode 100644 index bf8fd7840e..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_relations.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Focused-command relation entrypoint.""" - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import ( - resolve_static_relation_context_impl, -) -from sqlbuild.compiler.planner.models import DeferralInputs, PlannerRelationsContext, PlannerScope -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_static_relation_context( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - deferral: DeferralInputs | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, - relation_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerRelationsContext: - """Resolve canonical locations and source routing without warehouse inventory.""" - - return resolve_static_relation_context_impl( - project=project, - adapter=adapter, - scope=scope, - deferral=deferral, - project_config=project_config, - local_config=local_config, - relation_keys=relation_keys, - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_scope.py b/src/sqlbuild/compiler/planner/main/commands/_scope.py deleted file mode 100644 index e2b3c221f1..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_scope.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Focused-command scope entrypoint.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import ( - resolve_static_command_scope_impl, -) -from sqlbuild.compiler.planner.models import PlannerScope, PlannerSelection - - -def resolve_static_command_scope( - *, project: CompiledProjectView, selection: PlannerSelection, auto_load_sources: bool = False -) -> PlannerScope: - """Resolve command selection through the canonical build selector phase.""" - - return resolve_static_command_scope_impl( - project=project, selection=selection, auto_load_sources=auto_load_sources - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_seed.py b/src/sqlbuild/compiler/planner/main/commands/_seed.py deleted file mode 100644 index 5f6aee03b2..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_seed.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public focused seed-plan entrypoint.""" - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.command_planning.static import build_seed_command_plan_impl -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_seed_command_plan( - *, - project: CompiledProjectView, - scope: PlannerScope, - relations: PlannerRelationsContext, - fingerprints: dict[str, Fingerprint] | None = None, -) -> PlanOutput: - """Project selected direct seed work and canonical identities.""" - - return build_seed_command_plan_impl( - project=project, scope=scope, relations=relations, fingerprints=fingerprints - ) diff --git a/src/sqlbuild/compiler/planner/main/commands/_seed_state.py b/src/sqlbuild/compiler/planner/main/commands/_seed_state.py deleted file mode 100644 index 4c13face97..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_seed_state.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public targeted seed-state read entrypoint.""" - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.command_planning.seed_state import read_seed_fingerprints - - -def read_selected_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] -) -> dict[str, Fingerprint]: - """Read only fingerprint state required by selected direct seeds.""" - - return read_seed_fingerprints(adapter=adapter, connection=connection, seeds=seeds) diff --git a/src/sqlbuild/compiler/planner/main/commands/_sql_test.py b/src/sqlbuild/compiler/planner/main/commands/_sql_test.py deleted file mode 100644 index 089f16569a..0000000000 --- a/src/sqlbuild/compiler/planner/main/commands/_sql_test.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public focused SQL-test plan entrypoint.""" - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.command_planning.static import build_test_command_plan_impl -from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput - - -def build_test_command_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - relations: PlannerRelationsContext, - case_name: str | None = None, -) -> PlanOutput: - """Project selected SQL tests from canonical static state.""" - - return build_test_command_plan_impl( - project=project, - adapter=adapter, - scope=scope, - relations=relations, - case_name=case_name, - ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 3a910007b9..ea7d115776 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -9,12 +9,12 @@ build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, + build_selected_test_entries, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.models import ( RelationFixtureCompletion, RelationFixturePlanningContext, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 5761e52fc5..b6c625c5b3 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,8 +6,11 @@ import pytest from sqlbuild._native import ( + build_test_command_plan, plan_compiled_sql_test_artifacts, project_with, + resolve_static_command_scope, + resolve_static_relation_context, sql_test_plan_error_messages, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -27,9 +30,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context -from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( ChainStep, NativeSqlTestPlan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 76098d4a9a..832fda88be 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -8,6 +8,7 @@ import sqlbuild._native as sql_test_assembly from sqlbuild import _native as fixture_completion from sqlbuild._native import ( + build_selected_test_entries, compiled_model_record, project_with, ) @@ -21,8 +22,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.output import plan_output -from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences from sqlbuild.compiler.planner.models import ( FixtureColumnMetadata, @@ -101,7 +100,7 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is settings=SettingsConfig(sql_analysis=True), ) metadata_builder: Mock = Mock(wraps=fixture_completion.build_relation_fixture_context) - monkeypatch.setattr(plan_output, "build_relation_fixture_context", metadata_builder) + monkeypatch.setattr(fixture_completion, "build_relation_fixture_context", metadata_builder) fixture_completion.fixture_read_cache_info(reset=True) topology_builder: Mock = Mock(wraps=sql_test_assembly.resolve_sql_test_model_chains) monkeypatch.setattr(sql_test_assembly, "resolve_sql_test_model_chains", topology_builder) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py index 83600a755c..02fa4d0a77 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py @@ -4,7 +4,7 @@ import pytest -import sqlbuild.compiler.planner.classes.background_sql_test_planning as background_module +import sqlbuild._native as native_module from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( @@ -55,7 +55,7 @@ def plan( calls.append(selected_keys) return _PLANNED - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) with BackgroundSqlTestPlanning( sql_test_selection=SqlTestSelection(), @@ -97,7 +97,7 @@ def plan( calls.append(selected_keys) return _PLANNED - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) with BackgroundSqlTestPlanning( sql_test_selection=SqlTestSelection(), @@ -137,7 +137,7 @@ def plan( del project, adapter, selected_keys, sql_test_selection raise PlannerInputError(str(test_case.expected_error)) - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) with BackgroundSqlTestPlanning( sql_test_selection=SqlTestSelection(), @@ -179,7 +179,7 @@ def plan( release.wait(timeout=5) return PlannedSqlTests(selected_keys=selected_keys) - monkeypatch.setattr(background_module, "plan_selected_sql_tests", plan) + monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) with BackgroundSqlTestPlanning( sql_test_selection=SqlTestSelection(), diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 1107511c5a..0398232fca 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -10,19 +10,19 @@ import pytest from sqlbuild._native import ( + build_audit_command_plan, + build_seed_command_plan, + build_test_command_plan, compiled_model_record, project_with, + read_selected_seed_fingerprints, + resolve_static_command_scope, + resolve_static_relation_context, ) from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.main.commands._audit import build_audit_command_plan -from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context -from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope -from sqlbuild.compiler.planner.main.commands._seed import build_seed_command_plan -from sqlbuild.compiler.planner.main.commands._seed_state import read_selected_seed_fingerprints -from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, From 9f01c322c3adb91baf54c1fbbc08e560604afdf2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:19:11 +0100 Subject: [PATCH 03/33] refactor: host clone planning and planner entry facades natively Port clone planning, deferred-clone graph traversal and SQL test model chain grouping to native code, and route the remaining planner facade modules (scenarios, selection, SQL test artifacts, fixture helpers) directly to sqlbuild._native. --- .../_helpers/build_command/defer_clone.rs | 6 +- .../bindings/_helpers/clone/clone_pipeline.rs | 2 +- .../_helpers/compile_command/target_writer.rs | 5 +- .../compile_driver/selection_phases.rs | 3 +- .../_helpers/planner_entries/clone_plans.rs | 661 ++++++++++++++++++ .../bindings/_helpers/planner_entries/mod.rs | 1 + .../_helpers/planner_entries/plan_output.rs | 21 + .../_helpers/planner_entries/registration.rs | 1 + .../_helpers/planner_entries/values.rs | 1 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 38 + .../compiler/planner/_helpers/__init__.py | 1 - .../planner/_helpers/fixtures/__init__.py | 0 .../_helpers/fixtures/empty_fixture.py | 3 - .../planner/_helpers/graph/defer_clone.py | 71 -- .../planner/_helpers/migrations/__init__.py | 0 .../compiler/planner/_helpers/reuse/clone.py | 325 --------- .../_helpers/sql_tests/fixture_formatting.py | 3 - .../planner/classes/fixture_null_autofix.py | 4 +- .../compiler/planner/main/clone/__init__.py | 0 .../compiler/planner/main/clone/_clone.py | 114 --- .../main/clone/resolve_clone_boundary.py | 24 - .../main/clone/resolve_skipped_view_chain.py | 24 - .../_plan_compiled_sql_test_artifacts.py | 26 - .../main/execution/sql_test_model_chain.py | 20 - .../planner/main/scenarios/__init__.py | 0 .../_build_scenario_artifact_physical_name.py | 20 - .../is_scenario_artifact_physical_name.py | 13 - .../planner/main/scenarios/scenario.py | 29 - .../planner/main/selection/__init__.py | 0 .../main/selection/_resolve_planner_scopes.py | 26 - .../janitor/_helpers/classification.py | 4 +- .../executor/pipeline/_helpers/scenario.py | 2 +- .../compile/test_compile_target_writer.py | 6 +- .../_helpers/test_scenario_artifacts.py | 4 +- .../main/test_defer_clone_traversal.py | 7 +- .../lint/test_adapter_lexical_syntax.py | 2 +- 37 files changed, 738 insertions(+), 731 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/reuse/clone.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py delete mode 100644 src/sqlbuild/compiler/planner/main/clone/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/clone/_clone.py delete mode 100644 src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py delete mode 100644 src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py delete mode 100644 src/sqlbuild/compiler/planner/main/scenarios/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py delete mode 100644 src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py delete mode 100644 src/sqlbuild/compiler/planner/main/scenarios/scenario.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 0bb5ab41d2..8f5e3fbf08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -21,8 +21,6 @@ const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const CLONE_TYPES: &str = "sqlbuild.executor.clone.types"; const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; -const CLONE_BOUNDARY_MODULE: &str = "sqlbuild.compiler.planner.main.clone.resolve_clone_boundary"; -const VIEW_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain"; const TRANSIENT_MODELS_MODULE: &str = "sqlbuild.executor.clone.main._build_destination_transient_models"; const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; @@ -166,12 +164,12 @@ fn defer_clone_selection(scope: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec Ok(( traversed_names( scope, - (CLONE_BOUNDARY_MODULE, "resolve_clone_boundary"), + (NATIVE_MODULE, "resolve_clone_boundary"), callbacks, )?, traversed_names( scope, - (VIEW_CHAIN_MODULE, "resolve_skipped_view_chain"), + (NATIVE_MODULE, "resolve_skipped_view_chain"), callbacks, )?, selected_executable_model_count(scope)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 1677450c25..98bad823cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const CLONE_PLANNING_MODULE: &str = "sqlbuild.compiler.planner.main.clone._clone"; +const CLONE_PLANNING_MODULE: &str = "sqlbuild._native"; /// The `ClonePipelineResult` fields after the two projects, in `run_clone_planning` order. const PLANNED_FIELDS: [&str; 8] = [ "clone_plan", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 9e6bffcfb9..773c0ff562 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -48,9 +48,8 @@ const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const MODEL_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.execution.sql_test_model_chain"; -const PLAN_ARTIFACTS_MODULE: &str = - "sqlbuild.compiler.planner.main.execution._plan_compiled_sql_test_artifacts"; +const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; +const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 8167475805..11ae284645 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -11,8 +11,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const SPLIT_SELECTORS_MODULE: &str = "sqlbuild._native"; -const PLANNER_SCOPES_MODULE: &str = - "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; +const PLANNER_SCOPES_MODULE: &str = "sqlbuild._native"; const STATIC_SCOPE_MODULE: &str = "sqlbuild._native"; const STATIC_RELATIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs new file mode 100644 index 0000000000..514364147f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -0,0 +1,661 @@ +//! Clone planning: destination and origin entries, and deferred-clone graph traversal. + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::planner_entries::values::{ + COMPILE_MODELS, native, planner_model, planner_type, string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const VIEW_MATERIALIZATION: &str = "view"; +const GRAPH_ARGUMENTS: [&str; 4] = ["selected", "upstream", "is_clonable", "is_view"]; + +/// Whether a model is explicitly disabled with a boolean `enabled = false`. +fn disabled_model(model: &Bound<'_, PyAny>) -> PyResult { + let raw = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("enabled",))?; + Ok(raw.is_instance_of::() && !raw.is_truthy()?) +} + +fn tuple_of<'py>(py: Python<'py>, items: Vec>) -> PyResult> { + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// Entries keyed by node key, returned in `execution_order`. +fn in_order<'py>( + entries: &Bound<'py, PyDict>, + execution_order: &Bound<'py, PyAny>, +) -> PyResult> { + let mut ordered: Vec> = Vec::new(); + for key in execution_order.try_iter()? { + if let Some(entry) = entries.get_item(key?)? { + ordered.push(entry); + } + } + tuple_of(entries.py(), ordered) +} + +fn clone_plan_output<'py>( + project: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let upstream = + imported(py, "sqlbuild._native", "build_execution_upstream_deps")?.call1((project,))?; + let downstream = + imported(py, "sqlbuild._native", "build_downstream_deps")?.call1((&upstream,))?; + let selected = native( + py, + "resolve_graph_selection", + &[ + ("graph", project.getattr("lineage_graph")?), + ("select", select.clone()), + ("exclude", exclude.clone()), + ], + )?; + planner_model( + py, + "PlanOutput", + &[ + ( + "execution_order", + native( + py, + "topologically_order_keys", + &[("upstream", upstream.clone())], + )?, + ), + ("selected_keys", selected), + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ], + ) +} + +/// Locations and source columns that destination SQL resolves against. +struct CloneResolution<'py> { + model_locations: Bound<'py, PyAny>, + seed_locations: Bound<'py, PyAny>, + function_locations: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, + source_warehouse_columns: Bound<'py, PyAny>, +} + +fn clone_resolution<'py>( + project: &Bound<'py, PyAny>, + source_project: &Bound<'py, PyAny>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py = project.py(); + let model_locations = native( + py, + "build_model_locations", + &[("models", project.getattr("models")?)], + )?; + let seed_locations = native( + py, + "build_seed_locations", + &[("seeds", project.getattr("seeds")?)], + )?; + let function_locations = native( + py, + "build_function_locations", + &[("functions", project.getattr("functions")?)], + )?; + let source_map = PyDict::new(py); + for source in source_project.getattr("sources")?.try_iter()? { + let source = source?; + source_map.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; + } + let source_warehouse_columns = native( + py, + "gather_source_columns", + &[ + ("project", source_project.clone()), + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ], + )?; + Ok(CloneResolution { + model_locations, + seed_locations, + function_locations, + source_map: source_map.into_any(), + source_warehouse_columns, + }) +} + +fn model_entry<'py>( + model: &Bound<'py, PyAny>, + materialization_type: Bound<'py, PyAny>, + resolved_sql: Bound<'py, PyAny>, +) -> PyResult> { + let py = model.py(); + let action = if materialization_type.eq(VIEW_MATERIALIZATION)? { + "CREATE_VIEW" + } else { + "CREATE_TABLE" + }; + planner_model( + py, + "ModelPlanEntry", + &[ + ("key", model.getattr("key")?), + ("name", model.getattr("name")?), + ("relative_path", model.getattr("relative_path")?), + ("materialization_type", materialization_type), + ("action", planner_type(py, "PlanAction", action)?), + ("reason", planner_type(py, "PlanReason", "NO_CHANGE")?), + ("destination", model.getattr("destination")?), + ("fingerprint_query_sql", string(py, "")), + ("resolved_sql", resolved_sql), + ("logical_ddl", string(py, "")), + ], + ) +} + +fn clone_model_entries<'py>( + (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_project: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let resolution = clone_resolution(project, source_project, (adapter, connection))?; + let selected = plan.getattr("selected_keys")?; + let entries = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if !selected.contains(model.getattr("key")?)? || disabled_model(&model)? { + continue; + } + let materialization_type = + native(py, "get_materialization_type", &[("model", model.clone())])?; + let resolved_sql = if materialization_type.eq(VIEW_MATERIALIZATION)? { + let functions = PyDict::new(py); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + functions.set_item(function.getattr("name")?, function)?; + } + let context = planner_model( + py, + "ModelPlanContext", + &[ + ("model_locations", resolution.model_locations.clone()), + ("models_by_name", PyDict::new(py).into_any()), + ("functions_by_name", functions.into_any()), + ("seed_locations", resolution.seed_locations.clone()), + ("function_locations", resolution.function_locations.clone()), + ("source_map", resolution.source_map.clone()), + ( + "source_warehouse_columns", + resolution.source_warehouse_columns.clone(), + ), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?; + native( + py, + "resolve_model_sql", + &[ + ("adapter", adapter.clone()), + ("model", model.clone()), + ("snapshot", planner_model(py, "WarehouseSnapshot", &[])?), + ("context", context), + ( + "backfill", + planner_model( + py, + "BackfillResult", + &[( + "action", + planner_type(py, "BackfillAction", "FORWARD_ONLY")?, + )], + )?, + ), + ("full_refresh", PyBool::new(py, false).to_owned().into_any()), + ( + "cursor_overrides", + planner_model(py, "CursorOverridePair", &[])?, + ), + ], + )? + } else { + string(py, "") + }; + entries.set_item( + model.getattr("key")?, + model_entry(&model, materialization_type, resolved_sql)?, + )?; + } + in_order(&entries, &plan.getattr("execution_order")?) +} + +fn clone_source_entries<'py>( + project: &Bound<'py, PyAny>, + (selected, execution_order): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let entries = PyDict::new(py); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + let entry = source.getattr("source_entry")?; + if !selected.contains(source.getattr("key")?)? + || !entry.getattr("managed")?.is_truthy()? + || entry.getattr("loader")?.is_none() + || !entry.getattr("expression")?.is_none() + { + continue; + } + let table = entry.getattr("table")?; + let relation_name = if table.is_truthy()? { + table + } else { + entry.getattr("name")? + }; + let kwargs = PyDict::new(py); + kwargs.set_item("database", entry.getattr("database")?)?; + kwargs.set_item("schema", entry.getattr("schema")?)?; + kwargs.set_item("name", &relation_name)?; + let qualified = adapter + .getattr("render_qualified_name")? + .call((), Some(&kwargs))?; + let destination = imported(py, COMPILE_MODELS, "CompiledRelationLocation")?.call( + (), + Some(&{ + let location = PyDict::new(py); + location.set_item("database", entry.getattr("database")?)?; + location.set_item("schema", entry.getattr("schema")?)?; + location.set_item("name", relation_name)?; + location.set_item("qualified_name", qualified)?; + location + }), + )?; + entries.set_item( + source.getattr("key")?, + planner_model( + py, + "CloneSourcePlanEntry", + &[ + ("key", source.getattr("key")?), + ("name", source.getattr("name")?), + ("destination", destination), + ], + )?, + )?; + } + in_order(&entries, execution_order) +} + +fn clone_function_entries<'py>( + (project, plan): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_project: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let resolution = clone_resolution(project, source_project, (adapter, connection))?; + let selected = plan.getattr("selected_keys")?; + let entries = PyDict::new(py); + for function in project.getattr("functions")?.try_iter()? { + let function = function?; + if !selected.contains(function.getattr("key")?)? { + continue; + } + let body_sql = native( + py, + "resolve_function_sql", + &[ + ("adapter", adapter.clone()), + ("function", function.clone()), + ("model_locations", resolution.model_locations.clone()), + ("seed_locations", resolution.seed_locations.clone()), + ("function_locations", resolution.function_locations.clone()), + ("source_map", resolution.source_map.clone()), + ( + "source_warehouse_columns", + resolution.source_warehouse_columns.clone(), + ), + ( + "star_exclude_keyword", + adapter.call_method0("star_exclude_keyword")?, + ), + ], + )?; + let field = |name: &str| function.getattr(name); + let entry = planner_model( + py, + "FunctionPlanEntry", + &[ + ("key", field("key")?), + ("name", field("name")?), + ("relative_path", field("relative_path")?), + ("destination", field("destination")?), + ("arguments", field("arguments")?), + ("returns", field("returns")?), + ("body_sql", body_sql), + ( + "fingerprint_query_sql", + native( + py, + "build_compiled_function_fingerprint_sql", + &[("function", function.clone())], + )?, + ), + ("fingerprint_destination", field("fingerprint_destination")?), + ("return_columns", field("return_columns")?), + ("language", field("language")?), + ("source_file_path", field("source_file_path")?), + ("runtime_version", field("runtime_version")?), + ("entry_point", field("entry_point")?), + ("packages", field("packages")?), + ], + )?; + entries.set_item(function.getattr("key")?, entry)?; + } + in_order(&entries, &plan.getattr("execution_order")?) +} + +fn seed_entry<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py = seed.py(); + planner_model( + py, + "SeedPlanEntry", + &[ + ("key", seed.getattr("key")?), + ("name", seed.getattr("name")?), + ("destination", seed.getattr("destination")?), + ( + "file_path", + seed.getattr("seed_file")?.getattr("file_path")?, + ), + ("columns", PyTuple::empty(py).into_any()), + ( + "csv_settings", + seed.getattr("schema_entry")?.getattr("csv_settings")?, + ), + ], + ) +} + +/// Seeds whose `field` (the key or the name) is in `wanted`, as clone seed entries. +fn seed_entries<'py>( + project: &Bound<'py, PyAny>, + field: &str, + wanted: &Bound<'py, PyAny>, +) -> PyResult> { + let mut entries: Vec> = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed = seed?; + if wanted.contains(seed.getattr(field)?)? { + entries.push(seed_entry(&seed)?); + } + } + tuple_of(project.py(), entries) +} + +fn origin_model_entries<'py>( + project: &Bound<'py, PyAny>, + selected_names: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut entries: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if !selected_names.contains(model.getattr("name")?)? || disabled_model(&model)? { + continue; + } + let materialization_type = + native(py, "get_materialization_type", &[("model", model.clone())])?; + entries.push(model_entry(&model, materialization_type, string(py, ""))?); + } + tuple_of(py, entries) +} + +fn names_of<'py>(entries: &Bound<'py, PyAny>) -> PyResult> { + let mut names: Vec> = Vec::new(); + for entry in entries.try_iter()? { + names.push(entry?.getattr("name")?); + } + Ok(PyFrozenSet::new(entries.py(), names)?.into_any()) +} + +fn origin_source_entries<'py>( + project: &Bound<'py, PyAny>, + selected_names: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let mut keys: Vec> = Vec::new(); + let mut order: Vec> = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if selected_names.contains(source.getattr("name")?)? { + keys.push(source.getattr("key")?); + } + order.push(source.getattr("key")?); + } + clone_source_entries( + project, + ( + &PyFrozenSet::new(py, keys)?.into_any(), + &tuple_of(py, order)?, + ), + adapter, + ) +} + +/// Prepare planner outputs needed for clone execution. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn run_clone_planning<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "run_clone_planning", + kwargs, + ( + &[ + "project", + "select", + "exclude", + "adapter", + "connection", + "origin_project", + "origin_source_project", + "destination_source_project", + ], + &[], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let destination_sources = args.value("destination_source_project")?; + let plan = clone_plan_output(&project, &args.value("select")?, &args.value("exclude")?)?; + let models = clone_model_entries( + (&project, &plan), + (&adapter, &connection), + &destination_sources, + )?; + let seeds = seed_entries(&project, "key", &plan.getattr("selected_keys")?)?; + let functions = clone_function_entries( + (&project, &plan), + (&adapter, &connection), + &destination_sources, + )?; + let mut sources = PyTuple::empty(py).into_any(); + if destination_sources + .getattr("effective_target_name")? + .eq(project.getattr("effective_target_name")?)? + { + sources = clone_source_entries( + &project, + ( + &plan.getattr("selected_keys")?, + &plan.getattr("execution_order")?, + ), + &adapter, + )?; + } + let origin_project = args.value("origin_project")?; + let origin_models = origin_model_entries(&origin_project, &names_of(&models)?)?; + let origin_seeds = seed_entries(&origin_project, "name", &names_of(&seeds)?)?; + let origin_sources = origin_source_entries( + &args.value("origin_source_project")?, + &names_of(&sources)?, + &adapter, + )?; + let origin_by_name = PyDict::new(py); + for entry in origin_sources.try_iter()? { + let entry = entry?; + origin_by_name.set_item(entry.getattr("name")?, entry)?; + } + let mut kept_sources: Vec> = Vec::new(); + let mut kept_origins: Vec> = Vec::new(); + for entry in sources.try_iter()? { + let entry = entry?; + let Some(origin) = origin_by_name.get_item(entry.getattr("name")?)? else { + continue; + }; + if origin + .getattr("destination")? + .ne(entry.getattr("destination")?)? + { + kept_origins.push(origin); + kept_sources.push(entry); + } + } + PyTuple::new( + py, + [ + plan, + tuple_of(py, kept_sources)?, + models, + seeds, + functions, + origin_models, + origin_seeds, + tuple_of(py, kept_origins)?, + ], + ) +} + +/// The traversal callbacks and graph of one deferred-clone query. +struct CloneGraph<'py> { + selected: Bound<'py, PyAny>, + upstream: Bound<'py, PyAny>, + is_clonable: Bound<'py, PyAny>, + is_view: Bound<'py, PyAny>, +} + +impl<'py> CloneGraph<'py> { + fn read(args: &KeywordArguments<'py>) -> PyResult { + Ok(Self { + selected: args.value("selected")?, + upstream: args.value("upstream")?, + is_clonable: args.value("is_clonable")?, + is_view: args.value("is_view")?, + }) + } + + fn parents(&self, node: &Bound<'py, PyAny>) -> PyResult>> { + self.upstream + .call_method1("get", (node, PyTuple::empty(node.py())))? + .try_iter()? + .collect() + } + + fn holds(&self, check: &Bound<'py, PyAny>, node: &Bound<'py, PyAny>) -> PyResult { + check.call1((node,))?.is_truthy() + } + + /// Depth-first walk from every selected node in Python's recursion order. + fn walk(&self, skipped_views: bool) -> PyResult> { + let py = self.selected.py(); + let visited = PySet::empty(py)?; + let found = PySet::empty(py)?; + for start in self.selected.try_iter()? { + let start = start?; + if visited.contains(&start)? { + continue; + } + visited.add(&start)?; + let mut stack: Vec<(Vec>, usize)> = vec![(self.parents(&start)?, 0)]; + while let Some((parents, index)) = stack.last_mut() { + let Some(parent) = parents.get(*index).cloned() else { + stack.pop(); + continue; + }; + *index += 1; + let descend = if self.selected.contains(&parent)? { + true + } else if skipped_views { + let view = self.holds(&self.is_view, &parent)?; + if view && self.holds(&self.is_clonable, &parent)? { + found.add(&parent)?; + } + view + } else if self.holds(&self.is_view, &parent)? { + true + } else { + if self.holds(&self.is_clonable, &parent)? { + found.add(&parent)?; + } + false + }; + if descend && !visited.contains(&parent)? { + visited.add(&parent)?; + stack.push((self.parents(&parent)?, 0)); + } + } + } + PyFrozenSet::new(py, found.iter()) + } +} + +/// Return non-view clonable ancestors anchoring selected nodes. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_clone_boundary<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_clone_boundary", + kwargs, + (&GRAPH_ARGUMENTS, &[]), + )?; + CloneGraph::read(&args)?.walk(false) +} + +/// Return skipped view ancestors rebuilt over cloned boundaries. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn resolve_skipped_view_chain<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "resolve_skipped_view_chain", + kwargs, + (&GRAPH_ARGUMENTS, &[]), + )?; + CloneGraph::read(&args)?.walk(true) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_clone_planning, module)?)?; + module.add_function(wrap_pyfunction!(resolve_clone_boundary, module)?)?; + module.add_function(wrap_pyfunction!(resolve_skipped_view_chain, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 82b0c1a87c..8ff433c750 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -1,5 +1,6 @@ //! Plan entry and plan output assembly over native planner resolution. +mod clone_plans; mod command_plans; mod model_entries; mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 1fdd330837..c0d6f9153b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -795,6 +795,26 @@ fn plan_selected_sql_tests_into<'py>( Ok(()) } +/// Return model closures resolved natively in one project batch, keyed by SQL test key. +#[pyfunction] +#[pyo3(signature = (*, project, tests))] +fn sql_test_model_chain_names_by_key<'py>( + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let chains = native( + py, + "resolve_sql_test_model_chains", + &[("project", project.clone()), ("tests", tests.clone())], + )?; + let by_key = PyDict::new(py); + for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { + by_key.set_item(test?.getattr("key")?, chain?)?; + } + Ok(by_key) +} + /// Whether a test is named by the selection or covers a selected resource, and not excluded. #[pyfunction] #[pyo3(signature = (*, test, selected_keys, sql_test_selection))] @@ -840,5 +860,6 @@ pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(plan_selected_sql_tests, module)?)?; module.add_function(wrap_pyfunction!(plan_selected_sql_tests_into, module)?)?; module.add_function(wrap_pyfunction!(sql_test_is_selected, module)?)?; + module.add_function(wrap_pyfunction!(sql_test_model_chain_names_by_key, module)?)?; module.add_function(wrap_pyfunction!(build_selected_test_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index d1fe76c103..f87c46e6a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -3,6 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + super::clone_plans::register(module)?; super::command_plans::register(module)?; super::model_entries::register(module)?; super::plan_output::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index 0cbbac663b..694d1742f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -12,6 +12,7 @@ pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exception pub(super) const PLANNER_CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; pub(super) const DISCOVERY_MODELS: &str = "sqlbuild.compiler.discovery.models"; /// `sqlbuild._native.name(**pairs)`. diff --git a/python-boundary.toml b/python-boundary.toml index a70b835915..901904d9ba 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 19798 +remaining_lines = 18084 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a348b73457..d8de92af6b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -212,6 +212,7 @@ from sqlbuild.compiler.planner.models import ( AuditPlanEntry, BackfillResult, ChangeDetectionResult, + CloneSourcePlanEntry, ColumnMigrationPlanEntry, ColumnMigrationPlanning, ColumnRenameHint, @@ -4931,6 +4932,43 @@ def build_relation_command_plan( def read_selected_seed_fingerprints( *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] ) -> dict[str, Fingerprint]: ... +def run_clone_planning( + *, + project: CompiledProjectView, + select: tuple[str, ...], + exclude: tuple[str, ...], + adapter: BaseAdapter, + connection: Any, + origin_project: CompiledProjectView, + origin_source_project: CompiledProjectView, + destination_source_project: CompiledProjectView, +) -> tuple[ + PlanOutput, + tuple[CloneSourcePlanEntry, ...], + tuple[ModelPlanEntry, ...], + tuple[SeedPlanEntry, ...], + tuple[FunctionPlanEntry, ...], + tuple[ModelPlanEntry, ...], + tuple[SeedPlanEntry, ...], + tuple[CloneSourcePlanEntry, ...], +]: ... +def resolve_clone_boundary[K]( + *, + selected: frozenset[K], + upstream: Mapping[K, tuple[K, ...]], + is_clonable: Callable[[K], bool], + is_view: Callable[[K], bool], +) -> frozenset[K]: ... +def resolve_skipped_view_chain[K]( + *, + selected: frozenset[K], + upstream: Mapping[K, tuple[K, ...]], + is_clonable: Callable[[K], bool], + is_view: Callable[[K], bool], +) -> frozenset[K]: ... +def sql_test_model_chain_names_by_key( + *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] +) -> dict[CompiledObjectKey, tuple[str, ...]]: ... def resolve_future_cursor_config( *, model: CompiledModelView, project_config: FutureCursorsConfig | None ) -> FutureCursorsConfig | None: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/__init__.py b/src/sqlbuild/compiler/planner/_helpers/__init__.py deleted file mode 100644 index 3ea78f03da..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Planner helper utilities.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py deleted file mode 100644 index e5f142cbe5..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Recognition for the planner's empty relation fixture marker.""" - -from sqlbuild._native import is_empty_fixture_query as is_empty_fixture_query diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py b/src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py deleted file mode 100644 index ed367376f2..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/defer_clone.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Deferred clone graph traversal implementations.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - - -def resolve_clone_boundary_impl[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return non-view clonable ancestors anchoring selected nodes.""" - - boundary: frozenset[K] = frozenset() - visited: frozenset[K] = frozenset() - - def visit( - *, node: K, visited: frozenset[K], boundary: frozenset[K] - ) -> tuple[frozenset[K], frozenset[K]]: - if node in visited: - return visited, boundary - visited = visited | {node} - upstream_node: K - for upstream_node in upstream.get(node, ()): - if upstream_node in selected or is_view(upstream_node): - visited, boundary = visit(node=upstream_node, visited=visited, boundary=boundary) - elif is_clonable(upstream_node): - boundary = boundary | {upstream_node} - return visited, boundary - - selected_node: K - for selected_node in selected: - visited, boundary = visit(node=selected_node, visited=visited, boundary=boundary) - return boundary - - -def resolve_skipped_view_chain_impl[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return skipped view ancestors rebuilt over cloned boundaries.""" - - views: frozenset[K] = frozenset() - visited: frozenset[K] = frozenset() - - def visit( - *, node: K, visited: frozenset[K], views: frozenset[K] - ) -> tuple[frozenset[K], frozenset[K]]: - if node in visited: - return visited, views - visited = visited | {node} - upstream_node: K - for upstream_node in upstream.get(node, ()): - if upstream_node in selected: - visited, views = visit(node=upstream_node, visited=visited, views=views) - elif is_view(upstream_node): - if is_clonable(upstream_node): - views = views | {upstream_node} - visited, views = visit(node=upstream_node, visited=visited, views=views) - return visited, views - - selected_node: K - for selected_node in selected: - visited, views = visit(node=selected_node, visited=visited, views=views) - return views diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py b/src/sqlbuild/compiler/planner/_helpers/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py deleted file mode 100644 index 782aa273c7..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ /dev/null @@ -1,325 +0,0 @@ -"""Clone planner helper functions.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import ( - build_compiled_function_fingerprint_sql, - build_downstream_deps, - build_execution_upstream_deps, - build_function_locations, - build_model_locations, - build_seed_locations, - gather_source_columns, - get_materialization_type, - resolve_function_sql, - resolve_graph_selection, - resolve_model_sql, - topologically_order_keys, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledSourceView, -) -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CloneSourcePlanEntry, - CursorOverridePair, - FunctionPlanEntry, - ModelPlanContext, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_clone_plan_output( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> PlanOutput: - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream_deps - ) - selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selection( - graph=project.lineage_graph, select=select, exclude=exclude - ) - return PlanOutput( - execution_order=topologically_order_keys(upstream=upstream_deps), - selected_keys=selected_keys, - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - ) - - -def build_clone_model_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, - adapter: BaseAdapter, - connection: Any, - source_project: CompiledProjectView | None = None, -) -> tuple[ModelPlanEntry, ...]: - effective_source_project: CompiledProjectView = source_project or project - model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) - seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - source_map: dict[str, SourceEntry] = { - source.name: source.source_entry for source in effective_source_project.sources - } - source_warehouse_columns: dict[str, tuple[Any, ...]] = gather_source_columns( - project=effective_source_project, - adapter=adapter, - connection=connection, - ) - entries_by_key: dict[CompiledObjectKey, ModelPlanEntry] = {} - model: CompiledModelView - for model in project.models: - if model.key not in plan.selected_keys or is_disabled(model): - continue - materialization_type: MaterializationType = get_materialization_type(model) - resolved_sql: str = "" - if materialization_type == MaterializationType.VIEW: - resolved_sql = resolve_model_sql( - adapter=adapter, - model=model, - snapshot=WarehouseSnapshot(), - context=ModelPlanContext( - model_locations=model_locations, - models_by_name={}, - functions_by_name={function.name: function for function in project.functions}, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), - full_refresh=False, - cursor_overrides=CursorOverridePair(), - ) - entries_by_key[model.key] = ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, - action=( - PlanAction.CREATE_VIEW - if materialization_type == MaterializationType.VIEW - else PlanAction.CREATE_TABLE - ), - reason=PlanReason.NO_CHANGE, - destination=model.destination, - fingerprint_query_sql="", - resolved_sql=resolved_sql, - logical_ddl="", - ) - return tuple(entries_by_key[key] for key in plan.execution_order if key in entries_by_key) - - -def build_clone_source_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, - adapter: BaseAdapter, -) -> tuple[CloneSourcePlanEntry, ...]: - entries_by_key: dict[CompiledObjectKey, CloneSourcePlanEntry] = {} - source: CompiledSourceView - for source in project.sources: - source_entry: SourceEntry = source.source_entry - if ( - source.key not in plan.selected_keys - or not source_entry.managed - or source_entry.loader is None - or source_entry.expression is not None - ): - continue - relation_name: str = source_entry.table or source_entry.name - entries_by_key[source.key] = CloneSourcePlanEntry( - key=source.key, - name=source.name, - destination=CompiledRelationLocation( - database=source_entry.database, - schema=source_entry.schema, - name=relation_name, - qualified_name=adapter.render_qualified_name( - database=source_entry.database, - schema=source_entry.schema, - name=relation_name, - ), - ), - ) - return tuple(entries_by_key[key] for key in plan.execution_order if key in entries_by_key) - - -def build_clone_function_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, - adapter: BaseAdapter, - connection: Any, - source_project: CompiledProjectView | None = None, -) -> tuple[FunctionPlanEntry, ...]: - """Build destination-resolved function definitions for clone recreation.""" - effective_source_project: CompiledProjectView = source_project or project - model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) - seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) - function_locations: dict[str, CompiledRelationLocation] = build_function_locations( - project.functions - ) - source_map: dict[str, SourceEntry] = { - source.name: source.source_entry for source in effective_source_project.sources - } - source_warehouse_columns: dict[str, tuple[Any, ...]] = gather_source_columns( - project=effective_source_project, - adapter=adapter, - connection=connection, - ) - entries_by_key: dict[CompiledObjectKey, FunctionPlanEntry] = {} - function: CompiledFunctionView - for function in project.functions: - if function.key not in plan.selected_keys: - continue - entries_by_key[function.key] = FunctionPlanEntry( - key=function.key, - name=function.name, - relative_path=function.relative_path, - destination=function.destination, - arguments=function.arguments, - returns=function.returns, - body_sql=resolve_function_sql( - adapter=adapter, - function=function, - model_locations=model_locations, - seed_locations=seed_locations, - function_locations=function_locations, - source_map=source_map, - source_warehouse_columns=source_warehouse_columns, - star_exclude_keyword=adapter.star_exclude_keyword(), - ), - fingerprint_query_sql=build_compiled_function_fingerprint_sql(function), - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - return tuple(entries_by_key[key] for key in plan.execution_order if key in entries_by_key) - - -def build_origin_source_entries( - *, - project: CompiledProjectView, - selected_names: frozenset[str], - adapter: BaseAdapter, -) -> tuple[CloneSourcePlanEntry, ...]: - selected_keys: frozenset[CompiledObjectKey] = frozenset( - source.key for source in project.sources if source.name in selected_names - ) - return build_clone_source_entries( - project=project, - plan=PlanOutput( - execution_order=tuple(source.key for source in project.sources), - selected_keys=selected_keys, - ), - adapter=adapter, - ) - - -def build_origin_model_entries( - *, - project: CompiledProjectView, - selected_names: frozenset[str], -) -> tuple[ModelPlanEntry, ...]: - entries: list[ModelPlanEntry] = [] - model: CompiledModelView - for model in project.models: - if model.name not in selected_names or is_disabled(model): - continue - materialization_type: MaterializationType = get_materialization_type(model) - entries.append( - ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, - action=( - PlanAction.CREATE_VIEW - if materialization_type == MaterializationType.VIEW - else PlanAction.CREATE_TABLE - ), - reason=PlanReason.NO_CHANGE, - destination=model.destination, - fingerprint_query_sql="", - resolved_sql="", - logical_ddl="", - ) - ) - return tuple(entries) - - -def build_clone_seed_entries( - *, - project: CompiledProjectView, - plan: PlanOutput, -) -> tuple[SeedPlanEntry, ...]: - return tuple( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=tuple(), - csv_settings=seed.schema_entry.csv_settings, - ) - for seed in project.seeds - if seed.key in plan.selected_keys - ) - - -def build_origin_seed_entries( - *, - project: CompiledProjectView, - selected_names: frozenset[str], -) -> tuple[SeedPlanEntry, ...]: - return tuple( - SeedPlanEntry( - key=seed.key, - name=seed.name, - destination=seed.destination, - file_path=seed.seed_file.file_path, - columns=tuple(), - csv_settings=seed.schema_entry.csv_settings, - ) - for seed in project.seeds - if seed.name in selected_names - ) - - -def is_disabled(model: CompiledModelView) -> bool: - raw: object | None = model.config.values.get("enabled") - return isinstance(raw, bool) and not raw diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py deleted file mode 100644 index df9d703e7e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Contract-aware formatting of SQL test fixture projections.""" - -from sqlbuild._native import format_redundant_fixture_nulls as format_redundant_fixture_nulls diff --git a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py index 16cfc725de..ec4451aec7 100644 --- a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py +++ b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py @@ -4,10 +4,8 @@ from pathlib import Path +from sqlbuild._native import format_redundant_fixture_nulls from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting import ( - format_redundant_fixture_nulls, -) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax diff --git a/src/sqlbuild/compiler/planner/main/clone/__init__.py b/src/sqlbuild/compiler/planner/main/clone/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/clone/_clone.py b/src/sqlbuild/compiler/planner/main/clone/_clone.py deleted file mode 100644 index 795e3fcda5..0000000000 --- a/src/sqlbuild/compiler/planner/main/clone/_clone.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Public clone planning entrypoints.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.reuse.clone import ( - build_clone_function_entries, - build_clone_model_entries, - build_clone_plan_output, - build_clone_seed_entries, - build_clone_source_entries, - build_origin_model_entries, - build_origin_seed_entries, - build_origin_source_entries, -) -from sqlbuild.compiler.planner.models import ( - CloneSourcePlanEntry, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, -) - - -def run_clone_planning( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], - adapter: BaseAdapter, - connection: Any, - origin_project: CompiledProjectView, - origin_source_project: CompiledProjectView, - destination_source_project: CompiledProjectView, -) -> tuple[ - PlanOutput, - tuple[CloneSourcePlanEntry, ...], - tuple[ModelPlanEntry, ...], - tuple[SeedPlanEntry, ...], - tuple[FunctionPlanEntry, ...], - tuple[ModelPlanEntry, ...], - tuple[SeedPlanEntry, ...], - tuple[CloneSourcePlanEntry, ...], -]: - """Prepare planner outputs needed for clone execution.""" - - clone_plan: PlanOutput = build_clone_plan_output( - project=project, - select=select, - exclude=exclude, - ) - destination_model_entries: tuple[ModelPlanEntry, ...] = build_clone_model_entries( - project=project, - plan=clone_plan, - adapter=adapter, - connection=connection, - source_project=destination_source_project, - ) - destination_seed_entries: tuple[SeedPlanEntry, ...] = build_clone_seed_entries( - project=project, - plan=clone_plan, - ) - destination_function_entries: tuple[FunctionPlanEntry, ...] = build_clone_function_entries( - project=project, - plan=clone_plan, - adapter=adapter, - connection=connection, - source_project=destination_source_project, - ) - destination_source_entries: tuple[CloneSourcePlanEntry, ...] = () - if destination_source_project.effective_target_name == project.effective_target_name: - destination_source_entries = build_clone_source_entries( - project=project, - plan=clone_plan, - adapter=adapter, - ) - origin_model_entries: tuple[ModelPlanEntry, ...] = build_origin_model_entries( - project=origin_project, - selected_names=frozenset(entry.name for entry in destination_model_entries), - ) - origin_seed_entries: tuple[SeedPlanEntry, ...] = build_origin_seed_entries( - project=origin_project, - selected_names=frozenset(entry.name for entry in destination_seed_entries), - ) - origin_source_entries: tuple[CloneSourcePlanEntry, ...] = build_origin_source_entries( - project=origin_source_project, - selected_names=frozenset(entry.name for entry in destination_source_entries), - adapter=adapter, - ) - origin_source_entries_by_name: dict[str, CloneSourcePlanEntry] = { - entry.name: entry for entry in origin_source_entries - } - destination_source_entries = tuple( - entry - for entry in destination_source_entries - if (origin_entry := origin_source_entries_by_name.get(entry.name)) is not None - and origin_entry.destination != entry.destination - ) - origin_source_entries = tuple( - origin_source_entries_by_name[entry.name] for entry in destination_source_entries - ) - return ( - clone_plan, - destination_source_entries, - destination_model_entries, - destination_seed_entries, - destination_function_entries, - origin_model_entries, - origin_seed_entries, - origin_source_entries, - ) diff --git a/src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py b/src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py deleted file mode 100644 index b9868c49fe..0000000000 --- a/src/sqlbuild/compiler/planner/main/clone/resolve_clone_boundary.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Deferred clone boundary entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.compiler.planner._helpers.graph.defer_clone import resolve_clone_boundary_impl - - -def resolve_clone_boundary[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return non-view clonable ancestors anchoring selected nodes.""" - - return resolve_clone_boundary_impl( - selected=selected, - upstream=upstream, - is_clonable=is_clonable, - is_view=is_view, - ) diff --git a/src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py b/src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py deleted file mode 100644 index 75e37c97b5..0000000000 --- a/src/sqlbuild/compiler/planner/main/clone/resolve_skipped_view_chain.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Deferred clone skipped-view entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.compiler.planner._helpers.graph.defer_clone import resolve_skipped_view_chain_impl - - -def resolve_skipped_view_chain[K]( - *, - selected: frozenset[K], - upstream: Mapping[K, tuple[K, ...]], - is_clonable: Callable[[K], bool], - is_view: Callable[[K], bool], -) -> frozenset[K]: - """Return skipped view ancestors rebuilt over cloned boundaries.""" - - return resolve_skipped_view_chain_impl( - selected=selected, - upstream=upstream, - is_clonable=is_clonable, - is_view=is_view, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py deleted file mode 100644 index 62e4997dad..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" - -from sqlbuild._native import ( - plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import NativeSqlTestArtifact - - -def plan_compiled_sql_test_artifacts( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], - adapter: BaseAdapter, - sql_analysis_enabled: bool, -) -> tuple[NativeSqlTestArtifact, ...]: - """Plan and render SQL tests natively from the compiled objects, without a JSON request.""" - - return _plan_compiled_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py deleted file mode 100644 index 4412ed1b5c..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public planner entrypoint for batched SQL-test model closure discovery.""" - -from sqlbuild._native import ( - resolve_sql_test_model_chains, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def sql_test_model_chain_names_by_key( - *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] -) -> dict[CompiledObjectKey, tuple[str, ...]]: - """Return model closures resolved natively in one project batch.""" - - return { - test.key: chain - for test, chain in zip( - tests, resolve_sql_test_model_chains(project=project, tests=tests), strict=True - ) - } diff --git a/src/sqlbuild/compiler/planner/main/scenarios/__init__.py b/src/sqlbuild/compiler/planner/main/scenarios/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py deleted file mode 100644 index f1e0241e81..0000000000 --- a/src/sqlbuild/compiler/planner/main/scenarios/_build_scenario_artifact_physical_name.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public scenario artifact physical-name builder entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_scenario_artifact_physical_name as _build_scenario_artifact_physical_name, -) - - -def build_scenario_artifact_physical_name( - *, hash_prefix: str, kind: str, logical_name: str, identifier_limit: int -) -> str: - """Build one deterministic scenario artifact physical relation name.""" - - return _build_scenario_artifact_physical_name( - hash_prefix=hash_prefix, - kind=kind, - logical_name=logical_name, - identifier_limit=identifier_limit, - ) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py b/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py deleted file mode 100644 index bced73f581..0000000000 --- a/src/sqlbuild/compiler/planner/main/scenarios/is_scenario_artifact_physical_name.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public scenario artifact physical-name detection entrypoint.""" - -from __future__ import annotations - -from sqlbuild._native import ( - is_scenario_artifact_physical_name as _is_scenario_artifact_physical_name, -) - - -def is_scenario_artifact_physical_name(name: str) -> bool: - """Return whether a relation name matches SQLBuild's scenario artifact shape.""" - - return _is_scenario_artifact_physical_name(name) diff --git a/src/sqlbuild/compiler/planner/main/scenarios/scenario.py b/src/sqlbuild/compiler/planner/main/scenarios/scenario.py deleted file mode 100644 index b37b5a2a73..0000000000 --- a/src/sqlbuild/compiler/planner/main/scenarios/scenario.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public scenario planner entry point.""" - -from __future__ import annotations - -from sqlbuild._native import build_cli_scenario_plan -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_scenario_plan( - *, - scenario: CompiledSqlScenario, - pipeline_result: CompilePipelineResult, - adapter: BaseAdapter, - project_name: str, - source_lexical_syntax: SqlLexicalSyntax, -) -> ScenarioExecutionPlan: - """Build a scenario execution plan for an external entrypoint.""" - - return build_cli_scenario_plan( - scenario=scenario, - pipeline_result=pipeline_result, - adapter=adapter, - project_name=project_name, - source_lexical_syntax=source_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/__init__.py b/src/sqlbuild/compiler/planner/main/selection/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py b/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py deleted file mode 100644 index c1f60c120f..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Planner scope resolution entrypoint for compiler orchestration.""" - -from __future__ import annotations - -from sqlbuild._native import resolve_planner_scopes as _resolve_planner_scopes -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScopeResolution, - PlannerSelection, -) - - -def resolve_planner_scopes( - *, - project: CompiledProjectView, - selection: PlannerSelection, - policies: PlannerPolicies, -) -> PlannerScopeResolution: - """Resolve selected, stale-warning, and inspection scopes for one plan.""" - - return _resolve_planner_scopes( - project=project, - selection=selection, - policies=policies, - ) diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 2fbb4127c4..63bc113985 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -6,14 +6,12 @@ from fnmatch import fnmatchcase from typing import Any +from sqlbuild._native import is_scenario_artifact_physical_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.planner.main.scenarios.is_scenario_artifact_physical_name import ( - is_scenario_artifact_physical_name, -) from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.diff.models import QueryDiffArtifactInspection diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index a7efed0c55..f6acce603f 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -20,11 +20,11 @@ current_native_progress_projector, scenario_local_type_overrides_for_dialect, ) +from sqlbuild._native import build_cli_scenario_plan as build_scenario_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.main.scenarios.scenario import build_scenario_plan from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 047dd8951e..118615bd48 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -8,6 +8,8 @@ import pytest import sqlbuild._native as _native +import sqlbuild._native as model_chain_module +import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( build_manifest, compiled_model_record, @@ -26,10 +28,6 @@ CompiledResourceType, DiagnosticPhase, ) -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) -from sqlbuild.compiler.planner.main.execution import sql_test_model_chain as model_chain_module from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index f158cefe5b..e106f642a7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -7,6 +7,7 @@ build_scenario_hash_index, build_scenario_relation_map, compute_scenario_hash_prefix, + is_scenario_artifact_physical_name, parse_scenario_artifact_physical_name, ) from sqlbuild.compiler.planner.constants import ( @@ -14,9 +15,6 @@ SCENARIO_PLAN_RELATION_COLLISION, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.scenarios.is_scenario_artifact_physical_name import ( - is_scenario_artifact_physical_name, -) from sqlbuild.compiler.planner.models import ( ParsedScenarioArtifactName, ScenarioArtifactIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py index fe6beb6023..506301703b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_defer_clone_traversal.py @@ -1,11 +1,6 @@ import pytest -from sqlbuild.compiler.planner.main.clone.resolve_clone_boundary import ( - resolve_clone_boundary, -) -from sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain import ( - resolve_skipped_view_chain, -) +from sqlbuild._native import resolve_clone_boundary, resolve_skipped_view_chain from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( CloneBoundaryTestCase, ) diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index c20bee4cfb..1199b16010 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -8,8 +8,8 @@ import pytest +import sqlbuild._native as fixture_formatting_module import sqlbuild._native as native_module -import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module from sqlbuild._native import discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs From 8e2c96a77499d7e79a6e49ed34b7e5a6eb5034fb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:19:42 +0100 Subject: [PATCH 04/33] test: import native SQL test artifact planning directly --- tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py | 4 +--- tests/integration/src/sqlbuild/cli/commands/main/helpers.py | 4 +--- .../cli/commands/main/compile/test_compile_command.py | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index ad3bd2d6d9..90d7db2484 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -30,6 +30,7 @@ import pytest import sqlbuild._native as native_module +import sqlbuild._native as plan_artifacts_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings import sqlbuild.cli.compile_reuse.constants as reuse_constants @@ -65,9 +66,6 @@ ) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 32090718a4..ab7a1ba08f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -14,6 +14,7 @@ import pytest import sqlbuild._native as native_module +import sqlbuild._native as plan_artifacts_module from sqlbuild._native import PlannedStaticSqlTests, PreparedCompileArtifacts from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.exceptions import AdapterUserError @@ -28,9 +29,6 @@ from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 94e79a7794..bd858d132b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -11,14 +11,12 @@ import pytest import sqlbuild._native as discovery_entry +import sqlbuild._native as plan_artifacts_module from sqlbuild._native import run_compile from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.planner.main.execution import ( - _plan_compiled_sql_test_artifacts as plan_artifacts_module, -) from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, CompileCommandTestCase, From 18bab3799322304cae2de57f185c633130f17fe1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:25:50 +0100 Subject: [PATCH 05/33] refactor: host execution plan orchestration natively Port build_execution_plan and background SQL test planning to native code, deleting the Python execution entry point and BackgroundSqlTestPlanning class. --- .../_helpers/build_command/defer_clone.rs | 6 +- .../_helpers/dbt_interop/plan_output.rs | 6 +- .../planner_entries/execution_plan.rs | 422 ++++++++++++++++++ .../bindings/_helpers/planner_entries/mod.rs | 1 + .../_helpers/planner_entries/registration.rs | 1 + src/sqlbuild/_native.pyi | 21 + .../compiler/pipeline/main/_plan_result.py | 2 +- .../classes/background_sql_test_planning.py | 65 --- .../planner/main/execution/execution.py | 255 ----------- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- ..._planning.py => test_sql_test_planning.py} | 38 +- .../sqlbuild/compiler/planner/main/helpers.py | 2 +- 12 files changed, 469 insertions(+), 352 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs delete mode 100644 src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/execution.py rename tests/unit/src/sqlbuild/compiler/planner/classes/{test_background_sql_test_planning.py => test_sql_test_planning.py} (88%) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 8f5e3fbf08..6b41b598c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -162,11 +162,7 @@ fn defer_clone_selection(scope: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec let is_view: Bound<'_, PyAny> = membership(py, &keys, |key| recreated_over_clones(scope, key))?; let callbacks: (&Bound<'_, PyAny>, &Bound<'_, PyAny>) = (&is_clonable, &is_view); Ok(( - traversed_names( - scope, - (NATIVE_MODULE, "resolve_clone_boundary"), - callbacks, - )?, + traversed_names(scope, (NATIVE_MODULE, "resolve_clone_boundary"), callbacks)?, traversed_names( scope, (NATIVE_MODULE, "resolve_skipped_view_chain"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 7e9b95a669..0973c7b46d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -193,11 +193,7 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( )?; let discovered = environment.getattr("discovered_inputs")?; let result = call_with( - &imported( - py, - "sqlbuild.compiler.planner.main.execution.execution", - "build_execution_plan", - )?, + &imported(py, "sqlbuild._native", "build_execution_plan")?, &[ ("project", environment.getattr("project")?), ("adapter", environment.getattr("adapter")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs new file mode 100644 index 0000000000..a7b06a8daa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -0,0 +1,422 @@ +//! Top-level planner orchestration producing an execution plan. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::planner_entries::values::{ + COMPILE_TYPES, boolean, native, planner_model, replaced, string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use std::time::Instant; + +const INSPECTION_CATALOG: (&str, &str) = ( + "sqlbuild.adapter.relations.main.open_inspection_catalog", + "open_inspection_catalog", +); +const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; + +/// A future resolving to the planned SQL tests, planned on a daemon thread when enabled. +fn sql_test_planning<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (selected_keys, sql_test_selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + enabled: bool, +) -> PyResult> { + let py = project.py(); + let future = imported(py, "concurrent.futures", "Future")?.call0()?; + if !enabled { + future.call_method1( + "set_result", + (planner_model( + py, + "PlannedSqlTests", + &[ + ("selected_keys", selected_keys.clone()), + ("sql_test_selection", sql_test_selection.clone()), + ], + )?,), + )?; + return Ok(future); + } + let kwargs = PyDict::new(py); + kwargs.set_item("future", &future)?; + kwargs.set_item("project", project)?; + kwargs.set_item("adapter", adapter)?; + kwargs.set_item("selected_keys", selected_keys)?; + kwargs.set_item("sql_test_selection", sql_test_selection)?; + call_with( + &imported(py, "threading", "Thread")?, + &[ + ( + "target", + imported(py, "sqlbuild._native", "plan_selected_sql_tests_into")?, + ), + ("kwargs", kwargs.into_any()), + ("name", string(py, TEST_PLANNING_THREAD)), + ("daemon", boolean(py, true)?), + ], + )? + .call_method0("start")?; + Ok(future) +} + +/// Selected scopes, and the overrides with cursor overrides typed for the selected models. +fn scopes_and_cursor_overrides<'py>( + args: &KeywordArguments<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let project = args.value("project")?; + let py = project.py(); + let scopes = native( + py, + "resolve_planner_scopes", + &[ + ("project", project.clone()), + ("selection", args.value("selection")?), + ("policies", args.value("policies")?), + ], + )?; + let selected = scopes.getattr("selected_scope")?; + let models_by_name = selected.getattr("models_by_name")?; + let model_type = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("MODEL")?; + let mut models: Vec> = Vec::new(); + for key in selected.getattr("selected_keys")?.try_iter()? { + let key = key?; + let name = key.getattr("name")?; + if key.getattr("resource_type")?.eq(&model_type)? && models_by_name.contains(&name)? { + models.push(models_by_name.get_item(&name)?); + } + } + let overrides = args.value("overrides")?; + let typed = native( + py, + "type_cursor_overrides", + &[ + ("cursor_overrides", overrides.getattr("cursor_overrides")?), + ("selected_models", PyTuple::new(py, models)?.into_any()), + ], + )?; + Ok(( + scopes, + replaced(&overrides, &[("cursor_overrides", typed)])?, + )) +} + +/// Execution and stale-warning change results for the inspected scopes. +fn change_results<'py>( + project: &Bound<'py, PyAny>, + (scopes, snapshot, identities): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + overrides: &Bound<'py, PyAny>, + policies: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py = project.py(); + let versions = identities.getattr("version_identities")?; + let changes = native( + py, + "detect_changes", + &[ + ("project", project.clone()), + ("scope", scopes.getattr("inspection_scope")?), + ("snapshot", snapshot.clone()), + ("full_refresh", overrides.getattr("full_refresh")?), + ( + "expected_version_hashes", + versions.getattr("model_version_hashes")?, + ), + ( + "expected_metadata_jsons", + versions.getattr("model_metadata_jsons")?, + ), + ], + )?; + if !policies.getattr("selection_diagnostics")?.is_truthy()? { + let empty = planner_model( + py, + "PlannerChangeResults", + &[ + ("models", PyDict::new(py).into_any()), + ("functions", PyDict::new(py).into_any()), + ], + )?; + return Ok((changes, empty)); + } + let stale = native( + py, + "detect_stale_warning_changes", + &[ + ("project", project.clone()), + ("scopes", scopes.clone()), + ("snapshot", snapshot.clone()), + ("identities", identities.clone()), + ("execution_changes", changes.clone()), + ], + )?; + Ok((changes, stale)) +} + +/// The plan phases after warehouse inspection, run inside the inspection catalog. +fn planned_output<'py>( + args: &KeywordArguments<'py>, + runtime: &Bound<'py, PyAny>, + (scopes, overrides): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + test_planning: &Bound<'py, PyAny>, +) -> PyResult> { + let project = args.value("project")?; + let py = project.py(); + let deferral = args.value("deferral")?; + let policies = args.value("policies")?; + let warehouse = native( + py, + "gather_planner_warehouse_state", + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("overrides", overrides.clone()), + ("deferral", deferral.clone()), + ], + )?; + let started = Instant::now(); + let identities = native( + py, + "build_planner_identity_context", + &[ + ("project", project.clone()), + ("scopes", scopes.clone()), + ( + "include_stale_warning_identities", + policies.getattr("selection_diagnostics")?, + ), + ], + )?; + native( + py, + "check_selected_scope_buildability", + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("warehouse", warehouse.clone()), + ("deferral", deferral.clone()), + ], + )?; + let snapshot = warehouse.getattr("snapshot")?; + let (changes, stale_warning_changes) = change_results( + &project, + (scopes, &snapshot, &identities), + overrides, + &policies, + )?; + let inspection_scope = scopes.getattr("inspection_scope")?; + let resolved_actions = native( + py, + "resolve_model_actions", + &[ + ("scope", inspection_scope.clone()), + ("changes", changes.clone()), + ], + )?; + let source_freshness = native( + py, + "build_planner_source_freshness_result", + &[ + ("project", project.clone()), + ("adapter", args.value("adapter")?), + ("connection", args.value("connection")?), + ("scope", inspection_scope), + ("relations", warehouse.getattr("inspection_relations")?), + ( + "freshness_state_schemas", + snapshot.getattr("source_freshness_state_schemas")?, + ), + ], + )?; + let pruning = native( + py, + "prune_planner_execution_scope", + &[ + ("scopes", scopes.clone()), + ("resolved_actions", resolved_actions), + ], + )?; + let reconciliation = native( + py, + "reconcile_execution_changes", + &[ + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("pruning", pruning.clone()), + ("changes", changes), + ], + )?; + let entries = native( + py, + "build_planner_entry_results", + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("policies", policies.clone()), + ("deferral", deferral), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation.clone()), + ("source_freshness", source_freshness.clone()), + ], + )?; + let planned_sql_tests = test_planning.call_method0("result")?; + let mut plan_output = native( + py, + "assemble_base_plan_output", + &[ + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("identities", identities.clone()), + ("overrides", overrides.clone()), + ("pruning", pruning.clone()), + ("reconciliation", reconciliation), + ("entries", entries), + ("source_freshness", source_freshness.clone()), + ("planned_sql_tests", planned_sql_tests), + ], + )?; + plan_output = native( + py, + "with_storage_policies", + &[ + ("plan_output", plan_output), + ("runtime", runtime.clone()), + ("warehouse", warehouse.clone()), + ("scopes", scopes.clone()), + ("policies", policies.clone()), + ], + )?; + plan_output = native( + py, + "with_plan_warnings", + &[ + ("runtime", runtime.clone()), + ("scopes", scopes.clone()), + ("warehouse", warehouse), + ("identities", identities), + ("stale_warning_changes", stale_warning_changes), + ("pruning", pruning.clone()), + ("source_freshness", source_freshness.clone()), + ("plan_output", plan_output), + ("policies", policies.clone()), + ], + )?; + plan_output = native( + py, + "with_plan_metadata", + &[ + ("plan_output", plan_output), + ("pruning", pruning), + ("source_freshness", source_freshness), + ("policies", policies), + ], + )?; + let on_progress = args.value("on_progress")?; + if !on_progress.is_none() { + on_progress.call1((format!( + "Generated plan. ({:.2}s)", + started.elapsed().as_secs_f64() + ),))?; + } + Ok(plan_output) +} + +/// Build the execution plan for one selection against live warehouse state. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn build_execution_plan<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_execution_plan", + kwargs, + ( + &[ + "project", + "adapter", + "connection", + "selection", + "overrides", + "deferral", + "policies", + ], + &["on_progress", "project_config", "local_config"], + ), + )?; + let project = args.value("project")?; + let adapter = args.value("adapter")?; + let connection = args.value("connection")?; + let runtime = planner_model( + py, + "PlannerRuntime", + &[ + ("project", project.clone()), + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("project_config", args.value("project_config")?), + ("local_config", args.value("local_config")?), + ("on_progress", args.value("on_progress")?), + ], + )?; + let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; + let inspection = scopes.getattr("inspection_scope")?; + let test_planning = sql_test_planning( + (&project, &adapter), + ( + &inspection.getattr("selected_keys")?, + &inspection.getattr("sql_test_selection")?, + ), + args.value("policies")? + .getattr("plan_sql_tests")? + .is_truthy()?, + )?; + let catalog = call_with( + &imported(py, INSPECTION_CATALOG.0, INSPECTION_CATALOG.1)?, + &[("adapter", adapter), ("connection", connection)], + )?; + within(&catalog, |_| { + planned_output(&args, &runtime, (&scopes, &overrides), &test_planning) + }) +} + +/// Start SQL test planning on a daemon thread, or resolve to no tests when disabled. +#[pyfunction] +#[pyo3(signature = (**kwargs))] +fn start_sql_test_planning<'py>( + py: Python<'py>, + kwargs: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "start_sql_test_planning", + kwargs, + ( + &[ + "project", + "adapter", + "selected_keys", + "sql_test_selection", + "enabled", + ], + &[], + ), + )?; + sql_test_planning( + (&args.value("project")?, &args.value("adapter")?), + ( + &args.value("selected_keys")?, + &args.value("sql_test_selection")?, + ), + args.flag("enabled")?, + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_execution_plan, module)?)?; + module.add_function(wrap_pyfunction!(start_sql_test_planning, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 8ff433c750..ccf315c853 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -2,6 +2,7 @@ mod clone_plans; mod command_plans; +mod execution_plan; mod model_entries; mod plan_output; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index f87c46e6a9..b092b93a69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::clone_plans::register(module)?; super::command_plans::register(module)?; + super::execution_plan::register(module)?; super::model_entries::register(module)?; super::plan_output::register(module)?; super::relations::register(module) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d8de92af6b..0ac7ba18e6 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4969,6 +4969,27 @@ def resolve_skipped_view_chain[K]( def sql_test_model_chain_names_by_key( *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] ) -> dict[CompiledObjectKey, tuple[str, ...]]: ... +def build_execution_plan( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + selection: PlannerSelection, + overrides: PlannerOverrides, + deferral: DeferralInputs, + policies: PlannerPolicies, + on_progress: Callable[[str], None] | None = None, + project_config: ProjectConfig | None = None, + local_config: LocalConfig | None = None, +) -> PlanOutput: ... +def start_sql_test_planning( + *, + project: CompiledProjectView, + adapter: BaseAdapter, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection, + enabled: bool, +) -> Future[PlannedSqlTests]: ... def resolve_future_cursor_config( *, model: CompiledModelView, project_config: FutureCursorsConfig | None ) -> FutureCursorsConfig | None: ... diff --git a/src/sqlbuild/compiler/pipeline/main/_plan_result.py b/src/sqlbuild/compiler/pipeline/main/_plan_result.py index 3da6c17930..ad05558578 100644 --- a/src/sqlbuild/compiler/pipeline/main/_plan_result.py +++ b/src/sqlbuild/compiler/pipeline/main/_plan_result.py @@ -6,6 +6,7 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import build_execution_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( @@ -20,7 +21,6 @@ CompilePipelineResult, PythonRunPlanOutputs, ) -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, diff --git a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py b/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py deleted file mode 100644 index a368305335..0000000000 --- a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Plan SQL tests on a worker thread while the planner waits on the warehouse.""" - -from __future__ import annotations - -import threading -from concurrent.futures import Future -from types import TracebackType - -from sqlbuild._native import plan_selected_sql_tests_into -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import PlannedSqlTests, SqlTestSelection - - -class BackgroundSqlTestPlanning: - """Start warehouse-independent SQL test planning early and join it before assembly.""" - - def __init__( - self, - *, - project: CompiledProjectView, - adapter: BaseAdapter, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - enabled: bool, - ) -> None: - self._future: Future[PlannedSqlTests] | None = None - if enabled: - future: Future[PlannedSqlTests] = Future() - self._future = future - threading.Thread( - target=plan_selected_sql_tests_into, - kwargs={ - "future": future, - "project": project, - "adapter": adapter, - "selected_keys": selected_keys, - "sql_test_selection": sql_test_selection, - }, - name="sqlbuild-test-planning", - daemon=True, - ).start() - self._selected_keys: frozenset[CompiledObjectKey] = selected_keys - self._sql_test_selection: SqlTestSelection = sql_test_selection - - def __enter__(self) -> BackgroundSqlTestPlanning: - return self - - def __exit__( - self, - exception_type: type[BaseException] | None, - exception: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exception_type, exception, traceback - - def result(self) -> PlannedSqlTests: - """Return planned tests, raising any planning error at the original assembly point.""" - - if self._future is None: - return PlannedSqlTests( - selected_keys=self._selected_keys, sql_test_selection=self._sql_test_selection - ) - return self._future.result() diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py deleted file mode 100644 index 1df559aa82..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Top-level planner orchestration producing an execution plan.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from dataclasses import replace -from typing import Any - -from sqlbuild._native import ( - assemble_base_plan_output, - build_planner_entry_results, - build_planner_identity_context, - build_planner_source_freshness_result, - check_selected_scope_buildability, - detect_changes, - detect_stale_warning_changes, - gather_planner_warehouse_state, - prune_planner_execution_scope, - reconcile_execution_changes, - resolve_model_actions, - resolve_planner_scopes, - type_cursor_overrides, - with_plan_metadata, - with_plan_warnings, - with_storage_policies, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( - BackgroundSqlTestPlanning, -) -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannedSqlTests, - PlannerChangeReconciliation, - PlannerChangeResults, - PlannerEntryResults, - PlannerIdentityContext, - PlannerOverrides, - PlannerPolicies, - PlannerResolvedActions, - PlannerRuntime, - PlannerScope, - PlannerScopePruningResult, - PlannerScopeResolution, - PlannerSelection, - PlannerWarehouseState, - PlanOutput, - WarehouseSnapshot, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def build_execution_plan( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - connection: Any, - selection: PlannerSelection, - overrides: PlannerOverrides, - deferral: DeferralInputs, - policies: PlannerPolicies, - on_progress: Callable[[str], None] | None = None, - project_config: ProjectConfig | None = None, - local_config: LocalConfig | None = None, -) -> PlanOutput: - runtime: PlannerRuntime = PlannerRuntime( - project=project, - adapter=adapter, - connection=connection, - project_config=project_config, - local_config=local_config, - on_progress=on_progress, - ) - scopes: PlannerScopeResolution - scopes, overrides = _resolve_scopes_and_cursor_overrides( - project=project, - selection=selection, - policies=policies, - overrides=overrides, - ) - with ( - BackgroundSqlTestPlanning( - project=project, - adapter=adapter, - selected_keys=scopes.inspection_scope.selected_keys, - sql_test_selection=scopes.inspection_scope.sql_test_selection, - enabled=policies.plan_sql_tests, - ) as test_planning, - open_inspection_catalog(adapter=adapter, connection=connection), - ): - warehouse: PlannerWarehouseState = gather_planner_warehouse_state( - runtime=runtime, - scopes=scopes, - overrides=overrides, - deferral=deferral, - ) - plan_start: float = time.monotonic() - identities: PlannerIdentityContext = build_planner_identity_context( - project=project, - scopes=scopes, - include_stale_warning_identities=policies.selection_diagnostics, - ) - check_selected_scope_buildability( - runtime=runtime, - scopes=scopes, - warehouse=warehouse, - deferral=deferral, - ) - changes: PlannerChangeResults - stale_warning_changes: PlannerChangeResults - changes, stale_warning_changes = _detect_planner_change_results( - project=project, - scopes=scopes, - snapshot=warehouse.snapshot, - identities=identities, - overrides=overrides, - policies=policies, - ) - resolved_actions: PlannerResolvedActions = resolve_model_actions( - scope=scopes.inspection_scope, - changes=changes, - ) - source_freshness: DirectSourceFreshnessPlanningResult = ( - build_planner_source_freshness_result( - project=project, - adapter=adapter, - connection=connection, - scope=scopes.inspection_scope, - relations=warehouse.inspection_relations, - freshness_state_schemas=warehouse.snapshot.source_freshness_state_schemas, - ) - ) - pruning: PlannerScopePruningResult = prune_planner_execution_scope( - scopes=scopes, - resolved_actions=resolved_actions, - ) - reconciliation: PlannerChangeReconciliation = reconcile_execution_changes( - warehouse=warehouse, - identities=identities, - pruning=pruning, - changes=changes, - ) - entries: PlannerEntryResults = build_planner_entry_results( - runtime=runtime, - warehouse=warehouse, - identities=identities, - overrides=overrides, - policies=policies, - deferral=deferral, - pruning=pruning, - reconciliation=reconciliation, - source_freshness=source_freshness, - ) - planned_sql_tests: PlannedSqlTests = test_planning.result() - plan_output: PlanOutput = assemble_base_plan_output( - runtime=runtime, - warehouse=warehouse, - identities=identities, - overrides=overrides, - pruning=pruning, - reconciliation=reconciliation, - entries=entries, - source_freshness=source_freshness, - planned_sql_tests=planned_sql_tests, - ) - plan_output = with_storage_policies( - plan_output=plan_output, - runtime=runtime, - warehouse=warehouse, - scopes=scopes, - policies=policies, - ) - plan_output = with_plan_warnings( - runtime=runtime, - scopes=scopes, - warehouse=warehouse, - identities=identities, - stale_warning_changes=stale_warning_changes, - pruning=pruning, - source_freshness=source_freshness, - plan_output=plan_output, - policies=policies, - ) - plan_output = with_plan_metadata( - plan_output=plan_output, - pruning=pruning, - source_freshness=source_freshness, - policies=policies, - ) - if on_progress is not None: - on_progress(f"Generated plan. ({time.monotonic() - plan_start:.2f}s)") - return plan_output - - -def _resolve_scopes_and_cursor_overrides( - *, - project: CompiledProjectView, - selection: PlannerSelection, - policies: PlannerPolicies, - overrides: PlannerOverrides, -) -> tuple[PlannerScopeResolution, PlannerOverrides]: - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=project, - selection=selection, - policies=policies, - ) - selected_scope: PlannerScope = scopes.selected_scope - selected_models: tuple[CompiledModelView, ...] = tuple( - selected_scope.models_by_name[key.name] - for key in selected_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - and key.name in selected_scope.models_by_name - ) - return scopes, replace( - overrides, - cursor_overrides=type_cursor_overrides( - cursor_overrides=overrides.cursor_overrides, selected_models=selected_models - ), - ) - - -def _detect_planner_change_results( - *, - project: CompiledProjectView, - scopes: PlannerScopeResolution, - snapshot: WarehouseSnapshot, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - policies: PlannerPolicies, -) -> tuple[PlannerChangeResults, PlannerChangeResults]: - changes: PlannerChangeResults = detect_changes( - project=project, - scope=scopes.inspection_scope, - snapshot=snapshot, - full_refresh=overrides.full_refresh, - expected_version_hashes=identities.version_identities.model_version_hashes, - expected_metadata_jsons=identities.version_identities.model_metadata_jsons, - ) - if not policies.selection_diagnostics: - return changes, PlannerChangeResults(models={}, functions={}) - return changes, detect_stale_warning_changes( - project=project, - scopes=scopes, - snapshot=snapshot, - identities=identities, - execution_changes=changes, - ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index bc10eaabad..6df338e9b3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -14,6 +14,7 @@ from sqlbuild._native import ( build_compiled_function_fingerprint_sql, + build_execution_plan, compiled_model_record, compute_query_hash, discover_project_inputs, @@ -52,7 +53,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py similarity index 88% rename from tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py rename to tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 02fa4d0a77..0b8f24f86e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_background_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -1,15 +1,14 @@ from __future__ import annotations import threading +from concurrent.futures import Future import pytest import sqlbuild._native as native_module +from sqlbuild._native import start_sql_test_planning from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.classes.background_sql_test_planning import ( - BackgroundSqlTestPlanning, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import PlannedSqlTests, PlanWarning, SqlTestSelection from sqlbuild.compiler.planner.types import WarningSeverity @@ -57,14 +56,14 @@ def plan( monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ) as planning: - result: PlannedSqlTests = planning.result() + ) + result: PlannedSqlTests = planning.result() assert result == _PLANNED assert calls == [_KEYS] * test_case.expected_planner_calls @@ -99,14 +98,14 @@ def plan( monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ) as planning: - result: PlannedSqlTests = planning.result() + ) + result: PlannedSqlTests = planning.result() assert result == PlannedSqlTests(selected_keys=_KEYS) assert len(calls) == test_case.expected_planner_calls @@ -139,15 +138,15 @@ def plan( monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ) as planning: - with pytest.raises(PlannerInputError, match=str(test_case.expected_error)): - planning.result() + ) + with pytest.raises(PlannerInputError, match=str(test_case.expected_error)): + planning.result() @pytest.mark.parametrize( @@ -181,18 +180,19 @@ def plan( monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - with BackgroundSqlTestPlanning( + planning: Future[PlannedSqlTests] = start_sql_test_planning( sql_test_selection=SqlTestSelection(), project=object(), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, - ): - assert started.wait(timeout=5) - daemon_by_name: dict[str, bool] = { - thread.name: thread.daemon for thread in threading.enumerate() - } + ) + assert started.wait(timeout=5) + daemon_by_name: dict[str, bool] = { + thread.name: thread.daemon for thread in threading.enumerate() + } release.set() + planning.result() assert daemon_by_name["sqlbuild-test-planning"] is True assert test_case.expected_planner_calls == 1 diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 6fd2cae9c2..b4dd031f7c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -6,6 +6,7 @@ from typing import Any from sqlbuild._native import ( + build_execution_plan, build_version_identity_metadata_json, compute_query_hash, write_fingerprint, @@ -24,7 +25,6 @@ CompiledResourceType, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, From a328b332cab70e95b92075bf7114c5bb08482dfd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:39:28 +0100 Subject: [PATCH 06/33] refactor: host plan display helpers and planner classes natively Port the display-only plan, pre-build work text, query diff colouring, KnownInputColumns and the fixture-null autofix entry point to native code, removing the Python planner main package. --- crates/sqlbuild-core/src/cli_text/models.rs | 31 ++++ .../src/plan_display/constants.rs | 2 + .../main/format_pre_build_work.rs | 57 ++++++ .../plan_display/main/format_query_diff.rs | 26 +++ .../src/plan_display/main/mod.rs | 2 + .../plan_display/tests/test_plan_display.rs | 121 ++++++++++++- .../src/plan_display/tests/test_types.rs | 13 ++ .../_helpers/dbt_interop/plan_output.rs | 6 +- .../bindings/_helpers/format_command/run.rs | 4 +- .../_helpers/plan_output/migrations.rs | 84 ++++++++- .../_helpers/plan_output/plan_json.rs | 6 +- .../_helpers/plan_output/plan_text.rs | 39 ++-- .../_helpers/planner_entries/display_plan.rs | 169 ++++++++++++++++++ .../bindings/_helpers/planner_entries/mod.rs | 1 + .../_helpers/planner_entries/registration.rs | 1 + .../planner_migrations/column_planning.rs | 20 +-- .../planner_migrations/known_input_columns.rs | 50 ++++++ .../_helpers/planner_migrations/mod.rs | 1 + fensu.toml | 5 + src/sqlbuild/_native.pyi | 6 + .../planner/classes/fixture_null_autofix.py | 30 ---- .../planner/classes/known_input_columns.py | 31 ---- .../compiler/planner/main/__init__.py | 1 - .../compiler/planner/main/changes/__init__.py | 0 .../planner/main/changes/_query_diff.py | 44 ----- .../planner/main/execution/__init__.py | 0 .../planner/main/execution/display_plan.py | 98 ---------- .../planner/main/pre_build/__init__.py | 0 .../main/pre_build/pre_build_work_display.py | 53 ------ .../main/execution/test_display_plan.py | 5 +- .../planner/main/pre_build/helpers.py | 2 +- .../pre_build/test_pre_build_work_display.py | 3 +- 32 files changed, 600 insertions(+), 311 deletions(-) create mode 100644 crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs create mode 100644 crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs delete mode 100644 src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py delete mode 100644 src/sqlbuild/compiler/planner/classes/known_input_columns.py delete mode 100644 src/sqlbuild/compiler/planner/main/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/changes/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/changes/_query_diff.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/execution/display_plan.py delete mode 100644 src/sqlbuild/compiler/planner/main/pre_build/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index bb689cf0d2..249fcb0745 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -616,3 +616,34 @@ pub struct MigrationTextReport { pub display: DisplayLimit, pub manual_rename_hint: String, } + +/// One planned table-type conversion line. +#[derive(Clone, Debug)] +pub struct TableTypeText { + pub model_name: String, + pub desired_type: String, + pub actual_type: Option, + pub source: String, + pub irreversible_warning: Option, +} + +/// One planned retention change line. +#[derive(Clone, Debug)] +pub struct RetentionText { + pub scope: String, + pub desired_days: i64, + pub actual_days: Option, + pub effective_days: Option, + pub source: String, + pub direction: String, + pub phase: String, + pub irreversible_warning: Option, +} + +/// Storage work planned before model builds, ahead of the migration sections. +#[derive(Clone, Debug)] +pub struct PreBuildWorkText { + pub table_types: Vec, + pub retention: Vec, + pub display: DisplayLimit, +} diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index 0445540d36..aa6211db6f 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -33,3 +33,5 @@ pub(crate) const MIGRATION_MANUAL: &str = "manual"; pub(crate) const MIGRATION_AUTOMATIC: &str = "automatic"; pub(crate) const MIGRATION_INCOMPATIBLE: &str = "incompatible"; pub(crate) const WATERMARK_STRATEGY: &str = "watermark"; +pub(crate) const QUERY_DIFF_INDENT: &str = " "; +pub(crate) const UNIFIED_DIFF_HEADER_PREFIXES: [&str; 3] = ["---", "+++", "@@"]; diff --git a/crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs b/crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs new file mode 100644 index 0000000000..05f76d251a --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_pre_build_work.rs @@ -0,0 +1,57 @@ +//! Table-type and retention work that runs before model builds. + +use crate::cli_text::main::format_overflow_line::format_overflow_line; +use crate::cli_text::models::PreBuildWorkText; + +fn days(value: Option) -> String { + value.map_or_else(|| "missing".to_owned(), |days| format!("{days}d")) +} + +/// Append table-type conversions, with an overflow hint, and every retention change. +#[must_use] +pub fn format_pre_build_work(mut lines: Vec, work: &PreBuildWorkText) -> Vec { + if !work.table_types.is_empty() { + lines.push("Table type conversions".to_owned()); + let visible = work + .display + .max_entries + .map_or(work.table_types.len(), |limit| { + limit.min(work.table_types.len()) + }); + for entry in &work.table_types[..visible] { + lines.push(format!( + " {} desired={} actual={} source={}", + entry.model_name, + entry.desired_type, + entry.actual_type.as_deref().unwrap_or("unknown"), + entry.source + )); + if let Some(warning) = &entry.irreversible_warning { + lines.push(format!(" WARNING: {warning}")); + } + } + let hidden = i64::try_from(work.table_types.len() - visible).unwrap_or(i64::MAX); + if let Some(line) = format_overflow_line(hidden, " ", &work.display.overflow_flag) { + lines.push(line); + } + } + if !work.retention.is_empty() { + lines.push("Retention".to_owned()); + } + for entry in &work.retention { + lines.push(format!( + " {} desired={}d actual={} effective={} source={} direction={} phase={}", + entry.scope, + entry.desired_days, + days(entry.actual_days), + days(entry.effective_days), + entry.source, + entry.direction, + entry.phase + )); + if let Some(warning) = &entry.irreversible_warning { + lines.push(format!(" WARNING: {warning}")); + } + } + lines +} diff --git a/crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs b/crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs new file mode 100644 index 0000000000..321c688b44 --- /dev/null +++ b/crates/sqlbuild-core/src/plan_display/main/format_query_diff.rs @@ -0,0 +1,26 @@ +//! Coloured, indented unified-diff lines for a changed model query. + +use crate::cli_text::models::CliStyle; +use crate::plan_display::constants::{QUERY_DIFF_INDENT, UNIFIED_DIFF_HEADER_PREFIXES}; + +/// Indent each unified-diff line and colour additions, removals and headers. +#[must_use] +pub fn format_query_diff(diff_lines: &[String], style: CliStyle) -> Vec { + let mut formatted_lines: Vec = Vec::with_capacity(diff_lines.len()); + for line in diff_lines { + let formatted = format!("{QUERY_DIFF_INDENT}{line}"); + let header = UNIFIED_DIFF_HEADER_PREFIXES + .iter() + .any(|prefix| line.starts_with(prefix)); + formatted_lines.push(if line.starts_with('+') && !line.starts_with("+++") { + style.success(&formatted) + } else if line.starts_with('-') && !line.starts_with("---") { + style.error(&formatted) + } else if header { + style.muted(&formatted) + } else { + formatted + }); + } + formatted_lines +} diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index c9d5735ad4..ab3ac3c3a5 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -3,6 +3,8 @@ pub mod effective_microbatch_batch_size; pub mod effective_timestamp_grain; pub mod format_model_migrations; pub mod format_no_input_rows; +pub mod format_pre_build_work; +pub mod format_query_diff; pub mod format_waiting_on_empty_inputs; pub mod model_execution_annotation; pub mod model_materialization_label; diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs index 5e41e9cb63..012be8076b 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_display.rs @@ -1,7 +1,12 @@ +use crate::cli_text::models::{ + CliStyle, DisplayLimit, PreBuildWorkText, RetentionText, TableTypeText, +}; use crate::plan_display::main::cursor_bound_display::cursor_bound_display; use crate::plan_display::main::effective_microbatch_batch_size::effective_microbatch_batch_size; use crate::plan_display::main::effective_timestamp_grain::effective_timestamp_grain; use crate::plan_display::main::format_no_input_rows::format_no_input_rows; +use crate::plan_display::main::format_pre_build_work::format_pre_build_work; +use crate::plan_display::main::format_query_diff::format_query_diff; use crate::plan_display::main::format_waiting_on_empty_inputs::format_waiting_on_empty_inputs; use crate::plan_display::main::model_execution_annotation::model_execution_annotation; use crate::plan_display::main::model_materialization_label::model_materialization_label; @@ -9,7 +14,8 @@ use crate::plan_display::main::model_resource_type::model_resource_type; use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; use crate::plan_display::tests::helpers::{facts, timestamp_grains}; use crate::plan_display::tests::test_types::{ - BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, TimestampGrainTestCase, + BatchSizeTestCase, BoundTestCase, LabelTestCase, NamesTestCase, PreBuildWorkTestCase, + QueryDiffTestCase, TimestampGrainTestCase, }; #[test] @@ -260,3 +266,116 @@ fn given_cursor_grains_when_resolving_replay_grain_then_coarsest_timestamp_grain ); } } + +#[test] +fn given_storage_work_when_formatting_pre_build_work_then_matches_python_text() { + let table_type = TableTypeText { + model_name: "orders".into(), + desired_type: "transient".into(), + actual_type: None, + source: "model".into(), + irreversible_warning: Some("drops history".into()), + }; + let retention = RetentionText { + scope: "analytics.orders".into(), + desired_days: 7, + actual_days: Some(1), + effective_days: None, + source: "project".into(), + direction: "increase".into(), + phase: "pre_build".into(), + irreversible_warning: None, + }; + let test_cases = [ + PreBuildWorkTestCase { + description: "lists every entry without a limit", + work: PreBuildWorkText { + table_types: vec![table_type.clone()], + retention: vec![retention], + display: DisplayLimit { + max_entries: None, + overflow_flag: "--verbose".into(), + }, + }, + expected_lines: vec![ + "Table type conversions", + " orders desired=transient actual=unknown source=model", + " WARNING: drops history", + "Retention", + " analytics.orders desired=7d actual=1d effective=missing source=project direction=increase phase=pre_build", + ], + }, + PreBuildWorkTestCase { + description: "limits table types with an overflow hint", + work: PreBuildWorkText { + table_types: vec![table_type.clone(), table_type], + retention: vec![], + display: DisplayLimit { + max_entries: Some(0), + overflow_flag: "--verbose".into(), + }, + }, + expected_lines: vec![ + "Table type conversions", + "\u{1b}[2m ... and 2 more (use --verbose to show all)\u{1b}[0m", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + format_pre_build_work(Vec::new(), &test_case.work), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unified_diff_when_formatting_query_diff_then_lines_are_indented_and_coloured() { + let diff: Vec = [ + "--- previous", + "+++ current", + "@@ -1 +1 @@", + "-a", + "+b", + " c", + ] + .into_iter() + .map(String::from) + .collect(); + let test_cases = [ + QueryDiffTestCase { + description: "plain diff is only indented", + style: CliStyle { use_color: false }, + expected_lines: vec![ + " --- previous", + " +++ current", + " @@ -1 +1 @@", + " -a", + " +b", + " c", + ], + }, + QueryDiffTestCase { + description: "coloured diff marks additions, removals and headers", + style: CliStyle { use_color: true }, + expected_lines: vec![ + "\x1b[2m --- previous\x1b[0m", + "\x1b[2m +++ current\x1b[0m", + "\x1b[2m @@ -1 +1 @@\x1b[0m", + "\x1b[38;5;167m -a\x1b[0m", + "\x1b[32m +b\x1b[0m", + " c", + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + format_query_diff(&diff, test_case.style), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs index d924f3052d..30466be5cf 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_types.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::cli_text::models::{CliStyle, PreBuildWorkText}; use crate::plan_display::models::{ModelLabelFacts, ParsedBound, TimestampGrainFacts}; pub(super) struct LabelTestCase { @@ -35,3 +36,15 @@ pub(super) struct TimestampGrainTestCase { pub(super) facts: TimestampGrainFacts, pub(super) expected_grain: Result, &'static str>, } + +pub(super) struct PreBuildWorkTestCase { + pub(super) description: &'static str, + pub(super) work: PreBuildWorkText, + pub(super) expected_lines: Vec<&'static str>, +} + +pub(super) struct QueryDiffTestCase { + pub(super) description: &'static str, + pub(super) style: CliStyle, + pub(super) expected_lines: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 0973c7b46d..a85e301568 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -222,11 +222,7 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( ) => { call_with( - &imported( - py, - "sqlbuild.compiler.planner.main.execution.display_plan", - "build_display_only_sqlbuild_plan", - )?, + &imported(py, "sqlbuild._native", "build_display_only_sqlbuild_plan")?, &[ ("project", environment.getattr("project")?), ("selected_model_names", selected), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index d4356bf2a5..3d07d2ab76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::format_command::files::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const FIXES_MODULE: &str = "sqlbuild._native"; -const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild.compiler.planner.classes.fixture_null_autofix"; +const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild._native"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; const DESCRIPTION_PRESENT: &str = "description-present"; @@ -57,7 +57,7 @@ fn fixture_autofix<'py>( return Ok(PyDict::new(py)); }; Ok(call_with( - &imported(py, FIXTURE_AUTOFIX_MODULE, "FixtureNullAutofix")?.getattr("apply")?, + &imported(py, FIXTURE_AUTOFIX_MODULE, "format_redundant_fixture_nulls")?, &[ ("files", files.clone().into_any()), ("project_dir", request.project_dir.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs index 4e2f85dae8..f835c7cf12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs @@ -8,9 +8,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::{ CliStyle, ColumnMigrationText, MigrationDecisionText, MigrationText, MigrationTextReport, - OldNameText, + OldNameText, PreBuildWorkText, RetentionText, TableTypeText, +}; +use sqlbuild_core::plan_display::main::{ + format_model_migrations, format_pre_build_work, old_name_rows, }; -use sqlbuild_core::plan_display::main::{format_model_migrations, old_name_rows}; use std::collections::BTreeMap; fn label(value: &Bound<'_, PyAny>) -> PyResult> { @@ -85,8 +87,8 @@ fn column(entry: &Bound<'_, PyAny>) -> PyResult { }) } -#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] -fn format_migrations( +/// Migration sections appended to `lines`, in planner order. +pub(crate) fn migration_lines( lines: Vec, plan: &Bound<'_, PyAny>, display_options: &Bound<'_, PyAny>, @@ -125,6 +127,79 @@ fn format_migrations( )) } +#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] +fn format_migrations( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + migration_lines(lines, plan, display_options) +} + +fn days(value: &Bound<'_, PyAny>) -> PyResult> { + value.extract() +} + +/// Table-type, retention and migration work that runs before model builds. +pub(crate) fn pre_build_lines( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + let mut table_types: Vec = Vec::new(); + for entry in items(&plan.getattr("table_type_entries")?)? { + table_types.push(TableTypeText { + model_name: text(&entry.getattr("model_name")?)?, + desired_type: text(&entry.getattr("desired_type")?)?, + actual_type: optional_text(&entry.getattr("actual_type")?)? + .filter(|actual| !actual.is_empty()), + source: text(&entry.getattr("source")?)?, + irreversible_warning: optional_text(&entry.getattr("irreversible_warning")?)?, + }); + } + let mut retention: Vec = Vec::new(); + for entry in items(&plan.getattr("retention_entries")?)? { + let request = entry.getattr("request")?; + let mut parts: Vec = Vec::new(); + for field in ["database", "schema", "name"] { + let part = request.getattr(field)?; + if part.is_truthy()? { + parts.push(text(&part)?); + } + } + retention.push(RetentionText { + scope: parts.join("."), + desired_days: request.getattr("desired_days")?.extract()?, + actual_days: days(&entry.getattr("actual_days")?)?, + effective_days: days(&entry.getattr("effective_days")?)?, + source: text(&entry.getattr("source")?)?, + direction: text(&entry.getattr("direction")?.getattr("value")?)?, + phase: text(&entry.getattr("phase")?.getattr("value")?)?, + irreversible_warning: optional_text(&entry.getattr("irreversible_warning")?)?, + }); + } + let work = PreBuildWorkText { + table_types, + retention, + display: display_limit(display_options)?, + }; + migration_lines( + format_pre_build_work::format_pre_build_work(lines, &work), + plan, + display_options, + ) +} + +/// Format retention, table-type, and migration work that runs before model builds. +#[pyfunction(name = "format_pre_build_work", signature = (*, lines, plan, display_options))] +fn format_pre_build( + lines: Vec, + plan: &Bound<'_, PyAny>, + display_options: &Bound<'_, PyAny>, +) -> PyResult> { + pre_build_lines(lines, plan, display_options) +} + #[pyfunction(name = "old_name_rows", signature = (*, entry, style))] fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResult> { Ok(old_name_rows::old_name_rows( @@ -137,5 +212,6 @@ fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResu pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(format_migrations, module)?)?; + module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; module.add_function(wrap_pyfunction!(render_old_name, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 51239c58ac..123d07fb5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -362,7 +362,11 @@ fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> } /// `difflib.unified_diff` between two texts, each line without its newline. -fn unified_diff<'py>(py: Python<'py>, previous: &str, current: &str) -> PyResult> { +pub(crate) fn unified_diff<'py>( + py: Python<'py>, + previous: &str, + current: &str, +) -> PyResult> { let lines = |text: &str| -> PyResult> { PyString::new(py, text).call_method1("splitlines", (true,)) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs index e4377d901e..d2ebdc8e5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -6,14 +6,13 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; use sqlbuild_core::cli_text::models::{ - CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, - ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, SeedPlanText, - SourceFreshnessText, SourceLoadPlanText, + CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, + PlanWarningText, ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, + SeedPlanText, SourceFreshnessText, SourceLoadPlanText, }; +use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::plan_output::cursor_details::{ CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, @@ -22,13 +21,13 @@ use crate::bindings::_helpers::plan_output::display::{ cursor_bound_display, format_no_input_rows, format_waiting_on_empty_inputs, inclusive_cursor_end, model_materialization_label, }; +use crate::bindings::_helpers::plan_output::migrations::pre_build_lines; +use crate::bindings::_helpers::plan_output::plan_json::unified_diff; use crate::bindings::_helpers::plan_output::python_identity::{ definition_source_text, dependency_source_text, }; use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; -const QUERY_DIFF_MODULE: &str = "sqlbuild.compiler.planner.main.changes._query_diff"; -const PRE_BUILD_MODULE: &str = "sqlbuild.compiler.planner.main.pre_build.pre_build_work_display"; const DISPLAY_MODELS_MODULE: &str = "sqlbuild.presentation.models"; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; @@ -48,16 +47,17 @@ const PLAN_OPTIONS: (&[&str], &[&str]) = ( ], ); -/// `format_query_diff(previous=..., current=...)`. +/// The coloured unified diff between a previous and a current query. pub(crate) fn query_diff( previous: &Bound<'_, PyAny>, current: &Bound<'_, PyAny>, ) -> PyResult> { - call_with( - &imported(previous.py(), QUERY_DIFF_MODULE, "format_query_diff")?, - &[("previous", previous.clone()), ("current", current.clone())], - )? - .extract() + let lines = unified_diff( + previous.py(), + &previous.extract::()?, + ¤t.extract::()?, + )?; + Ok(format_query_diff(&lines, CliStyle { use_color: true })) } fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { @@ -437,14 +437,7 @@ fn report(arguments: &KeywordArguments<'_>) -> PyResult { reason: text(&seed.getattr("reason")?)?, }); } - let pre_build: Bound<'_, PyAny> = call_with( - &imported(py, PRE_BUILD_MODULE, "format_pre_build_work")?, - &[ - ("lines", PyList::empty(py).into_any()), - ("plan", plan.clone()), - ("display_options", display_options.clone()), - ], - )?; + let pre_build: Vec = pre_build_lines(Vec::new(), &plan, &display_options)?; Ok(PlanTextReport { use_color: flag_or("use_color", true)?, include_header: flag_or("include_header", true)?, @@ -476,7 +469,7 @@ fn report(arguments: &KeywordArguments<'_>) -> PyResult { metadata.call_method1("get", ("direct_remaining_stale_model_names",))?, ))?, pruned_models: pruned_models(&metadata)?, - pre_build_lines: pre_build.extract()?, + pre_build_lines: pre_build, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs new file mode 100644 index 0000000000..ff15696e1c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -0,0 +1,169 @@ +//! Display-only plans for external plan surfaces when warehouse planning cannot resolve. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_entries::values::{ + NATIVE_MODULE, boolean, native, none, planner_model, planner_type, string, +}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const VIEW: &str = "view"; +const INCREMENTAL: &str = "incremental"; +const CUSTOM: &str = "custom"; + +fn display_action( + materialization_type: &Bound<'_, PyAny>, + full_refresh: bool, +) -> PyResult<&'static str> { + Ok(if materialization_type.eq(VIEW)? { + "CREATE_VIEW" + } else if materialization_type.eq(INCREMENTAL)? { + if full_refresh { + "CREATE_TABLE" + } else { + "INCREMENTAL_APPEND" + } + } else if materialization_type.eq(CUSTOM)? { + "CUSTOM" + } else { + "CREATE_TABLE" + }) +} + +fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { + native( + model.py(), + "get_config_str", + &[ + ("values", model.getattr("config")?.getattr("values")?), + ("key", string(model.py(), key)), + ], + ) +} + +fn display_entry<'py>( + model: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py = model.py(); + let materialization_type = native(py, "get_materialization_type", &[("model", model.clone())])?; + let effective = native( + py, + "resolve_model_full_refresh", + &[ + ("model", model.clone()), + ("cli_full_refresh", boolean(py, full_refresh)?), + ], + )? + .is_truthy()?; + let query_sql = model.getattr("query_sql")?; + let custom_name = if materialization_type.eq(CUSTOM)? { + config(model, "materialized")? + } else { + none(py) + }; + planner_model( + py, + "ModelPlanEntry", + &[ + ("key", model.getattr("key")?), + ("name", model.getattr("name")?), + ("relative_path", model.getattr("relative_path")?), + ( + "action", + planner_type( + py, + "PlanAction", + display_action(&materialization_type, effective)?, + )?, + ), + ("materialization_type", materialization_type), + ( + "reason", + planner_type( + py, + "PlanReason", + if effective { + "FULL_REFRESH" + } else { + "NO_CHANGE" + }, + )?, + ), + ("destination", model.getattr("destination")?), + ("fingerprint_query_sql", query_sql.clone()), + ("resolved_sql", query_sql), + ("logical_ddl", string(py, "")), + ( + "incremental_strategy", + config(model, "incremental_strategy")?, + ), + ("incremental_mode", config(model, "incremental_mode")?), + ("cursor_column", config(model, "cursor_column")?), + ("cursor_type", config(model, "cursor_type")?), + ( + "backfill", + planner_model( + py, + "BackfillResult", + &[( + "action", + planner_type( + py, + "BackfillAction", + if effective { "FULL" } else { "FORWARD_ONLY" }, + )?, + )], + )?, + ), + ("custom_materialization_name", custom_name), + ], + ) +} + +/// Build a display-only plan when warehouse planning cannot fully resolve. +#[pyfunction] +#[pyo3(signature = (*, project, selected_model_names, full_refresh))] +fn build_display_only_sqlbuild_plan<'py>( + project: &Bound<'py, PyAny>, + selected_model_names: &Bound<'py, PyAny>, + full_refresh: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let selected = PyFrozenSet::new( + py, + selected_model_names + .try_iter()? + .collect::>>()?, + )?; + let mut entries: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + if selected.contains(model.getattr("name")?)? { + entries.push(display_entry(&model, full_refresh)?); + } + } + let keys: Vec> = entries + .iter() + .map(|entry| entry.getattr("key")) + .collect::>()?; + let upstream = + imported(py, NATIVE_MODULE, "build_execution_upstream_deps")?.call1((project,))?; + let downstream = imported(py, NATIVE_MODULE, "build_downstream_deps")?.call1((&upstream,))?; + planner_model( + py, + "PlanOutput", + &[ + ("execution_order", PyTuple::new(py, &keys)?.into_any()), + ("model_entries", PyTuple::new(py, entries)?.into_any()), + ("selected_keys", PyFrozenSet::new(py, keys)?.into_any()), + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ], + ) +} + +pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_display_only_sqlbuild_plan, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index ccf315c853..452f3c67a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -2,6 +2,7 @@ mod clone_plans; mod command_plans; +mod display_plan; mod execution_plan; mod model_entries; mod plan_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs index b092b93a69..32e77e42bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/registration.rs @@ -5,6 +5,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::clone_plans::register(module)?; super::command_plans::register(module)?; + super::display_plan::register(module)?; super::execution_plan::register(module)?; super::model_entries::register(module)?; super::plan_output::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 200d78f652..761bc780a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_migrations::column_objects::{model, replace}; +use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -66,17 +67,14 @@ fn plan_column_migrations<'py>( &state.getattr("inspect_column_events")?, &[("schemas", schemas.into_any())], )?; - let inputs = call_with( - &imported( - py, - "sqlbuild.compiler.planner.classes.known_input_columns", - "KnownInputColumns", - )?, - &[ - ("model_columns", snapshot.getattr("existing_columns")?), - ("source_columns", args.value("source_columns")?), - ], - )?; + let inputs = Bound::new( + py, + KnownInputColumns::new( + &snapshot.getattr("existing_columns")?, + &args.value("source_columns")?, + ), + )? + .into_any(); let mut entries: Vec> = Vec::new(); let mut warnings: Vec> = Vec::new(); let mut hints: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs new file mode 100644 index 0000000000..358b2d003c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/known_input_columns.rs @@ -0,0 +1,50 @@ +//! Physical columns of the relations a model reads, as inspected by the planner. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyFrozenSet; +use pyo3::{pyclass, pymethods}; + +const SOURCE_INPUT_FUNCTION: &str = "__source"; +const REF_INPUT_FUNCTION: &str = "__ref"; + +/// Look up the lower-cased physical columns a query binds against, or None when unknown. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct KnownInputColumns { + model_columns: Py, + source_columns: Py, +} + +impl KnownInputColumns { + /// Inspected model and source columns by relation name. + pub(crate) fn new(model_columns: &Bound<'_, PyAny>, source_columns: &Bound<'_, PyAny>) -> Self { + Self { + model_columns: model_columns.clone().unbind(), + source_columns: source_columns.clone().unbind(), + } + } +} + +#[pymethods] +impl KnownInputColumns { + fn __call__<'py>( + &self, + py: Python<'py>, + kind: &str, + name: &str, + ) -> PyResult>> { + let inspected = match kind { + REF_INPUT_FUNCTION => self.model_columns.bind(py), + SOURCE_INPUT_FUNCTION => self.source_columns.bind(py), + _ => return Ok(None), + }; + let columns = inspected.call_method1("get", (name,))?; + if columns.is_none() || !columns.is_truthy()? { + return Ok(None); + } + let mut names: Vec = Vec::new(); + for column in columns.try_iter()? { + names.push(column?.getattr("name")?.extract::()?.to_lowercase()); + } + PyFrozenSet::new(py, names).map(Some) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 1d488c57e2..a6a080315f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -13,6 +13,7 @@ mod discovery_models; mod discovery_requests; mod errors; mod fingerprint; +mod known_input_columns; mod matching; mod model_entries; mod model_handover; diff --git a/fensu.toml b/fensu.toml index c3c999eea3..4e1d503db5 100644 --- a/fensu.toml +++ b/fensu.toml @@ -58,6 +58,11 @@ rule = "FFR309" path = "src/sqlbuild/spec/contracts/__init__.py" reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/planner/__init__.py" +reason = "Native bindings own planning, plan entries, plan output and command plans; this package only declares the plan models, types, constants, errors and the fixture inference cache the native planner reads." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0ac7ba18e6..1bcae18060 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4990,6 +4990,12 @@ def start_sql_test_planning( sql_test_selection: SqlTestSelection, enabled: bool, ) -> Future[PlannedSqlTests]: ... +def build_display_only_sqlbuild_plan( + *, project: CompiledProjectView, selected_model_names: tuple[str, ...], full_refresh: bool +) -> PlanOutput: ... +def format_pre_build_work( + *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions +) -> list[str]: ... def resolve_future_cursor_config( *, model: CompiledModelView, project_config: FutureCursorsConfig | None ) -> FutureCursorsConfig | None: ... diff --git a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py deleted file mode 100644 index ec4451aec7..0000000000 --- a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Compiler-backed safe fixture-null source rewrites.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import format_redundant_fixture_nulls -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -class FixtureNullAutofix: - """Apply fixture-null rewrites proven safe by planner completion.""" - - @staticmethod - def apply( - *, - files: dict[Path, str], - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - sql_lexical_syntax: SqlLexicalSyntax, - ) -> dict[Path, str]: - """Return contract-safe fixture source rewrites.""" - - return format_redundant_fixture_nulls( - files=files, - project_dir=project_dir, - discovered_inputs=discovered_inputs, - sql_lexical_syntax=sql_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/planner/classes/known_input_columns.py b/src/sqlbuild/compiler/planner/classes/known_input_columns.py deleted file mode 100644 index e8d30db508..0000000000 --- a/src/sqlbuild/compiler/planner/classes/known_input_columns.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Physical columns of the relations a model reads, as inspected by the planner.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.constants import REF_INPUT_FUNCTION, SOURCE_INPUT_FUNCTION - - -class KnownInputColumns: - """Look up the lower-cased physical columns a query binds against, or None when unknown.""" - - def __init__( - self, - *, - model_columns: Mapping[str, tuple[ColumnInfo, ...]], - source_columns: Mapping[str, tuple[ColumnInfo, ...]], - ) -> None: - self._inspected: dict[str, Mapping[str, tuple[ColumnInfo, ...]]] = { - REF_INPUT_FUNCTION: model_columns, - SOURCE_INPUT_FUNCTION: source_columns, - } - - def __call__(self, kind: str, name: str) -> frozenset[str] | None: - """Return the inspected columns of one __source or __ref relation, or None.""" - - columns: tuple[ColumnInfo, ...] = self._inspected.get(kind, {}).get(name, ()) - if not columns: - return None - return frozenset(column.name.lower() for column in columns) diff --git a/src/sqlbuild/compiler/planner/main/__init__.py b/src/sqlbuild/compiler/planner/main/__init__.py deleted file mode 100644 index 1f8ed9be06..0000000000 --- a/src/sqlbuild/compiler/planner/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Planner main package.""" diff --git a/src/sqlbuild/compiler/planner/main/changes/__init__.py b/src/sqlbuild/compiler/planner/main/changes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/changes/_query_diff.py b/src/sqlbuild/compiler/planner/main/changes/_query_diff.py deleted file mode 100644 index 2bc4a905ce..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/_query_diff.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Public SQL/text diff formatting entrypoint.""" - -from __future__ import annotations - -import difflib - -from sqlbuild.compiler.planner.constants import ( - UNIFIED_DIFF_ADDITION_HEADER_PREFIX, - UNIFIED_DIFF_ADDITION_PREFIX, - UNIFIED_DIFF_REMOVAL_HEADER_PREFIX, - UNIFIED_DIFF_REMOVAL_PREFIX, -) -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_query_diff(*, previous: str, current: str) -> list[str]: - """Format a unified diff between previous and current SQL/text.""" - - style: CliStyle = CliStyle(use_color=True) - previous_lines: list[str] = previous.splitlines(keepends=True) - current_lines: list[str] = current.splitlines(keepends=True) - diff_lines: list[str] = list( - difflib.unified_diff(previous_lines, current_lines, fromfile="previous", tofile="current") - ) - result: list[str] = [] - line: str - for line in diff_lines: - stripped: str = line.rstrip("\n") - formatted: str = f" {stripped}" - if ( - stripped[:1] == UNIFIED_DIFF_ADDITION_PREFIX - and stripped[:3] != UNIFIED_DIFF_ADDITION_HEADER_PREFIX - ): - result.append(style.success(formatted)) - elif ( - stripped[:1] == UNIFIED_DIFF_REMOVAL_PREFIX - and stripped[:3] != UNIFIED_DIFF_REMOVAL_HEADER_PREFIX - ): - result.append(style.error(formatted)) - elif stripped.startswith(("---", "+++", "@@")): - result.append(style.muted(formatted)) - else: - result.append(formatted) - return result diff --git a/src/sqlbuild/compiler/planner/main/execution/__init__.py b/src/sqlbuild/compiler/planner/main/execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py deleted file mode 100644 index 8dab9c5583..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Display-only planner helpers for external plan surfaces.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_downstream_deps, - build_execution_upstream_deps, - get_config_str, - get_materialization_type, - resolve_model_full_refresh, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) - - -def build_display_only_sqlbuild_plan( - *, project: CompiledProjectView, selected_model_names: tuple[str, ...], full_refresh: bool -) -> PlanOutput: - """Build a display-only plan when warehouse planning cannot fully resolve.""" - - selected_names: frozenset[str] = frozenset(selected_model_names) - model_entries: list[ModelPlanEntry] = [] - for model in project.models: - if model.name not in selected_names: - continue - materialization_type: MaterializationType = get_materialization_type(model) - effective_full_refresh: bool = resolve_model_full_refresh( - model=model, - cli_full_refresh=full_refresh, - ) - model_entries.append( - ModelPlanEntry( - key=model.key, - name=model.name, - relative_path=model.relative_path, - materialization_type=materialization_type, - action=_display_action( - materialization_type=materialization_type, - full_refresh=effective_full_refresh, - ), - reason=( - PlanReason.FULL_REFRESH if effective_full_refresh else PlanReason.NO_CHANGE - ), - destination=model.destination, - fingerprint_query_sql=model.query_sql, - resolved_sql=model.query_sql, - logical_ddl="", - incremental_strategy=get_config_str( - values=model.config.values, key="incremental_strategy" - ), - incremental_mode=get_config_str(values=model.config.values, key="incremental_mode"), - cursor_column=get_config_str(values=model.config.values, key="cursor_column"), - cursor_type=get_config_str(values=model.config.values, key="cursor_type"), - backfill=BackfillResult( - action=( - BackfillAction.FULL - if effective_full_refresh - else BackfillAction.FORWARD_ONLY - ) - ), - custom_materialization_name=( - get_config_str(values=model.config.values, key="materialized") - if materialization_type == MaterializationType.CUSTOM - else None - ), - ) - ) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - return PlanOutput( - execution_order=tuple(entry.key for entry in model_entries), - model_entries=tuple(model_entries), - selected_keys=frozenset(entry.key for entry in model_entries), - upstream_deps=upstream_deps, - downstream_deps=build_downstream_deps(upstream_deps), - ) - - -def _display_action( - *, materialization_type: MaterializationType, full_refresh: bool = False -) -> PlanAction: - if materialization_type == MaterializationType.VIEW: - return PlanAction.CREATE_VIEW - if materialization_type == MaterializationType.INCREMENTAL: - return PlanAction.CREATE_TABLE if full_refresh else PlanAction.INCREMENTAL_APPEND - if materialization_type == MaterializationType.CUSTOM: - return PlanAction.CUSTOM - return PlanAction.CREATE_TABLE diff --git a/src/sqlbuild/compiler/planner/main/pre_build/__init__.py b/src/sqlbuild/compiler/planner/main/pre_build/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py b/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py deleted file mode 100644 index 7915fda271..0000000000 --- a/src/sqlbuild/compiler/planner/main/pre_build/pre_build_work_display.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Plan text for storage and migration work that runs before model builds.""" - -from __future__ import annotations - -from sqlbuild._native import append_overflow_line, format_model_migrations, visible_entries -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions - - -def format_pre_build_work( - *, lines: list[str], plan: PlanOutput, display_options: DisplayOptions -) -> list[str]: - """Format retention, table-type, and migration work that runs before model builds.""" - - if plan.table_type_entries: - lines.append("Table type conversions") - for table_type_entry in visible_entries( - entries=plan.table_type_entries, options=display_options - ): - actual_type: str = table_type_entry.actual_type or "unknown" - lines.append( - f" {table_type_entry.model_name} desired={table_type_entry.desired_type} " - f"actual={actual_type} source={table_type_entry.source}" - ) - if table_type_entry.irreversible_warning is not None: - lines.append(f" WARNING: {table_type_entry.irreversible_warning}") - lines = append_overflow_line( - lines=lines, - total_count=len(plan.table_type_entries), - visible_count=len( - visible_entries(entries=plan.table_type_entries, options=display_options) - ), - indent=" ", - options=display_options, - ) - if plan.retention_entries: - lines.append("Retention") - for entry in plan.retention_entries: - scope: str = ".".join( - part - for part in (entry.request.database, entry.request.schema, entry.request.name) - if part - ) - actual: str = "missing" if entry.actual_days is None else f"{entry.actual_days}d" - effective: str = "missing" if entry.effective_days is None else f"{entry.effective_days}d" - lines.append( - f" {scope} desired={entry.request.desired_days}d actual={actual} " - f"effective={effective} source={entry.source} direction={entry.direction.value} " - f"phase={entry.phase.value}" - ) - if entry.irreversible_warning is not None: - lines.append(f" WARNING: {entry.irreversible_warning}") - return format_model_migrations(lines=lines, plan=plan, display_options=display_options) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index e3ddb6f0ba..44a05fa22b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild._native import format_plan +from sqlbuild._native import build_display_only_sqlbuild_plan, format_plan from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -15,9 +15,6 @@ CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.main.execution.display_plan import ( - build_display_only_sqlbuild_plan, -) from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import PlanAction, PlanReason from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index 7091b8cc87..33ec9ff9f4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild._native import format_pre_build_work from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import ( @@ -11,7 +12,6 @@ MigrationDiscovery, MigrationPromotion, ) -from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import format_pre_build_work from sqlbuild.compiler.planner.models import ( ColumnMigrationPlanEntry, ModelMigrationPlanEntry, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index ea481f74eb..04e9cc6423 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -4,14 +4,13 @@ import pytest -from sqlbuild._native import format_plan +from sqlbuild._native import format_plan, format_pre_build_work from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationCompatibility, MigrationDecision, MigrationDiscovery, ) -from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import format_pre_build_work from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.presentation.models import DisplayOptions from tests.unit.src.sqlbuild.compiler.planner.main.pre_build._test_types import ( From 6278842b2b546a7988af90683ea0d2d17a7632ea Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:00:02 +0100 Subject: [PATCH 07/33] chore: reconcile integrated planner assembly boundary --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 901904d9ba..2287850613 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 18084 +remaining_lines = 17507 [[stays]] path = "executor" From 6f29a9dd55a99b352e7487fc6cb8e06e29ce3379 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:59:37 +0100 Subject: [PATCH 08/33] refactor: observe, plan and store source freshness natively Source freshness observation, planning comparison, downstream propagation, latest-state reads, appends and state-table SQL now run in Rust. Adapter queries and metadata lookups stay adapter callbacks, so custom adapter overrides keep applying. The Python package keeps only its models, types, constants and errors. --- .../_helpers/freshness/freshness_command.rs | 41 +- .../_helpers/freshness/freshness_observe.rs | 187 +----- .../src/bindings/_helpers/freshness/mod.rs | 4 + .../_helpers/freshness/observation.rs | 617 ++++++++++++++++++ .../bindings/_helpers/freshness/planning.rs | 388 +++++++++++ .../_helpers/freshness/registration.rs | 11 +- .../bindings/_helpers/freshness/state_io.rs | 332 ++++++++++ .../bindings/_helpers/freshness/support.rs | 81 +++ .../_helpers/planner_identity/graph.rs | 2 +- .../_helpers/planner_warehouse/constants.rs | 8 - .../planner_warehouse/source_freshness.rs | 13 +- .../bindings/_helpers/state_rows/freshness.rs | 8 +- .../sqlbuild-state/src/freshness/constants.rs | 14 + .../src/freshness/main/configuration_error.rs | 11 + .../sqlbuild-state/src/freshness/main/mod.rs | 4 + .../src/freshness/main/read_latest_sql.rs | 13 + .../freshness/main/relevant_sources_sql.rs | 13 + .../freshness/main/single_value_problem.rs | 20 + crates/sqlbuild-state/src/freshness/models.rs | 8 + crates/sqlbuild-state/src/freshness/tests.rs | 2 + .../src/freshness/tests/test_observation.rs | 143 ++++ .../src/freshness/tests/test_types.rs | 24 + fensu.toml | 5 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 65 ++ .../adapter/contract/classes/base_adapter.py | 10 +- .../contract/classes/duckdb_backed_adapter.py | 10 +- .../bigquery/classes/bigquery_adapter.py | 10 +- .../databricks/classes/databricks_adapter.py | 10 +- .../postgres/classes/postgres_adapter.py | 10 +- .../snowflake/classes/snowflake_adapter.py | 10 +- .../sqlserver/classes/sqlserver_adapter.py | 10 +- .../_helpers/freshness/source_freshness.py | 2 +- .../source_freshness/_helpers/__init__.py | 1 - .../source_freshness/_helpers/datetime.py | 25 - .../source_freshness/_helpers/observation.py | 323 --------- .../source_freshness/_helpers/planning.py | 337 ---------- .../source_freshness/_helpers/propagation.py | 69 -- .../compiler/source_freshness/_helpers/sql.py | 119 ---- .../compiler/source_freshness/constants.py | 2 - .../source_freshness/main/__init__.py | 1 - .../source_freshness/main/_planning.py | 43 -- .../source_freshness/main/_propagation.py | 23 - .../main/adapter_observation.py | 30 - .../source_freshness/main/create_table_sql.py | 27 - .../source_freshness/main/observation.py | 30 - .../compiler/source_freshness/main/read.py | 96 --- .../source_freshness/main/read_latest_sql.py | 21 - .../compiler/source_freshness/main/write.py | 52 -- .../compiler/source_freshness/models.py | 2 +- .../cli/commands/main/janitor/helpers.py | 3 +- .../source_freshness/main/test_read_write.py | 3 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 4 +- .../test_observation.py | 8 +- .../compiler/source_freshness/main/helpers.py | 10 +- .../main/shared/_helpers/test_sql.py | 15 +- .../main/test_adapter_observation.py | 6 +- .../source_freshness/main/test_observation.py | 6 +- .../source_freshness/main/test_planning.py | 10 +- .../source_freshness/main/test_propagation.py | 4 +- .../source_freshness/main/test_read.py | 3 +- 61 files changed, 1858 insertions(+), 1503 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/configuration_error.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs create mode 100644 crates/sqlbuild-state/src/freshness/main/single_value_problem.rs create mode 100644 crates/sqlbuild-state/src/freshness/tests/test_observation.rs delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/datetime.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/observation.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/planning.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/propagation.py delete mode 100644 src/sqlbuild/compiler/source_freshness/_helpers/sql.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/__init__.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/_planning.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/_propagation.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/adapter_observation.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/create_table_sql.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/observation.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/read.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py delete mode 100644 src/sqlbuild/compiler/source_freshness/main/write.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index a71149e344..bf3216e726 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -15,9 +15,9 @@ use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness use sqlbuild_core::freshness_text::models::FreshnessSourceResult; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::freshness::freshness_observe::{ - Observer, PreviousState, observe_freshness, -}; +use crate::bindings::_helpers::freshness::freshness_observe::{PreviousState, observe_freshness}; +use crate::bindings::_helpers::freshness::observation::Observer; +use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -29,7 +29,6 @@ const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const UPSTREAM_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const STATE_READ_MODULE: &str = "sqlbuild.compiler.source_freshness.main.read"; const STATE_TABLE_NAME: &str = "_sqlbuild_source_freshness"; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; const STALE_REQUIRES_STATE_CODE: &str = "C238"; @@ -180,35 +179,23 @@ fn previous_records<'py>( ], )?), )?; - let read: Bound<'py, PyAny> = imported(py, STATE_READ_MODULE, "read_latest_source_freshness")?; + let read: StateRead<'py> = StateRead { + connection: connection.clone(), + execute: adapter.getattr("execute")?, + render_qualified_name: adapter.getattr("render_qualified_name")?, + render_read_latest_sql: adapter.getattr("render_read_latest_source_freshness_sql")?, + }; let records: Bound<'py, PyDict> = PyDict::new(py); for schema in &schemas { let exists: Bound<'py, PyDict> = keywords(py, &[("database", &database)])?; exists.set_item("schema", schema)?; exists.set_item("name", STATE_TABLE_NAME)?; - let options: Bound<'py, PyDict> = keywords( - py, - &[ - ("connection", connection), - ("execute", &adapter.getattr("execute")?), - ( - "table_exists", - &lookup.call_method("exists", (), Some(&exists))?, - ), - ("database", &database), - ( - "render_qualified_name", - &adapter.getattr("render_qualified_name")?, - ), - ( - "render_read_latest_sql", - &adapter.getattr("render_read_latest_source_freshness_sql")?, - ), - ], - )?; - options.set_item("schema", schema)?; + let table_exists: bool = lookup + .call_method("exists", (), Some(&exists))? + .is_truthy()?; + let schema: Bound<'py, PyAny> = PyString::new(py, schema).into_any(); records.update( - read.call((), Some(&options))? + read_latest(&read, (&database, &schema), table_exists, None)? .getattr("records")? .downcast()?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs index 99f698dc69..d98cb58a74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_observe.rs @@ -6,23 +6,21 @@ use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::freshness::observation::{ + Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, + strategy, +}; +use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; use sqlbuild_core::freshness_text::main::freshness_status::freshness_status; use sqlbuild_core::freshness_text::models::{ FreshnessSourceResult, FreshnessStatus, StateComparison, }; -const ADAPTER_OBSERVATION_MODULE: &str = - "sqlbuild.compiler.source_freshness.main.adapter_observation"; -const OBSERVATION_MODULE: &str = "sqlbuild.compiler.source_freshness.main.observation"; -const NATIVE_MODULE: &str = "sqlbuild._native"; -const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; -const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; const RUN_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.run_scope"; const ADAPTER_STRATEGY: &str = "adapter"; const UNAVAILABLE_REASON: &str = "unavailable"; @@ -31,25 +29,11 @@ const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavai const MISSING_STATE_MESSAGE: &str = "previous source freshness state missing"; const NOT_RETURNED_MESSAGE: &str = "freshness metadata was not returned"; -/// The adapter, connection and observation time every source is observed with. -pub(crate) struct Observer<'py> { - pub(crate) adapter: Bound<'py, PyAny>, - pub(crate) connection: Bound<'py, PyAny>, - pub(crate) observed_at: Bound<'py, PyAny>, -} - /// Previous state records by identity; absent means no comparison. pub(crate) struct PreviousState<'py> { pub(crate) by_identity: Option>, } -fn freshness_function<'py>( - py: Python<'py>, - (module, name): (&str, &str), -) -> PyResult> { - py.import(module)?.getattr(name) -} - fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); @@ -78,96 +62,6 @@ fn target_result( Ok(result) } -fn strategy(source: &Bound<'_, PyAny>) -> PyResult> { - let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; - if freshness.is_none() { - return Ok(None); - } - optional_text(freshness.getattr("strategy")?) -} - -/// The source as observed: itself with freshness config, an adapter-metadata copy, or none. -fn observation_source<'py>( - adapter: &Bound<'py, PyAny>, - source: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = source.py(); - if !source.getattr("freshness")?.is_none() { - return Ok(Some(source.clone())); - } - if !source.getattr("expression")?.is_none() - || source.getattr("table")?.is_none() - || !adapter - .call_method0("supports_table_freshness_metadata")? - .is_truthy()? - { - return Ok(None); - } - let config: Bound<'py, PyDict> = PyDict::new(py); - config.set_item( - "strategy", - py.import(CONTRACT_TYPES)? - .getattr("SourceFreshnessStrategy")? - .getattr("ADAPTER")?, - )?; - let changes: Bound<'py, PyDict> = PyDict::new(py); - changes.set_item( - "freshness", - py.import(CONTRACT_MODELS)? - .getattr("SourceFreshnessConfig")? - .call((), Some(&config))?, - )?; - Ok(Some( - py.import("dataclasses")? - .getattr("replace")? - .call((source,), Some(&changes))?, - )) -} - -/// The state record of one observation, with its normalized version and hash. -fn record<'py>( - observation: &Bound<'py, PyAny>, - source: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = observation.py(); - let normalize: Bound<'py, PyDict> = PyDict::new(py); - normalize.set_item("value", observation.getattr("data_version")?)?; - normalize.set_item("value_kind", observation.getattr("value_kind")?)?; - let data_version: Bound<'py, PyAny> = freshness_function( - py, - (NATIVE_MODULE, "normalize_source_freshness_data_version"), - )? - .call((), Some(&normalize))?; - let hash: Bound<'py, PyDict> = PyDict::new(py); - hash.set_item("source_name", observation.getattr("source_name")?)?; - hash.set_item("strategy", observation.getattr("strategy")?)?; - hash.set_item("value_kind", observation.getattr("value_kind")?)?; - hash.set_item("data_version", &data_version)?; - let data_version_hash: Bound<'py, PyAny> = - freshness_function(py, (NATIVE_MODULE, "source_freshness_data_version_hash"))? - .call((), Some(&hash))?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("source_name", observation.getattr("source_name")?)?; - fields.set_item("target_database", source.getattr("database")?)?; - fields.set_item("target_schema", source.getattr("schema")?)?; - fields.set_item("target_name", source.getattr("table")?)?; - fields.set_item("run_id", OBSERVATION_RUN_ID)?; - fields.set_item( - "strategy", - observation.getattr("strategy")?.getattr("value")?, - )?; - fields.set_item( - "value_kind", - observation.getattr("value_kind")?.getattr("value")?, - )?; - fields.set_item("data_version", &data_version)?; - fields.set_item("data_version_hash", data_version_hash)?; - fields.set_item("observed_at", observation.getattr("observed_at")?)?; - py.import(FRESHNESS_MODELS)? - .getattr("SourceFreshnessRecord")? - .call((), Some(&fields)) -} - fn previous_record<'py>( state: &PreviousState<'py>, record: &Bound<'py, PyAny>, @@ -185,7 +79,11 @@ fn observed_result( state: &PreviousState<'_>, ) -> PyResult { let py: Python<'_> = source.py(); - let current: Bound<'_, PyAny> = record(observation, source)?; + let current: Bound<'_, PyAny> = observation_record( + observation, + source, + &PyString::new(py, OBSERVATION_RUN_ID).into_any(), + )?; let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; let (lag_tolerance, policy) = if freshness.is_none() { (py.None().into_bound(py), py.None().into_bound(py)) @@ -195,27 +93,18 @@ fn observed_result( freshness.getattr("age_policy")?, ) }; - let age: Bound<'_, PyDict> = PyDict::new(py); - age.set_item("policy", policy)?; - age.set_item("data_version", observation.getattr("data_version")?)?; - age.set_item("observed_at", observation.getattr("observed_at")?)?; - let age_status: Bound<'_, PyAny> = - freshness_function(py, (NATIVE_MODULE, "evaluate_source_freshness_age_policy"))? - .call((), Some(&age))?; + let age_status: Option> = age_policy( + &policy, + &observation.getattr("data_version")?, + &observation.getattr("observed_at")?, + )?; let previous: Option> = previous_record(state, ¤t)?; let comparison: StateComparison = match &previous { None if state.by_identity.is_some() => StateComparison::Missing, None => StateComparison::NotCompared, Some(previous) => { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("previous_record", previous)?; - options.set_item("current_record", ¤t)?; - options.set_item("lag_tolerance", &lag_tolerance)?; - let equivalent: bool = - freshness_function(py, (NATIVE_MODULE, "source_freshness_records_equivalent"))? - .call((), Some(&options))? - .is_truthy()?; - if equivalent { + let tolerance: Option = lag_tolerance.extract()?; + if equivalent(previous, ¤t, tolerance.as_deref())? { StateComparison::Equivalent { same_hash: previous .getattr("data_version_hash")? @@ -242,10 +131,9 @@ fn observed_result( if comparison == StateComparison::Missing { result.message = Some(MISSING_STATE_MESSAGE.to_owned()); } - result.age_status = if age_status.is_none() { - None - } else { - Some(age_status.getattr("value")?.extract()?) + result.age_status = match age_status { + None => None, + Some(status) => Some(status.getattr("value")?.extract()?), }; Ok(result) } @@ -256,26 +144,7 @@ fn adapter_batch<'py>( sources: &[Bound<'py, PyAny>], ) -> PyResult, String>> { let py: Python<'py> = observer.adapter.py(); - let batch_class: Bound<'py, PyAny> = py - .import(FRESHNESS_MODELS)? - .getattr("AdapterSourceFreshnessBatch")?; - if sources.is_empty() { - return Ok(Ok(batch_class.call0()?)); - } - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("adapter", &observer.adapter)?; - options.set_item("connection", &observer.connection)?; - options.set_item("sources", PyTuple::new(py, sources)?)?; - options.set_item("observed_at", &observer.observed_at)?; - match freshness_function( - py, - ( - ADAPTER_OBSERVATION_MODULE, - "observe_adapter_sources_freshness", - ), - )? - .call((), Some(&options)) - { + match observe_adapter_batch(observer, sources) { Ok(batch) => Ok(Ok(batch)), Err(error) => Ok(Err(exception_message(py, error)?)), } @@ -329,17 +198,7 @@ fn configured_observation<'py>( (name, source): (&str, &Bound<'py, PyAny>), ) -> PyResult, FreshnessSourceResult>> { let py: Python<'py> = source.py(); - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("adapter", &observer.adapter)?; - options.set_item("connection", &observer.connection)?; - options.set_item("source", source)?; - options.set_item("observed_at", &observer.observed_at)?; - match freshness_function( - py, - (OBSERVATION_MODULE, "observe_configured_source_freshness"), - )? - .call((), Some(&options)) - { + match observe_configured(observer, source) { Ok(observation) => Ok(Ok(observation)), Err(error) => Ok(Err(target_result( source, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index a2e1f90b98..ed6018a87d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -1,3 +1,7 @@ pub(crate) mod freshness_command; pub(crate) mod freshness_observe; +pub(crate) mod observation; +pub(crate) mod planning; pub(crate) mod registration; +pub(crate) mod state_io; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs new file mode 100644 index 0000000000..2d8525fd51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -0,0 +1,617 @@ +//! Observe configured and adapter-metadata source freshness through adapter callbacks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; +use sqlbuild_state::freshness::models::QueryShape; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::freshness::support::{ + ADAPTER_MODELS, ADAPTER_TYPES, CONTRACT_MODELS, CONTRACT_TYPES, FRESHNESS_MODELS, + FRESHNESS_TYPES, LIFECYCLE_MODULE, OBSERVABILITY_MODELS, OBSERVATION_ERROR, exception_text, + freshness_error, presumed_utc_datetime, presumed_utc_value, text, +}; +use crate::bindings::_helpers::references::source_relations::render_source_relation; +use crate::bindings::_helpers::state_rows::freshness::{normalize, version_hash}; +use crate::bindings::_helpers::state_rows::timestamps::{in_utc, offset_aware}; + +const ADAPTER_STRATEGY: &str = "adapter"; +const COLUMN_STRATEGY: &str = "column"; +const DEFAULT_STRATEGY: &str = "sql"; +const METADATA_OPERATION: &str = "source_freshness_metadata_observation"; +const QUERY_OPERATION: &str = "source_freshness_query_observation"; + +/// The adapter, connection and observation time every source is observed with. +pub(crate) struct Observer<'py> { + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) observed_at: Bound<'py, PyAny>, +} + +fn observation_error(py: Python<'_>, message: &str) -> pyo3::PyErr { + freshness_error(py, (OBSERVATION_ERROR, message), None) +} + +/// The configured strategy value of a source, or `None` without freshness config. +pub(crate) fn strategy(source: &Bound<'_, PyAny>) -> PyResult> { + let freshness: Bound<'_, PyAny> = source.getattr("freshness")?; + if freshness.is_none() { + return Ok(None); + } + Ok(Some(freshness.getattr("strategy")?.str()?.extract()?)) +} + +/// The source as observed: itself with freshness config, an adapter-metadata copy, or none. +pub(crate) fn observation_source<'py>( + adapter: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = source.py(); + if !source.getattr("freshness")?.is_none() { + return Ok(Some(source.clone())); + } + if !source.getattr("expression")?.is_none() + || source.getattr("table")?.is_none() + || !adapter + .call_method0("supports_table_freshness_metadata")? + .is_truthy()? + { + return Ok(None); + } + let config: Bound<'py, PyAny> = call_with( + &imported(py, CONTRACT_MODELS, "SourceFreshnessConfig")?, + &[("strategy", adapter_strategy(py)?)], + )?; + replaced_source(source, config).map(Some) +} + +fn replaced_source<'py>( + source: &Bound<'py, PyAny>, + config: Bound<'py, PyAny>, +) -> PyResult> { + let changes: Bound<'py, PyDict> = PyDict::new(source.py()); + changes.set_item("freshness", config)?; + imported(source.py(), "dataclasses", "replace")?.call((source,), Some(&changes)) +} + +fn adapter_strategy(py: Python<'_>) -> PyResult> { + imported(py, CONTRACT_TYPES, "SourceFreshnessStrategy")?.getattr("ADAPTER") +} + +/// `OperationLifecycle(operation_kind="freshness", ...)` for one observation phase. +fn lifecycle<'py>( + py: Python<'py>, + (operation_name, strategy): (&str, &str), + item_count: Option, +) -> PyResult> { + let attributes: Bound<'py, PyAny> = call_with( + &imported(py, OBSERVABILITY_MODELS, "OperationAttributes")?, + &[ + ("phase", text(py, "observe")), + ("strategy", text(py, strategy)), + ("target_kind", text(py, "source")), + ], + )?; + let mut options: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("operation_kind", text(py, "freshness")), + ("operation_name", text(py, operation_name)), + ]; + if let Some(item_count) = item_count { + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", item_count)?; + options.push(("metadata", metadata.into_any())); + } + options.push(("attributes", attributes)); + call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &options, + ) +} + +fn observation<'py>( + source_name: &Bound<'py, PyAny>, + (strategy, value_kind): (Bound<'py, PyAny>, Bound<'py, PyAny>), + (data_version, observed_at): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + call_with( + &imported( + source_name.py(), + FRESHNESS_MODELS, + "SourceFreshnessObservation", + )?, + &[ + ("source_name", source_name.clone()), + ("strategy", strategy), + ("data_version", data_version), + ("value_kind", value_kind), + ("observed_at", observed_at), + ], + ) +} + +/// Observe one source freshness config and return a comparable data version. +pub(crate) fn observe_configured<'py>( + observer: &Observer<'py>, + source: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source.py(); + let strategy: Option = strategy(source)?; + let operation: &str = if strategy.as_deref() == Some(ADAPTER_STRATEGY) { + METADATA_OPERATION + } else { + QUERY_OPERATION + }; + let manager: Bound<'py, PyAny> = lifecycle( + py, + (operation, strategy.as_deref().unwrap_or(DEFAULT_STRATEGY)), + None, + )?; + within(&manager, |_| { + observe_configured_inner(observer, source, strategy.as_deref()) + }) +} + +fn observe_configured_inner<'py>( + observer: &Observer<'py>, + source: &Bound<'py, PyAny>, + strategy: Option<&str>, +) -> PyResult> { + let py: Python<'py> = source.py(); + let name: Bound<'py, PyAny> = source.getattr("name")?; + let config: Bound<'py, PyAny> = source.getattr("freshness")?; + let Some(strategy) = strategy else { + return Err(observation_error( + py, + &format!( + "source '{}' does not define freshness configuration", + name.str()? + ), + )); + }; + if strategy == ADAPTER_STRATEGY { + return observe_adapter_source(observer, source); + } + let value_kind: Bound<'py, PyAny> = config.getattr("value_kind")?; + let sql: Bound<'py, PyAny> = if strategy == COLUMN_STRATEGY { + let column: Bound<'py, PyAny> = config.getattr("column")?; + if column.is_none() || value_kind.is_none() { + return Err(observation_error( + py, + &format!( + "source '{}' has incomplete column freshness configuration", + name.str()? + ), + )); + } + let filter: Bound<'py, PyAny> = config.getattr("filter")?; + let where_sql: String = if filter.is_none() { + String::new() + } else { + format!(" WHERE {}", filter.str()?) + }; + let relation: String = render_source_relation(source, Some(&observer.adapter))?; + call_with( + &observer + .adapter + .getattr("render_source_freshness_max_query")?, + &[ + ("column", column), + ("source_relation", text(py, &relation)), + ( + "source_is_subquery", + PyBool::new(py, !source.getattr("expression")?.is_none()) + .to_owned() + .into_any(), + ), + ("where_sql", text(py, &where_sql)), + ], + )? + } else { + let query: Bound<'py, PyAny> = config.getattr("query")?; + if query.is_none() || value_kind.is_none() { + return Err(observation_error( + py, + &format!( + "source '{}' has incomplete sql freshness configuration", + name.str()? + ), + )); + } + query + }; + let data_version: Bound<'py, PyAny> = + presumed_utc_value(&single_data_version(observer, &name, sql)?)?; + observation( + &name, + (config.getattr("strategy")?, value_kind), + (data_version, observer.observed_at.clone()), + ) +} + +fn single_data_version<'py>( + observer: &Observer<'py>, + source_name: &Bound<'py, PyAny>, + sql: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_name.py(); + let name: String = source_name.str()?.extract()?; + let result: Bound<'py, PyAny> = match call_with( + &observer.adapter.getattr("query")?, + &[ + ("connection", observer.connection.clone()), + ("sql", sql), + ("limit", py.None().into_bound(py)), + ], + ) { + Ok(result) => result, + Err(error) => { + let Some(message) = exception_text(py, &error)? else { + return Err(error); + }; + return Err(freshness_error( + py, + ( + OBSERVATION_ERROR, + &format!("source '{name}' freshness query failed: {message}"), + ), + Some(error), + )); + } + }; + let columns: usize = result.getattr("columns")?.len()?; + let rows: Bound<'py, PyAny> = result.getattr("rows")?; + let row_count: usize = rows.len()?; + let first: Option> = if columns == 1 && row_count == 1 { + Some(rows.get_item(0)?.get_item(0)?) + } else { + None + }; + let shape: QueryShape = QueryShape { + columns, + rows: row_count, + first_value_null: first.as_ref().is_some_and(|value| value.is_none()), + }; + if let Some(problem) = single_value_problem(&name, shape) { + return Err(observation_error(py, &problem)); + } + first.ok_or_else(|| observation_error(py, "freshness query returned no value")) +} + +fn unsupported_message( + adapter: &Bound<'_, PyAny>, + source: &Bound<'_, PyAny>, +) -> PyResult> { + if !adapter + .call_method0("supports_table_freshness_metadata")? + .is_truthy()? + { + return Ok(Some(format!( + "adapter '{}' does not support table freshness metadata", + adapter.getattr("adapter_name")?.str()? + ))); + } + if !source.getattr("expression")?.is_none() || source.getattr("table")?.is_none() { + return Ok(Some(format!( + "source '{}' adapter freshness requires a physical table source", + source.getattr("name")?.str()? + ))); + } + Ok(None) +} + +/// `SourceFreshnessUnknown(source_name=..., reason=, message=...)`. +pub(crate) fn unknown<'py>( + source_name: &Bound<'py, PyAny>, + (reason, message): (&str, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = source_name.py(); + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessUnknown")?, + &[ + ("source_name", source_name.clone()), + ( + "reason", + imported(py, FRESHNESS_TYPES, "SourceFreshnessUnknownReason")?.getattr(reason)?, + ), + ("message", message), + ], + ) +} + +/// The unknown outcome of unavailable metadata or a null data version, if any. +fn metadata_unknown<'py>( + source_name: &Bound<'py, PyAny>, + metadata: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = source_name.py(); + let name: String = source_name.str()?.extract()?; + let unavailable: Bound<'py, PyAny> = + imported(py, ADAPTER_TYPES, "TableFreshnessStatus")?.getattr("UNAVAILABLE")?; + if metadata.getattr("status")?.eq(unavailable)? { + let message: Bound<'py, PyAny> = metadata.getattr("message")?; + let message: String = if message.is_truthy()? { + message.str()?.extract()? + } else { + format!("source '{name}' freshness metadata unavailable") + }; + return unknown(source_name, ("UNAVAILABLE", text(py, &message))).map(Some); + } + if metadata.getattr("data_version")?.is_none() { + let message: String = format!("source '{name}' freshness data_version is null"); + return unknown(source_name, ("UNAVAILABLE", text(py, &message))).map(Some); + } + Ok(None) +} + +fn metadata_observation<'py>( + source_name: &Bound<'py, PyAny>, + metadata: &Bound<'py, PyAny>, + fallback_observed_at: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_name.py(); + let observed_at: Bound<'py, PyAny> = metadata.getattr("observed_at")?; + let observed_at: Bound<'py, PyAny> = if observed_at.is_none() { + fallback_observed_at.clone() + } else { + presumed_utc_datetime(&observed_at)? + }; + observation( + source_name, + ( + adapter_strategy(py)?, + imported(py, CONTRACT_TYPES, "SourceFreshnessValueKind")? + .call1((metadata.getattr("value_kind")?,))?, + ), + ( + presumed_utc_value(&metadata.getattr("data_version")?)?, + observed_at, + ), + ) +} + +fn observe_adapter_source<'py>( + observer: &Observer<'py>, + source: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source.py(); + if let Some(message) = unsupported_message(&observer.adapter, source)? { + return Err(observation_error(py, &message)); + } + let metadata: Bound<'py, PyAny> = call_with( + &observer.adapter.getattr("get_table_freshness_metadata")?, + &[ + ("connection", observer.connection.clone()), + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", source.getattr("table")?), + ], + )?; + let name: Bound<'py, PyAny> = source.getattr("name")?; + if let Some(unknown) = metadata_unknown(&name, &metadata)? { + return Err(observation_error( + py, + &unknown.getattr("message")?.str()?.extract::()?, + )); + } + metadata_observation(&name, &metadata, &observer.observed_at) +} + +/// Observe adapter metadata freshness per source, batching the warehouse lookups. +pub(crate) fn observe_adapter_batch<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = observer.adapter.py(); + let batch_class: Bound<'py, PyAny> = + imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?; + if sources.is_empty() { + return batch_class.call0(); + } + let manager: Bound<'py, PyAny> = lifecycle( + py, + (METADATA_OPERATION, ADAPTER_STRATEGY), + Some(sources.len()), + )?; + within(&manager, |entered| { + let batch: Bound<'py, PyAny> = observe_adapter_batch_inner(observer, sources)?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", sources.len())?; + metadata.set_item("unknown_count", batch.getattr("unknown")?.len()?)?; + call_with( + &entered.getattr("completed")?, + &[("metadata", metadata.into_any())], + )?; + Ok(batch) + }) +} + +fn observe_adapter_batch_inner<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = observer.adapter.py(); + let batch_class: Bound<'py, PyAny> = + imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?; + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + let requests: Bound<'py, PyDict> = PyDict::new(py); + let request_class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS, "TableFreshnessRequest")?; + for source in sources { + let name: Bound<'py, PyAny> = source.getattr("name")?; + if let Some(message) = unsupported_message(&observer.adapter, source)? { + unknown_sources + .set_item(&name, unknown(&name, ("UNAVAILABLE", text(py, &message)))?)?; + continue; + } + requests.set_item( + &name, + call_with( + &request_class, + &[ + ("database", source.getattr("database")?), + ("schema", source.getattr("schema")?), + ("name", source.getattr("table")?.str()?.into_any()), + ], + )?, + )?; + } + if requests.is_empty() { + return call_with(&batch_class, &[("unknown", unknown_sources.into_any())]); + } + let unique: Bound<'py, PyDict> = PyDict::new(py); + for request in requests.values() { + unique.set_item(request, py.None())?; + } + let metadata_by_request: Bound<'py, PyAny> = call_with( + &observer.adapter.getattr("get_tables_freshness_metadata")?, + &[ + ("connection", observer.connection.clone()), + ("requests", PyTuple::new(py, unique.keys())?.into_any()), + ], + )?; + let metadata_class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS, "TableFreshnessMetadata")?; + let observations: Bound<'py, PyDict> = PyDict::new(py); + for (name, request) in requests.iter() { + let missing: Bound<'py, PyAny> = call_with( + &metadata_class.getattr("unavailable")?, + &[( + "message", + text( + py, + &format!( + "source '{}' freshness metadata was not returned", + name.str()? + ), + ), + )], + )?; + let metadata: Bound<'py, PyAny> = + metadata_by_request.call_method1("get", (request, missing))?; + if let Some(unknown) = metadata_unknown(&name, &metadata)? { + unknown_sources.set_item(&name, unknown)?; + continue; + } + observations.set_item( + &name, + metadata_observation(&name, &metadata, &observer.observed_at)?, + )?; + } + call_with( + &batch_class, + &[ + ("observations", observations.into_any()), + ("unknown", unknown_sources.into_any()), + ], + ) +} + +/// The state record of one observation, with its normalized version and hash. +pub(crate) fn observation_record<'py>( + observation: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = observation.py(); + let source_name: Bound<'py, PyAny> = observation.getattr("source_name")?; + let strategy: Bound<'py, PyAny> = observation.getattr("strategy")?; + let value_kind: Bound<'py, PyAny> = observation.getattr("value_kind")?; + let data_version: Bound<'py, PyAny> = + normalize(&observation.getattr("data_version")?, &value_kind)?; + let data_version_hash: String = version_hash( + &source_name.extract::()?, + &strategy.extract::()?, + &value_kind.extract::()?, + &data_version.extract::()?, + )?; + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessRecord")?, + &[ + ("source_name", source_name), + ("target_database", source.getattr("database")?), + ("target_schema", source.getattr("schema")?), + ("target_name", source.getattr("table")?), + ("run_id", run_id.clone()), + ("strategy", strategy.getattr("value")?), + ("value_kind", value_kind.getattr("value")?), + ("data_version", data_version), + ("data_version_hash", text(py, &data_version_hash)), + ("observed_at", observation.getattr("observed_at")?), + ], + ) +} + +/// Observe one source freshness config and return a comparable data version. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, source, observed_at))] +fn observe_configured_source_freshness<'py>( + adapter: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, + observed_at: Bound<'py, PyAny>, +) -> PyResult> { + let observer: Observer<'py> = Observer { + adapter, + connection, + observed_at, + }; + observe_configured(&observer, source) +} + +/// Observe adapter metadata freshness per source, batching the warehouse lookups. +#[pyfunction] +#[pyo3(signature = (*, adapter, connection, sources, observed_at))] +fn observe_adapter_sources_freshness<'py>( + adapter: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, + observed_at: Bound<'py, PyAny>, +) -> PyResult> { + let observer: Observer<'py> = Observer { + adapter, + connection, + observed_at, + }; + let sources: Vec> = sources.try_iter()?.collect::>()?; + observe_adapter_batch(&observer, &sources) +} + +/// Build the state record of one observation. +#[pyfunction] +#[pyo3(signature = (*, observation, source, run_id))] +fn source_freshness_record_from_observation<'py>( + observation: &Bound<'py, PyAny>, + source: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + observation_record(observation, source, run_id) +} + +/// An aware freshness `datetime` in UTC; a naive value is an observation error. +#[pyfunction] +#[pyo3(signature = (*, value, field_name))] +fn require_aware_utc_datetime<'py>( + value: &Bound<'py, PyAny>, + field_name: &str, +) -> PyResult> { + if !offset_aware(value)? { + return Err(observation_error( + value.py(), + &format!("source freshness {field_name} must be timezone-aware"), + )); + } + in_utc(value) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(require_aware_utc_datetime, module)?)?; + module.add_function(wrap_pyfunction!( + observe_configured_source_freshness, + module + )?)?; + module.add_function(wrap_pyfunction!(observe_adapter_sources_freshness, module)?)?; + module.add_function(wrap_pyfunction!( + source_freshness_record_from_observation, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs new file mode 100644 index 0000000000..e352a88857 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -0,0 +1,388 @@ +//! Planning-time source freshness: observe, compare with latest state and map downstream impact. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::main::configuration_error::configuration_error; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::freshness::observation::{ + Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, + strategy, unknown, +}; +use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; +use crate::bindings::_helpers::freshness::support::{ + ADAPTER_EXCEPTIONS, EXCEPTIONS_MODULE, FRESHNESS_MODELS, FRESHNESS_TYPES, OBSERVATION_ERROR, + text, +}; +use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; +use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; + +const ADAPTER_STRATEGY: &str = "adapter"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; + +/// The message of an `AdapterUserError` or a source freshness observation error, if it is one. +fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult>> { + let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); + if value.is_instance(&imported(py, ADAPTER_EXCEPTIONS, "AdapterUserError")?)? { + return value.getattr("message").map(Some); + } + if value.is_instance(&imported(py, EXCEPTIONS_MODULE, OBSERVATION_ERROR)?)? { + return Ok(Some(value.str()?.into_any())); + } + Ok(None) +} + +fn freshness_strategy_is_adapter(source: &Bound<'_, PyAny>) -> PyResult { + Ok(strategy(source)?.as_deref() == Some(ADAPTER_STRATEGY)) +} + +/// Latest previous records by identity across the state schemas, newest observation winning. +fn previous_records<'py>( + arguments: &KeywordArguments<'py>, + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_names: Option<&[String]>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let read: StateRead<'py> = StateRead { + connection: connection.clone(), + execute: adapter.getattr("execute")?, + render_qualified_name: arguments.value("render_qualified_name")?, + render_read_latest_sql: adapter.getattr("render_read_latest_source_freshness_sql")?, + }; + let database: Bound<'py, PyAny> = arguments.value("state_database")?; + let exists_by_schema: Bound<'py, PyAny> = arguments.value("state_table_exists_by_schema")?; + let merged: Bound<'py, PyDict> = PyDict::new(py); + for schema in arguments.value("state_schemas")?.try_iter()? { + let schema: Bound<'py, PyAny> = schema?; + let exists: bool = exists_by_schema + .call_method1("get", (&schema, false))? + .is_truthy()?; + let previous: Bound<'py, PyAny> = + read_latest(&read, (&database, &schema), exists, source_names)?; + for (identity, candidate) in previous.getattr("records")?.downcast::()?.iter() { + let replace: bool = match merged.get_item(&identity)? { + None => true, + Some(current) => candidate + .getattr("observed_at")? + .gt(current.getattr("observed_at")?)?, + }; + if replace { + merged.set_item(identity, candidate)?; + } + } + } + Ok(merged) +} + +/// Records sorted by `str(record.identity)`, stably. +fn sorted_by_identity<'py>( + records: impl Iterator>, +) -> PyResult>> { + let mut keyed: Vec<(String, Bound<'py, PyAny>)> = records + .map(|record| Ok((record.getattr("identity")?.str()?.extract()?, record))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, record)| record).collect()) +} + +fn sorted_dict<'py>(py: Python<'py>, items: &Bound<'py, PyDict>) -> PyResult> { + let mut keyed: Vec<(String, Bound<'py, PyAny>)> = items + .iter() + .map(|(name, value)| Ok((name.extract()?, value))) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + let sorted: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in keyed { + sorted.set_item(name, value)?; + } + Ok(sorted) +} + +/// The adapter batch, or every source unknown with the batch failure's message. +fn adapter_batch<'py>( + observer: &Observer<'py>, + sources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = observer.adapter.py(); + let error: PyErr = match observe_adapter_batch(observer, sources) { + Ok(batch) => return Ok(batch), + Err(error) => error, + }; + let Some(message) = handled_message(py, &error)? else { + return Err(error); + }; + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + for source in sources { + let name: Bound<'py, PyAny> = source.getattr("name")?; + unknown_sources.set_item(&name, unknown(&name, ("ERROR", message.clone()))?)?; + } + call_with( + &imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?, + &[("unknown", unknown_sources.into_any())], + ) +} + +/// Observe direct source freshness and compare it to latest append-only state. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_direct_source_freshness_planning_result<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_direct_source_freshness_planning_result", + options, + ( + &[ + "adapter", + "connection", + "sources", + "state_database", + "state_schemas", + "observed_at", + "run_id", + "render_qualified_name", + "state_table_exists_by_schema", + ], + &["filter_previous_to_sources"], + ), + )?; + let observer: Observer<'py> = Observer { + adapter: arguments.value("adapter")?, + connection: arguments.value("connection")?, + observed_at: arguments.value("observed_at")?, + }; + let run_id: Bound<'py, PyAny> = arguments.value("run_id")?; + let sources: Vec> = arguments + .value("sources")? + .try_iter()? + .collect::>()?; + let source_names: Option> = + if arguments.value("filter_previous_to_sources")?.is_truthy()? { + let mut names: Vec = sources + .iter() + .map(|source| source.getattr("name")?.extract()) + .collect::>()?; + names.sort(); + Some(names) + } else { + None + }; + let previous: Bound<'py, PyDict> = previous_records( + &arguments, + (&observer.adapter, &observer.connection), + source_names.as_deref(), + )?; + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + let observed_sources: Bound<'py, PyDict> = PyDict::new(py); + let mut adapter_sources: Vec> = Vec::new(); + for source in &sources { + if source.getattr("managed")?.is_truthy()? { + continue; + } + let name: Bound<'py, PyAny> = source.getattr("name")?; + let Some(observed) = observation_source(&observer.adapter, source)? else { + unknown_sources.set_item( + &name, + unknown(&name, ("NO_FRESHNESS_CONFIG", text(py, NO_CONFIG_MESSAGE)))?, + )?; + continue; + }; + observed_sources.set_item(&name, &observed)?; + if freshness_strategy_is_adapter(&observed)? { + adapter_sources.push(observed); + } + } + let batch: Bound<'py, PyAny> = adapter_batch(&observer, &adapter_sources)?; + unknown_sources.update(batch.getattr("unknown")?.downcast()?)?; + let observations: Bound<'py, PyAny> = batch.getattr("observations")?; + let mut observed_records: Vec> = Vec::new(); + let changed: Bound<'py, PySet> = PySet::empty(py)?; + let unchanged: Bound<'py, PySet> = PySet::empty(py)?; + let age_statuses: Bound<'py, PyDict> = PyDict::new(py); + for (name, source) in observed_sources.iter() { + let observation: Bound<'py, PyAny> = if freshness_strategy_is_adapter(&source)? { + let observation: Bound<'py, PyAny> = observations.call_method1("get", (&name,))?; + if observation.is_none() { + continue; + } + observation + } else { + match observe_configured(&observer, &source) { + Ok(observation) => observation, + Err(error) => { + let Some(message) = handled_message(py, &error)? else { + return Err(error); + }; + let is_observation_error: bool = !error.value(py).is_instance(&imported( + py, + ADAPTER_EXCEPTIONS, + "AdapterUserError", + )?)?; + if is_observation_error && configuration_error(&message.extract::()?) { + return Err(error); + } + unknown_sources.set_item(&name, unknown(&name, ("ERROR", message))?)?; + continue; + } + } + }; + let record: Bound<'py, PyAny> = observation_record(&observation, &source, &run_id)?; + observed_records.push(record.clone()); + let freshness: Bound<'py, PyAny> = source.getattr("freshness")?; + let (policy, lag_tolerance) = if freshness.is_none() { + (py.None().into_bound(py), None) + } else { + ( + freshness.getattr("age_policy")?, + freshness + .getattr("lag_tolerance")? + .extract::>()?, + ) + }; + let identity: Bound<'py, PyAny> = record.getattr("identity")?; + if let Some(status) = age_policy( + &policy, + &observation.getattr("data_version")?, + &observation.getattr("observed_at")?, + )? { + age_statuses.set_item(&identity, status)?; + } + match previous.get_item(&identity)? { + None => changed.add(&identity)?, + Some(previous_record) => { + if equivalent(&previous_record, &record, lag_tolerance.as_deref())? { + unchanged.add(&identity)?; + } else { + changed.add(&identity)?; + } + } + } + } + let unknown_names: Bound<'py, PyDict> = sorted_dict(py, &unknown_sources)?; + call_with( + &imported(py, FRESHNESS_MODELS, "DirectSourceFreshnessPlanningResult")?, + &[ + ( + "observed_records", + PyTuple::new(py, sorted_by_identity(observed_records.into_iter())?)?.into_any(), + ), + ( + "previous_records", + PyTuple::new(py, sorted_by_identity(previous.values().into_iter())?)?.into_any(), + ), + ("changed_identities", frozen(&changed)?), + ("unchanged_identities", frozen(&unchanged)?), + ( + "unknown_source_names", + PyTuple::new(py, unknown_names.keys())?.into_any(), + ), + ("unknown_sources", unknown_names.into_any()), + ("age_statuses", age_statuses.into_any()), + ], + ) +} + +/// `frozenset(items)`, keeping CPython's copied set layout and iteration order. +fn frozen<'py>(items: &Bound<'py, PySet>) -> PyResult> { + items.py().get_type::().call1((items,)) +} + +fn downstream_models<'py>( + source_name: &Bound<'py, PyAny>, + downstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = source_name.py(); + let key: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS, "CompiledObjectKey")?, + &[ + ( + "resource_type", + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?, + ), + ("name", source_name.clone()), + ], + )?; + build_downstream_model_name_closure(&PyTuple::new(py, [key])?.into_any(), downstream_deps) +} + +/// Map changed and unknown source freshness roots to downstream model names. +#[pyfunction] +#[pyo3(signature = (*, source_freshness, scope))] +pub(crate) fn build_direct_source_freshness_propagation_result<'py>( + source_freshness: &Bound<'py, PyAny>, + scope: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let downstream_deps: Bound<'py, PyAny> = scope.getattr("downstream_deps")?; + let changed: Bound<'py, PyDict> = PyDict::new(py); + let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); + let errors: Bound<'py, PyDict> = PyDict::new(py); + let stale: Bound<'py, PySet> = PySet::empty(py)?; + let blocked: Bound<'py, PySet> = PySet::empty(py)?; + for identity in source_freshness.getattr("changed_identities")?.try_iter()? { + let identity: Bound<'py, PyAny> = identity?; + let models: Bound<'py, PyFrozenSet> = + downstream_models(&identity.getattr("source_name")?, &downstream_deps)?; + stale.call_method1("update", (&models,))?; + changed.set_item(identity, models)?; + } + for name in source_freshness + .getattr("unknown_source_names")? + .try_iter()? + { + let name: Bound<'py, PyAny> = name?; + let models: Bound<'py, PyFrozenSet> = downstream_models(&name, &downstream_deps)?; + stale.call_method1("update", (&models,))?; + unknown_sources.set_item(name, models)?; + } + let error_status: Bound<'py, PyAny> = + imported(py, FRESHNESS_TYPES, "SourceFreshnessAgeStatus")?.getattr("ERROR")?; + for (identity, status) in source_freshness + .getattr("age_statuses")? + .downcast::()? + .iter() + { + if status.ne(&error_status)? { + continue; + } + let models: Bound<'py, PyFrozenSet> = + downstream_models(&identity.getattr("source_name")?, &downstream_deps)?; + blocked.call_method1("update", (&models,))?; + errors.set_item(identity, models)?; + } + call_with( + &imported( + py, + FRESHNESS_MODELS, + "DirectSourceFreshnessPropagationResult", + )?, + &[ + ("changed_source_model_names", changed.into_any()), + ("unknown_source_model_names", unknown_sources.into_any()), + ("error_source_model_names", errors.into_any()), + ("stale_model_names", frozen(&stale)?), + ("blocked_model_names", frozen(&blocked)?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_planning_result, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_direct_source_freshness_propagation_result, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index 0f2cffdbe0..050369cb2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -1,10 +1,15 @@ -//! Register the native `sqb freshness` command and its observation entry. +//! Register the native `sqb freshness` command, observation, planning and state functions. use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::freshness::{freshness_command, freshness_observe}; +use crate::bindings::_helpers::freshness::{ + freshness_command, freshness_observe, observation, planning, state_io, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness_command::register(module)?; - freshness_observe::register(module) + freshness_observe::register(module)?; + observation::register(module)?; + planning::register(module)?; + state_io::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs new file mode 100644 index 0000000000..be7f68cdd4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -0,0 +1,332 @@ +//! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; +use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::freshness::support::{ + FRESHNESS_CONSTANTS, FRESHNESS_MODELS, INPUT_ERROR, STATE_TABLE_DDL_MODULE, exception_text, + freshness_error, optional_str, presumed_utc_datetime, text, +}; +use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; + +const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; + +/// The qualified state table name the adapter renders for one target schema. +pub(crate) fn qualified_table_name( + render_qualified_name: &Bound<'_, PyAny>, + (database, schema): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = render_qualified_name.py(); + let name: Bound<'_, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", text(py, SOURCE_FRESHNESS_TABLE_NAME)), + ], + )?; + if name.is_none() { + return Err(freshness_error( + py, + (INPUT_ERROR, MISSING_SCHEMA_MESSAGE), + None, + )); + } + name.extract() +} + +/// SQL selecting the latest source freshness row per identity. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name))] +fn build_source_freshness_read_latest_sql( + database: &Bound<'_, PyAny>, + schema: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + Ok(read_latest_sql(&qualified_table_name( + render_qualified_name, + (database, schema), + )?)) +} + +/// Create-if-missing DDL for the source freshness state table. +#[pyfunction] +#[pyo3(signature = (*, database, schema, render_qualified_name, render_framework_type))] +fn build_source_freshness_create_table_sql<'py>( + database: &Bound<'py, PyAny>, + schema: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult> { + create_table_sql( + (database, schema), + render_qualified_name, + render_framework_type, + ) +} + +fn create_table_sql<'py>( + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + render_qualified_name: &Bound<'py, PyAny>, + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = database.py(); + let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; + let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; + call_with( + &imported(py, STATE_TABLE_DDL_MODULE, "render_state_table_create_sql")?, + &[ + ("qualified_name", text(py, &qualified_name)), + ("columns", constants.getattr("SOURCE_FRESHNESS_COLUMNS")?), + ( + "column_types", + constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?, + ), + ( + "required_columns", + PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(), + ), + ("render_framework_type", render_framework_type.clone()), + ], + ) +} + +/// The callbacks and location one latest-state read uses. +pub(crate) struct StateRead<'py> { + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) execute: Bound<'py, PyAny>, + pub(crate) render_qualified_name: Bound<'py, PyAny>, + pub(crate) render_read_latest_sql: Bound<'py, PyAny>, +} + +fn empty_set<'py>(schema: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = schema.py(); + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessSet")?, + &[ + ("schema", schema.clone()), + ("records", PyDict::new(py).into_any()), + ], + ) +} + +/// Latest records of one schema, trusting the caller-resolved table existence. +pub(crate) fn read_latest<'py>( + read: &StateRead<'py>, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + table_exists: bool, + source_names: Option<&[String]>, +) -> PyResult> { + let py: Python<'py> = schema.py(); + let qualified_name: String = + qualified_table_name(&read.render_qualified_name, (database, schema))?; + if !table_exists { + return empty_set(schema); + } + let mut read_sql: String = call_with( + &read.render_read_latest_sql, + &[("database", database.clone()), ("schema", schema.clone())], + )? + .extract()?; + if let Some(source_names) = source_names { + if source_names.is_empty() { + return empty_set(schema); + } + read_sql = relevant_sources_sql(&read_sql, source_names); + } + let result: Bound<'py, PyAny> = match call_with( + &read.execute, + &[ + ("connection", read.connection.clone()), + ("sql", text(py, &read_sql)), + ], + ) { + Ok(result) => result, + Err(error) => { + if exception_text(py, &error)?.is_none() { + return Err(error); + } + return Err(freshness_error( + py, + ( + INPUT_ERROR, + &format!( + "Unable to read source freshness records from {qualified_name}. This can \ +happen after upgrading from an older sqlbuild version; delete or rebuild the SQLBuild source \ +freshness table to regenerate source freshness state." + ), + ), + Some(error), + )); + } + }; + let records: Bound<'py, PyDict> = PyDict::new(py); + for row in result.call_method0("fetchall")?.try_iter()? { + let record: Bound<'py, PyAny> = row_record(&row?)?; + records.set_item(record.getattr("identity")?, record)?; + } + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessSet")?, + &[("schema", schema.clone()), ("records", records.into_any())], + ) +} + +fn row_record<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = row.py(); + let raw_observed_at: Bound<'py, PyAny> = row.get_item(9)?; + let observed_at: Bound<'py, PyAny> = if date_time_typed(&raw_observed_at)? { + raw_observed_at + } else { + imported(py, DATETIME_MODULE, "datetime")? + .call_method1("fromisoformat", (raw_observed_at.str()?,))? + }; + let observed_at: Bound<'py, PyAny> = presumed_utc_datetime(&observed_at)?; + let field = |index: usize| -> PyResult> { + Ok(row.get_item(index)?.str()?.into_any()) + }; + let optional = + |index: usize| -> PyResult> { optional_str(&row.get_item(index)?) }; + call_with( + &imported(py, FRESHNESS_MODELS, "SourceFreshnessRecord")?, + &[ + ("source_name", field(0)?), + ("target_database", optional(1)?), + ("target_schema", optional(2)?), + ("target_name", optional(3)?), + ("run_id", field(4)?), + ("strategy", field(5)?), + ("value_kind", field(6)?), + ("data_version", optional(7)?), + ("data_version_hash", field(8)?), + ("observed_at", observed_at), + ], + ) +} + +/// Read latest source freshness rows, trusting the caller-resolved table existence. +#[pyfunction] +#[pyo3(signature = (**options))] +fn read_latest_source_freshness<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "read_latest_source_freshness", + options, + ( + &[ + "connection", + "execute", + "table_exists", + "database", + "schema", + "render_qualified_name", + "render_read_latest_sql", + ], + &["source_names"], + ), + )?; + let read: StateRead<'py> = StateRead { + connection: arguments.value("connection")?, + execute: arguments.value("execute")?, + render_qualified_name: arguments.value("render_qualified_name")?, + render_read_latest_sql: arguments.value("render_read_latest_sql")?, + }; + let source_names: Option> = arguments.value("source_names")?.extract()?; + read_latest( + &read, + (&arguments.value("database")?, &arguments.value("schema")?), + arguments.value("table_exists")?.is_truthy()?, + source_names.as_deref(), + ) +} + +fn execute_sql( + (execute, connection): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + sql: Bound<'_, PyAny>, +) -> PyResult<()> { + call_with(execute, &[("connection", connection.clone()), ("sql", sql)])?; + Ok(()) +} + +/// Append source freshness rows, creating the table once if needed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_source_freshness_records<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<()> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "write_source_freshness_records", + options, + ( + &[ + "connection", + "execute", + "database", + "schema", + "records", + "renderers", + ], + &[], + ), + )?; + let connection: &Bound<'py, PyAny> = &arguments.value("connection")?; + let execute: &Bound<'py, PyAny> = &arguments.value("execute")?; + let database: &Bound<'py, PyAny> = &arguments.value("database")?; + let schema: &Bound<'py, PyAny> = &arguments.value("schema")?; + let records: &Bound<'py, PyAny> = &arguments.value("records")?; + let renderers: &Bound<'py, PyAny> = &arguments.value("renderers")?; + if !records.is_truthy()? { + return Ok(()); + } + let location = || -> Vec<(&str, Bound<'py, PyAny>)> { + vec![("database", database.clone()), ("schema", schema.clone())] + }; + let render_create: Bound<'py, PyAny> = renderers.getattr("render_create_table_sql")?; + let create_sql: Bound<'py, PyAny> = if render_create.is_none() { + create_table_sql( + (database, schema), + &renderers.getattr("render_qualified_name")?, + &renderers.getattr("render_framework_type")?, + )? + } else { + call_with(&render_create, &location())? + }; + execute_sql((execute, connection), create_sql)?; + let render_indexes: Bound<'py, PyAny> = renderers.getattr("render_create_index_sqls")?; + if !render_indexes.is_none() { + for index_sql in call_with(&render_indexes, &location())?.try_iter()? { + execute_sql((execute, connection), index_sql?)?; + } + } + let mut insert: Vec<(&str, Bound<'py, PyAny>)> = location(); + insert.push(("records", records.clone())); + let insert_sql: Bound<'py, PyAny> = + call_with(&renderers.getattr("render_insert_records_sql")?, &insert)?; + execute_sql((execute, connection), insert_sql) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_source_freshness_read_latest_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_source_freshness_create_table_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(read_latest_source_freshness, module)?)?; + module.add_function(wrap_pyfunction!(write_source_freshness_records, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs new file mode 100644 index 0000000000..877d03130d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -0,0 +1,81 @@ +//! Python objects, exceptions and datetimes the source freshness glue shares. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; + +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.source_freshness.exceptions"; +pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; +pub(crate) const FRESHNESS_TYPES: &str = "sqlbuild.compiler.source_freshness.types"; +pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; +pub(crate) const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; +pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +pub(crate) const ADAPTER_EXCEPTIONS: &str = "sqlbuild.adapter.contract.exceptions"; +pub(crate) const LIFECYCLE_MODULE: &str = + "sqlbuild.runtime.observability.classes.operation_lifecycle"; +pub(crate) const OBSERVABILITY_MODELS: &str = "sqlbuild.runtime.observability.models"; +pub(crate) const STATE_TABLE_DDL_MODULE: &str = + "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; +pub(crate) const OBSERVATION_ERROR: &str = "SourceFreshnessObservationError"; +pub(crate) const INPUT_ERROR: &str = "SourceFreshnessInputError"; + +/// `exceptions.(message)` as a raisable error, with an optional `__cause__`. +pub(crate) fn freshness_error( + py: Python<'_>, + (class, message): (&str, &str), + cause: Option, +) -> PyErr { + let error: PyErr = + match imported(py, EXCEPTIONS_MODULE, class).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => return error, + }; + if let Some(cause) = cause { + error.set_cause(py, Some(cause)); + } + error +} + +/// `str(exc)` of an `Exception`, or the error itself for anything else. +pub(crate) fn exception_text(py: Python<'_>, error: &PyErr) -> PyResult> { + if !error.is_instance_of::(py) { + return Ok(None); + } + Ok(Some(error.value(py).str()?.extract()?)) +} + +/// `value` when it is not a `datetime`; otherwise in UTC, a naive value read as UTC. +pub(crate) fn presumed_utc_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if !date_time_typed(value)? { + return Ok(value.clone()); + } + presumed_utc_datetime(value) +} + +/// A `datetime` in UTC, a naive or offset-less value read as UTC. +pub(crate) fn presumed_utc_datetime<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if offset_aware(value)? { + return in_utc(value); + } + let changes: Bound<'py, PyDict> = PyDict::new(value.py()); + changes.set_item("tzinfo", imported(value.py(), DATETIME_MODULE, "UTC")?)?; + value.call_method("replace", (), Some(&changes)) +} + +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// `str(value)`, or `None` for `None`. +pub(crate) fn optional_str<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + Ok(value.str()?.into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 7257d905c0..eb2556c564 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -292,7 +292,7 @@ fn topologically_order_keys<'py>( /// Model names reachable downstream from the given keys, including model roots. #[pyfunction] #[pyo3(signature = (*, start_keys, downstream_deps))] -fn build_downstream_model_name_closure<'py>( +pub(crate) fn build_downstream_model_name_closure<'py>( start_keys: &Bound<'py, PyAny>, downstream_deps: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index c44191bdae..15f735c851 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -43,14 +43,6 @@ pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ( "sqlbuild.compiler.discovery.main._yaml_entry_line", "yaml_entry_line", ); -pub(crate) const FRESHNESS_PLANNING: (&str, &str) = ( - "sqlbuild.compiler.source_freshness.main._planning", - "build_direct_source_freshness_planning_result", -); -pub(crate) const FRESHNESS_PROPAGATION: (&str, &str) = ( - "sqlbuild.compiler.source_freshness.main._propagation", - "build_direct_source_freshness_propagation_result", -); pub(crate) const RUN_SCOPE: (&str, &str) = ("sqlbuild.runtime.observability.main.run_scope", "run_scope"); pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 0a688192c6..703a42e003 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -2,17 +2,18 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::freshness::planning::{ + build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, +}; use crate::bindings::_helpers::planner_phases::objects::{ model_key_names, planner_model, replace, text, tuple, }; -use crate::bindings::_helpers::planner_warehouse::constants::{ - FRESHNESS_PLANNING, FRESHNESS_PROPAGATION, RUN_SCOPE, -}; +use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_warehouse::main::freshness_unknown_message::freshness_unknown_message; use std::collections::BTreeSet; @@ -151,7 +152,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( &imported(py, RUN_SCOPE.0, RUN_SCOPE.1)?.call1((project.getattr("run_id")?,))?, |_| { call_with( - &imported(py, FRESHNESS_PLANNING.0, FRESHNESS_PLANNING.1)?, + wrap_pyfunction!(build_direct_source_freshness_planning_result, py)?.as_any(), &[ ("adapter", adapter.clone()), ("connection", connection.clone()), @@ -177,7 +178,7 @@ pub(crate) fn build_planner_source_freshness_result<'py>( }, )?; let propagation = call_with( - &imported(py, FRESHNESS_PROPAGATION.0, FRESHNESS_PROPAGATION.1)?, + wrap_pyfunction!(build_direct_source_freshness_propagation_result, py)?.as_any(), &[ ("source_freshness", result.clone()), ("scope", scope.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index a2377ff690..f1db12a90e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -49,7 +49,7 @@ fn python_error(py: Python<'_>, error: FreshnessError) -> PyErr { /// Hash the stable source freshness identity used as a graph input. #[pyfunction(name = "source_freshness_data_version_hash")] #[pyo3(signature = (*, source_name, strategy, value_kind, data_version))] -fn version_hash( +pub(crate) fn version_hash( source_name: &str, strategy: &str, value_kind: &str, @@ -66,7 +66,7 @@ fn observation_error(py: Python<'_>, message: &str) -> PyErr { /// A source freshness value normalized for stable state storage. #[pyfunction(name = "normalize_source_freshness_data_version")] #[pyo3(signature = (*, value, value_kind))] -fn normalize<'py>( +pub(crate) fn normalize<'py>( value: &Bound<'py, PyAny>, value_kind: &Bound<'py, PyAny>, ) -> PyResult> { @@ -163,7 +163,7 @@ fn comparable(record: &Bound<'_, PyAny>) -> PyResult { /// Whether two source freshness records are equivalent for skip decisions. #[pyfunction(name = "source_freshness_records_equivalent")] #[pyo3(signature = (*, previous_record, current_record, lag_tolerance = None))] -fn equivalent( +pub(crate) fn equivalent( previous_record: &Bound<'_, PyAny>, current_record: &Bound<'_, PyAny>, lag_tolerance: Option<&str>, @@ -197,7 +197,7 @@ fn aware_utc(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult( +pub(crate) fn age_policy<'py>( policy: &Bound<'py, PyAny>, data_version: &Bound<'py, PyAny>, observed_at: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 280fa1be44..19e375d039 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -28,3 +28,17 @@ pub(crate) const MARCH_BASED_MONTH_DAYS: i128 = 153; pub(crate) const MONTH_LENGTH_DIVISOR: i128 = 5; pub(crate) const MONTHS_FROM_MARCH: i128 = 9; pub(crate) const FEBRUARY: u32 = 2; +pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +/// Columns a source freshness state table must have, in Python's declaration order. +pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ + "source_name", + "run_id", + "strategy", + "value_kind", + "data_version_hash", + "observed_at", +]; +pub(crate) const STATE_SELECT_COLUMNS: &str = "source_name, target_database, target_schema, \ +target_name, run_id, strategy, value_kind, data_version, data_version_hash, observed_at"; +pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; +pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/configuration_error.rs b/crates/sqlbuild-state/src/freshness/main/configuration_error.rs new file mode 100644 index 0000000000..8d25018f4a --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/configuration_error.rs @@ -0,0 +1,11 @@ +//! `configuration_error`: whether an observation failure is a source configuration error. + +use crate::freshness::constants::{ + INCOMPLETE_CONFIGURATION_FRAGMENT, PHYSICAL_TABLE_SOURCE_FRAGMENT, +}; + +/// Configuration errors fail planning; every other observation failure marks the source unknown. +pub fn configuration_error(message: &str) -> bool { + message.contains(PHYSICAL_TABLE_SOURCE_FRAGMENT) + || message.contains(INCOMPLETE_CONFIGURATION_FRAGMENT) +} diff --git a/crates/sqlbuild-state/src/freshness/main/mod.rs b/crates/sqlbuild-state/src/freshness/main/mod.rs index 6e7cddd828..9ec47ed663 100644 --- a/crates/sqlbuild-state/src/freshness/main/mod.rs +++ b/crates/sqlbuild-state/src/freshness/main/mod.rs @@ -1,4 +1,8 @@ pub mod age_status; +pub mod configuration_error; pub mod data_version_hash; pub mod normalized_data_version; +pub mod read_latest_sql; pub mod records_equivalent; +pub mod relevant_sources_sql; +pub mod single_value_problem; diff --git a/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs new file mode 100644 index 0000000000..9dc677915d --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs @@ -0,0 +1,13 @@ +//! `read_latest_sql`: the windowed SELECT of the latest state row per source identity. + +use crate::freshness::constants::STATE_SELECT_COLUMNS; + +/// Latest row per source, target database, schema and name, newest observation and run first. +pub fn read_latest_sql(qualified_name: &str) -> String { + format!( + "SELECT {STATE_SELECT_COLUMNS} FROM (SELECT {STATE_SELECT_COLUMNS}, ROW_NUMBER() OVER (\ +PARTITION BY source_name, target_database, target_schema, target_name \ +ORDER BY observed_at DESC, run_id DESC) AS __sqlbuild_latest_rank FROM {qualified_name}) \ +AS __sqlbuild_latest_source_freshness WHERE __sqlbuild_latest_rank = 1" + ) +} diff --git a/crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs b/crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs new file mode 100644 index 0000000000..99b2be1e49 --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/relevant_sources_sql.rs @@ -0,0 +1,13 @@ +//! `relevant_sources_sql`: a latest-state read narrowed to the named sources. + +/// `read_sql` filtered to `source_names`, each quoted as a SQL string literal. +pub fn relevant_sources_sql(read_sql: &str, source_names: &[String]) -> String { + let literals: Vec = source_names + .iter() + .map(|name| format!("'{}'", name.replace('\'', "''"))) + .collect(); + format!( + "SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE source_name IN ({})", + literals.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/freshness/main/single_value_problem.rs b/crates/sqlbuild-state/src/freshness/main/single_value_problem.rs new file mode 100644 index 0000000000..7f972a450b --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/main/single_value_problem.rs @@ -0,0 +1,20 @@ +//! `single_value_problem`: why a freshness query result is not one non-null value. + +use crate::freshness::models::QueryShape; + +/// The observation error message for a result that is not exactly one non-null value. +pub fn single_value_problem(source_name: &str, shape: QueryShape) -> Option { + if shape.columns != 1 { + return Some(format!( + "source '{source_name}' freshness query must return exactly one column" + )); + } + if shape.rows != 1 { + return Some(format!( + "source '{source_name}' freshness query must return exactly one row" + )); + } + shape + .first_value_null + .then(|| format!("source '{source_name}' freshness data_version cannot be null")) +} diff --git a/crates/sqlbuild-state/src/freshness/models.rs b/crates/sqlbuild-state/src/freshness/models.rs index 99ca646dfe..dbe1341a71 100644 --- a/crates/sqlbuild-state/src/freshness/models.rs +++ b/crates/sqlbuild-state/src/freshness/models.rs @@ -30,3 +30,11 @@ pub enum AgeStatus { Warn, Error, } + +/// The shape of a single-value freshness query result. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct QueryShape { + pub columns: usize, + pub rows: usize, + pub first_value_null: bool, +} diff --git a/crates/sqlbuild-state/src/freshness/tests.rs b/crates/sqlbuild-state/src/freshness/tests.rs index ead02b3221..8ac14ee7fc 100644 --- a/crates/sqlbuild-state/src/freshness/tests.rs +++ b/crates/sqlbuild-state/src/freshness/tests.rs @@ -2,5 +2,7 @@ mod equivalence; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_observation.rs"] +mod observation; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-state/src/freshness/tests/test_observation.rs b/crates/sqlbuild-state/src/freshness/tests/test_observation.rs new file mode 100644 index 0000000000..77274157bb --- /dev/null +++ b/crates/sqlbuild-state/src/freshness/tests/test_observation.rs @@ -0,0 +1,143 @@ +use crate::freshness::main::configuration_error::configuration_error; +use crate::freshness::main::read_latest_sql::read_latest_sql; +use crate::freshness::main::relevant_sources_sql::relevant_sources_sql; +use crate::freshness::main::single_value_problem::single_value_problem; +use crate::freshness::models::QueryShape; +use crate::freshness::tests::test_types::{ + ConfigurationErrorTestCase, ReadLatestTestCase, RelevantSourcesTestCase, SingleValueTestCase, +}; + +#[test] +fn given_qualified_name_when_reading_latest_then_ranks_newest_row_per_identity() { + let test_cases = [ReadLatestTestCase { + description: "one schema-qualified state table", + qualified_name: "main.state", + expected_sql: "SELECT source_name, target_database, target_schema, target_name, run_id, \ +strategy, value_kind, data_version, data_version_hash, observed_at FROM (SELECT source_name, \ +target_database, target_schema, target_name, run_id, strategy, value_kind, data_version, \ +data_version_hash, observed_at, ROW_NUMBER() OVER (PARTITION BY source_name, target_database, \ +target_schema, target_name ORDER BY observed_at DESC, run_id DESC) AS __sqlbuild_latest_rank \ +FROM main.state) AS __sqlbuild_latest_source_freshness WHERE __sqlbuild_latest_rank = 1", + }]; + + for test_case in test_cases { + assert_eq!( + read_latest_sql(test_case.qualified_name), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_source_names_when_narrowing_latest_read_then_quotes_each_literal() { + let test_cases = [ + RelevantSourcesTestCase { + description: "one plain name", + source_names: &["orders"], + expected_sql: "SELECT * FROM (R) AS __sqlbuild_relevant WHERE source_name IN ('orders')", + }, + RelevantSourcesTestCase { + description: "quotes are doubled", + source_names: &["a", "o'b"], + expected_sql: "SELECT * FROM (R) AS __sqlbuild_relevant WHERE source_name IN ('a', 'o''b')", + }, + ]; + + for test_case in test_cases { + let names: Vec = test_case + .source_names + .iter() + .map(|name| (*name).to_owned()) + .collect(); + + assert_eq!( + relevant_sources_sql("R", &names), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_query_shape_when_checking_single_value_then_reports_first_problem() { + let test_cases = [ + SingleValueTestCase { + description: "two columns win over two rows", + shape: QueryShape { + columns: 2, + rows: 2, + first_value_null: false, + }, + expected_problem: Some("source 's' freshness query must return exactly one column"), + }, + SingleValueTestCase { + description: "no rows", + shape: QueryShape { + columns: 1, + rows: 0, + first_value_null: false, + }, + expected_problem: Some("source 's' freshness query must return exactly one row"), + }, + SingleValueTestCase { + description: "a null value", + shape: QueryShape { + columns: 1, + rows: 1, + first_value_null: true, + }, + expected_problem: Some("source 's' freshness data_version cannot be null"), + }, + SingleValueTestCase { + description: "one value", + shape: QueryShape { + columns: 1, + rows: 1, + first_value_null: false, + }, + expected_problem: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + single_value_problem("s", test_case.shape).as_deref(), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_observation_message_when_classifying_then_configuration_errors_are_detected() { + let test_cases = [ + ConfigurationErrorTestCase { + description: "physical table requirement", + message: "source 's' adapter freshness requires a physical table source", + expected_configuration_error: true, + }, + ConfigurationErrorTestCase { + description: "incomplete configuration", + message: "source 's' has incomplete sql freshness configuration", + expected_configuration_error: true, + }, + ConfigurationErrorTestCase { + description: "query failure", + message: "source 's' freshness query failed: boom", + expected_configuration_error: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + configuration_error(test_case.message), + test_case.expected_configuration_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/freshness/tests/test_types.rs b/crates/sqlbuild-state/src/freshness/tests/test_types.rs index 0632d8083a..ec118da065 100644 --- a/crates/sqlbuild-state/src/freshness/tests/test_types.rs +++ b/crates/sqlbuild-state/src/freshness/tests/test_types.rs @@ -10,3 +10,27 @@ pub(crate) struct EquivalenceTestCase { pub(crate) lag_tolerance: Option<&'static str>, pub(crate) expected_equivalent: Result, } + +pub(crate) struct RelevantSourcesTestCase { + pub(crate) description: &'static str, + pub(crate) source_names: &'static [&'static str], + pub(crate) expected_sql: &'static str, +} + +pub(crate) struct SingleValueTestCase { + pub(crate) description: &'static str, + pub(crate) shape: crate::freshness::models::QueryShape, + pub(crate) expected_problem: Option<&'static str>, +} + +pub(crate) struct ConfigurationErrorTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_configuration_error: bool, +} + +pub(crate) struct ReadLatestTestCase { + pub(crate) description: &'static str, + pub(crate) qualified_name: &'static str, + pub(crate) expected_sql: &'static str, +} diff --git a/fensu.toml b/fensu.toml index 4e1d503db5..9601450583 100644 --- a/fensu.toml +++ b/fensu.toml @@ -63,6 +63,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/planner/__init__.py" reason = "Native bindings own planning, plan entries, plan output and command plans; this package only declares the plan models, types, constants, errors and the fixture inference cache the native planner reads." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/source_freshness/__init__.py" +reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types, constants and errors they build and raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 2287850613..bdf5fc8a58 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 17507 +remaining_lines = 16308 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1bcae18060..beb3e065ab 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -342,9 +342,14 @@ from sqlbuild.compiler.scopes.models import ( ) from sqlbuild.compiler.scopes.types import DeclarationKind, JsonValue, ScopeQueryTarget from sqlbuild.compiler.source_freshness.models import ( + AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, + DirectSourceFreshnessPropagationResult, SourceFreshnessIdentity, + SourceFreshnessObservation, SourceFreshnessRecord, + SourceFreshnessRenderers, + SourceFreshnessSet, ) from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, @@ -4802,6 +4807,66 @@ def evaluate_source_freshness_age_policy( data_version: object, observed_at: datetime, ) -> SourceFreshnessAgeStatus | None: ... +def require_aware_utc_datetime(*, value: datetime, field_name: str) -> datetime: ... +def build_source_freshness_read_latest_sql( + *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_source_freshness_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def read_latest_source_freshness( + *, + connection: object, + execute: AdapterExecute[Any, Any], + table_exists: bool, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_read_latest_sql: Callable[..., str], + source_names: tuple[str, ...] | None = None, +) -> SourceFreshnessSet: ... +def write_source_freshness_records( + *, + connection: object, + execute: AdapterExecute[Any, Any], + database: str | None, + schema: str, + records: tuple[SourceFreshnessRecord, ...], + renderers: SourceFreshnessRenderers, +) -> None: ... +def observe_configured_source_freshness( + *, adapter: StrictAdapter, connection: object, source: SourceEntry, observed_at: datetime +) -> SourceFreshnessObservation: ... +def observe_adapter_sources_freshness( + *, + adapter: StrictAdapter, + connection: object, + sources: tuple[SourceEntry, ...], + observed_at: datetime, +) -> AdapterSourceFreshnessBatch: ... +def source_freshness_record_from_observation( + *, observation: SourceFreshnessObservation, source: SourceEntry, run_id: str +) -> SourceFreshnessRecord: ... +def build_direct_source_freshness_planning_result( + *, + adapter: StrictAdapter, + connection: object, + sources: tuple[SourceEntry, ...], + state_database: str | None, + state_schemas: tuple[str, ...], + observed_at: datetime, + run_id: str, + render_qualified_name: Callable[..., str | None], + state_table_exists_by_schema: Mapping[str, bool], + filter_previous_to_sources: bool = False, +) -> DirectSourceFreshnessPlanningResult: ... +def build_direct_source_freshness_propagation_result( + *, source_freshness: DirectSourceFreshnessPlanningResult, scope: PlannerScope +) -> DirectSourceFreshnessPropagationResult: ... def redact_diagnostic_text(value: str) -> str: ... def redact_diagnostic_value(*, name: str, value: object) -> object: ... def sql_rule_lint_key(*, config: LintConfig) -> str: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 9578d2fc0b..cb33a480c7 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2166,11 +2166,9 @@ def render_read_latest_source_freshness_sql( ) -> str: """Render SQL that reads latest source freshness rows per identity.""" - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -2190,9 +2188,7 @@ def render_create_fingerprint_index_sqls( def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: """Render DDL that creates the source freshness table when it is missing.""" - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index cf6a389d47..ec8ecff6f5 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -807,20 +807,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 8184bbba30..4601cc88e8 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -261,20 +261,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 478eaf5c33..a15d065224 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -357,20 +357,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 8eb92916f0..5cd919f515 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1593,9 +1593,7 @@ def render_prune_fingerprint_history_sql( ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, @@ -1770,11 +1768,9 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 826a015573..6938184a72 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -533,20 +533,16 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return self._with_state_table_retention( create_sql=build_source_freshness_create_table_sql( diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 2dcddfb1e7..ccfda8d0b2 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -448,9 +448,7 @@ def render_prune_fingerprint_history_sql( ) def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, - ) + from sqlbuild._native import build_source_freshness_create_table_sql return build_source_freshness_create_table_sql( database=database, @@ -720,11 +718,9 @@ def render_read_latest_source_freshness_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.source_freshness.main.read_latest_sql import ( - build_read_latest_sql, - ) + from sqlbuild._native import build_source_freshness_read_latest_sql - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 90fd1430ad..354e97f2ce 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -5,9 +5,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import write_source_freshness_records from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessIdentity, diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/__init__.py b/src/sqlbuild/compiler/source_freshness/_helpers/__init__.py deleted file mode 100644 index cf805ebda2..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Shared source freshness helper implementations.""" diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/datetime.py b/src/sqlbuild/compiler/source_freshness/_helpers/datetime.py deleted file mode 100644 index 6ad95d119f..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/datetime.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Source freshness datetime invariant helpers.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError - - -def require_aware_utc_datetime(*, value: datetime, field_name: str) -> datetime: - """Require an aware freshness datetime and normalize it to UTC.""" - - if value.tzinfo is None or value.utcoffset() is None: - raise SourceFreshnessObservationError( - f"source freshness {field_name} must be timezone-aware" - ) - return value.astimezone(UTC) - - -def normalize_presumed_utc_datetime(*, value: datetime) -> datetime: - """Treat a naive adapter or legacy-state datetime as UTC and normalize aware values.""" - - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py b/src/sqlbuild/compiler/source_freshness/_helpers/observation.py deleted file mode 100644 index 3d309d31b7..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py +++ /dev/null @@ -1,323 +0,0 @@ -"""Shared source freshness observation helper implementation.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild._native import render_source_relation -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.models import ( - QueryResult, - TableFreshnessMetadata, - TableFreshnessRequest, -) -from sqlbuild.adapter.contract.types import TableFreshnessStatus -from sqlbuild.compiler.source_freshness._helpers.datetime import normalize_presumed_utc_datetime -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, - SourceFreshnessObservation, - SourceFreshnessUnknown, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessUnknownReason -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.runtime.observability.models import OperationAttributes -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -def observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - """Observe one source freshness config and return a comparable data version.""" - - config: SourceFreshnessConfig | None = source.freshness - strategy: str = config.strategy.value if config is not None else "sql" - operation_name: str = ( - "source_freshness_metadata_observation" - if config is not None and config.strategy == SourceFreshnessStrategy.ADAPTER - else "source_freshness_query_observation" - ) - with OperationLifecycle( - operation_kind="freshness", - operation_name=operation_name, - attributes=OperationAttributes(phase="observe", strategy=strategy, target_kind="source"), - ): - return _observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - - -def _observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - config: SourceFreshnessConfig | None = source.freshness - if config is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' does not define freshness configuration" - ) - if config.strategy == SourceFreshnessStrategy.ADAPTER: - return _observe_adapter_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - - if config.strategy == SourceFreshnessStrategy.COLUMN: - if config.column is None or config.value_kind is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' has incomplete column freshness configuration" - ) - where_sql: str = f" WHERE {config.filter}" if config.filter is not None else "" - sql: str = adapter.render_source_freshness_max_query( - column=config.column, - source_relation=render_source_relation(entry=source, adapter=adapter), - source_is_subquery=source.expression is not None, - where_sql=where_sql, - ) - data_version: object = _normalize_adapter_datetime( - _query_single_data_version( - adapter=adapter, - connection=connection, - source_name=source.name, - sql=sql, - ) - ) - return SourceFreshnessObservation( - source_name=source.name, - strategy=config.strategy, - data_version=data_version, - value_kind=config.value_kind, - observed_at=observed_at, - ) - - if config.query is None or config.value_kind is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' has incomplete sql freshness configuration" - ) - data_version = _normalize_adapter_datetime( - _query_single_data_version( - adapter=adapter, - connection=connection, - source_name=source.name, - sql=config.query, - ) - ) - return SourceFreshnessObservation( - source_name=source.name, - strategy=config.strategy, - data_version=data_version, - value_kind=config.value_kind, - observed_at=observed_at, - ) - - -def _observe_adapter_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - if not adapter.supports_table_freshness_metadata(): - raise SourceFreshnessObservationError( - f"adapter '{adapter.adapter_name}' does not support table freshness metadata" - ) - if source.expression is not None or source.table is None: - raise SourceFreshnessObservationError( - f"source '{source.name}' adapter freshness requires a physical table source" - ) - metadata: TableFreshnessMetadata = adapter.get_table_freshness_metadata( - connection=connection, - database=source.database, - schema=source.schema, - name=source.table, - ) - source_unknown: SourceFreshnessUnknown | None = _adapter_metadata_unknown( - source_name=source.name, metadata=metadata - ) - if source_unknown is not None: - raise SourceFreshnessObservationError(source_unknown.message) - return SourceFreshnessObservation( - source_name=source.name, - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=_normalize_adapter_datetime(metadata.data_version), - value_kind=SourceFreshnessValueKind(metadata.value_kind), - observed_at=( - normalize_presumed_utc_datetime(value=metadata.observed_at) - if metadata.observed_at is not None - else observed_at - ), - ) - - -def observe_adapter_sources_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - """Observe adapter metadata freshness per source, batching the warehouse lookups.""" - - if not sources: - return AdapterSourceFreshnessBatch() - with OperationLifecycle( - operation_kind="freshness", - operation_name="source_freshness_metadata_observation", - metadata={"item_count": len(sources)}, - attributes=OperationAttributes(phase="observe", strategy="adapter", target_kind="source"), - ) as lifecycle: - batch: AdapterSourceFreshnessBatch = _observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=sources, - observed_at=observed_at, - ) - lifecycle.completed( - metadata={"item_count": len(sources), "unknown_count": len(batch.unknown)} - ) - return batch - - -def _observe_adapter_sources_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - unknown: dict[str, SourceFreshnessUnknown] = {} - requests_by_source_name: dict[str, TableFreshnessRequest] = {} - source: SourceEntry - for source in sources: - unsupported_message: str | None = _adapter_freshness_unsupported_message( - adapter=adapter, source=source - ) - if unsupported_message is not None: - unknown[source.name] = SourceFreshnessUnknown( - source_name=source.name, - reason=SourceFreshnessUnknownReason.UNAVAILABLE, - message=unsupported_message, - ) - continue - requests_by_source_name[source.name] = TableFreshnessRequest( - database=source.database, - schema=source.schema, - name=str(source.table), - ) - if not requests_by_source_name: - return AdapterSourceFreshnessBatch(unknown=unknown) - - metadata_by_request: dict[TableFreshnessRequest, TableFreshnessMetadata] = ( - adapter.get_tables_freshness_metadata( - connection=connection, - requests=tuple(dict.fromkeys(requests_by_source_name.values())), - ) - ) - observations: dict[str, SourceFreshnessObservation] = {} - source_name: str - request: TableFreshnessRequest - for source_name, request in requests_by_source_name.items(): - metadata: TableFreshnessMetadata = metadata_by_request.get( - request, - TableFreshnessMetadata.unavailable( - message=f"source '{source_name}' freshness metadata was not returned" - ), - ) - source_unknown: SourceFreshnessUnknown | None = _adapter_metadata_unknown( - source_name=source_name, metadata=metadata - ) - if source_unknown is not None: - unknown[source_name] = source_unknown - continue - observations[source_name] = SourceFreshnessObservation( - source_name=source_name, - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=_normalize_adapter_datetime(metadata.data_version), - value_kind=SourceFreshnessValueKind(metadata.value_kind), - observed_at=( - normalize_presumed_utc_datetime(value=metadata.observed_at) - if metadata.observed_at is not None - else observed_at - ), - ) - return AdapterSourceFreshnessBatch(observations=observations, unknown=unknown) - - -def _adapter_freshness_unsupported_message( - *, adapter: StrictAdapter, source: SourceEntry -) -> str | None: - if not adapter.supports_table_freshness_metadata(): - return f"adapter '{adapter.adapter_name}' does not support table freshness metadata" - if source.expression is not None or source.table is None: - return f"source '{source.name}' adapter freshness requires a physical table source" - return None - - -def _adapter_metadata_unknown( - *, source_name: str, metadata: TableFreshnessMetadata -) -> SourceFreshnessUnknown | None: - if metadata.status == TableFreshnessStatus.UNAVAILABLE: - return SourceFreshnessUnknown( - source_name=source_name, - reason=SourceFreshnessUnknownReason.UNAVAILABLE, - message=metadata.message or f"source '{source_name}' freshness metadata unavailable", - ) - if metadata.data_version is None: - return SourceFreshnessUnknown( - source_name=source_name, - reason=SourceFreshnessUnknownReason.UNAVAILABLE, - message=f"source '{source_name}' freshness data_version is null", - ) - return None - - -def _query_single_data_version( - *, - adapter: StrictAdapter, - connection: Any, - source_name: str, - sql: str, -) -> object: - try: - result: QueryResult = adapter.query(connection=connection, sql=sql, limit=None) - except Exception as exc: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness query failed: {exc}" - ) from exc - if len(result.columns) != 1: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness query must return exactly one column" - ) - if len(result.rows) != 1: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness query must return exactly one row" - ) - data_version: object = result.rows[0][0] - if data_version is None: - raise SourceFreshnessObservationError( - f"source '{source_name}' freshness data_version cannot be null" - ) - return data_version - - -def _normalize_adapter_datetime(value: object) -> object: - """Normalize driver datetimes, whose naive values are documented as presumed UTC.""" - - if not isinstance(value, datetime): - return value - return normalize_presumed_utc_datetime(value=value) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py b/src/sqlbuild/compiler/source_freshness/_helpers/planning.py deleted file mode 100644 index 3a29acfe20..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/planning.py +++ /dev/null @@ -1,337 +0,0 @@ -"""Direct source freshness planning observation and comparison helpers.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild._native import ( - evaluate_source_freshness_age_policy, - normalize_source_freshness_data_version, - source_freshness_data_version_hash, - source_freshness_records_equivalent, -) -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.source_freshness.constants import ( - INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT, - PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT, -) -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.adapter_observation import ( - observe_adapter_sources_freshness, -) -from sqlbuild.compiler.source_freshness.main.observation import ( - observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, - DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, - SourceFreshnessObservation, - SourceFreshnessRecord, - SourceFreshnessSet, - SourceFreshnessUnknown, -) -from sqlbuild.compiler.source_freshness.types import ( - SourceFreshnessAgeStatus, - SourceFreshnessUnknownReason, -) -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy - - -def build_direct_source_freshness_planning_result( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - state_database: str | None, - state_schemas: tuple[str, ...], - observed_at: datetime, - run_id: str, - render_qualified_name: Callable[..., str | None], - state_table_exists_by_schema: dict[str, bool], - filter_previous_to_sources: bool = False, -) -> DirectSourceFreshnessPlanningResult: - previous_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = ( - _read_previous_records( - adapter=adapter, - connection=connection, - state_database=state_database, - state_schemas=state_schemas, - render_qualified_name=render_qualified_name, - state_table_exists_by_schema=state_table_exists_by_schema, - source_names=( - tuple(sorted(source.name for source in sources)) - if filter_previous_to_sources - else None - ), - ) - ) - observed_records: list[SourceFreshnessRecord] = [] - unconfigured_sources: dict[str, SourceFreshnessUnknown] = {} - changed_identities: set[SourceFreshnessIdentity] = set() - unchanged_identities: set[SourceFreshnessIdentity] = set() - age_statuses: dict[SourceFreshnessIdentity, SourceFreshnessAgeStatus] = {} - observation_sources_by_name: dict[str, SourceEntry] = {} - adapter_observation_sources: list[SourceEntry] = [] - - source: SourceEntry - for source in sources: - if source.managed: - continue - observation_source: SourceEntry | None = _source_for_observation( - adapter=adapter, - source=source, - ) - if observation_source is None: - unconfigured_sources[source.name] = SourceFreshnessUnknown( - source_name=source.name, - reason=SourceFreshnessUnknownReason.NO_FRESHNESS_CONFIG, - message="no freshness config and adapter metadata unavailable", - ) - continue - observation_sources_by_name[source.name] = observation_source - if observation_source.freshness is not None and ( - observation_source.freshness.strategy == SourceFreshnessStrategy.ADAPTER - ): - adapter_observation_sources.append(observation_source) - - adapter_batch: AdapterSourceFreshnessBatch = _observe_adapter_batch( - adapter=adapter, - connection=connection, - sources=tuple(adapter_observation_sources), - observed_at=observed_at, - ) - unknown_sources: dict[str, SourceFreshnessUnknown] = { - **unconfigured_sources, - **adapter_batch.unknown, - } - - for source_name, observation_source in observation_sources_by_name.items(): - if observation_source.freshness is not None and ( - observation_source.freshness.strategy == SourceFreshnessStrategy.ADAPTER - ): - observation: SourceFreshnessObservation | None = adapter_batch.observations.get( - source_name - ) - if observation is None: - continue - else: - try: - observation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=observation_source, - observed_at=observed_at, - ) - except AdapterUserError as exc: - unknown_sources[source_name] = _error_unknown( - source_name=source_name, message=exc.message - ) - continue - except SourceFreshnessObservationError as exc: - if _source_freshness_error_is_configuration_error(error=exc): - raise - unknown_sources[source_name] = _error_unknown( - source_name=source_name, message=str(exc) - ) - continue - observed_record: SourceFreshnessRecord = source_freshness_record_from_observation( - observation=observation, - source=observation_source, - run_id=run_id, - ) - observed_records.append(observed_record) - age_status: SourceFreshnessAgeStatus | None = evaluate_source_freshness_age_policy( - policy=observation_source.freshness.age_policy - if observation_source.freshness is not None - else None, - data_version=observation.data_version, - observed_at=observation.observed_at, - ) - if age_status is not None: - age_statuses[observed_record.identity] = age_status - previous_record: SourceFreshnessRecord | None = previous_records_by_identity.get( - observed_record.identity - ) - if previous_record is None: - changed_identities.add(observed_record.identity) - continue - if source_freshness_records_equivalent( - previous_record=previous_record, - current_record=observed_record, - lag_tolerance=observation_source.freshness.lag_tolerance - if observation_source.freshness is not None - else None, - ): - unchanged_identities.add(observed_record.identity) - else: - changed_identities.add(observed_record.identity) - - return DirectSourceFreshnessPlanningResult( - observed_records=tuple(sorted(observed_records, key=lambda record: str(record.identity))), - previous_records=tuple( - sorted(previous_records_by_identity.values(), key=lambda record: str(record.identity)) - ), - changed_identities=frozenset(changed_identities), - unchanged_identities=frozenset(unchanged_identities), - unknown_source_names=tuple(sorted(unknown_sources)), - unknown_sources=dict(sorted(unknown_sources.items())), - age_statuses=age_statuses, - ) - - -def _observe_adapter_batch( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - try: - return observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=sources, - observed_at=observed_at, - ) - except (AdapterUserError, SourceFreshnessObservationError) as exc: - message: str = exc.message if isinstance(exc, AdapterUserError) else str(exc) - return AdapterSourceFreshnessBatch( - unknown={ - source.name: _error_unknown(source_name=source.name, message=message) - for source in sources - } - ) - - -def _error_unknown(*, source_name: str, message: str) -> SourceFreshnessUnknown: - return SourceFreshnessUnknown( - source_name=source_name, - reason=SourceFreshnessUnknownReason.ERROR, - message=message, - ) - - -def _source_freshness_error_is_configuration_error( - *, error: SourceFreshnessObservationError -) -> bool: - message: str = str(error) - return ( - PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT in message - or INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT in message - ) - - -def source_freshness_record_from_observation( - *, observation: SourceFreshnessObservation, source: SourceEntry, run_id: str -) -> SourceFreshnessRecord: - normalized_data_version: str = normalize_source_freshness_data_version( - value=observation.data_version, - value_kind=observation.value_kind, - ) - return SourceFreshnessRecord( - source_name=observation.source_name, - target_database=source.database, - target_schema=source.schema, - target_name=source.table, - run_id=run_id, - strategy=observation.strategy.value, - value_kind=observation.value_kind.value, - data_version=normalized_data_version, - data_version_hash=source_freshness_data_version_hash( - source_name=observation.source_name, - strategy=observation.strategy, - value_kind=observation.value_kind, - data_version=normalized_data_version, - ), - observed_at=observation.observed_at, - ) - - -def _read_previous_records( - *, - adapter: StrictAdapter, - connection: Any, - state_database: str | None, - state_schemas: tuple[str, ...], - render_qualified_name: Callable[..., str | None], - state_table_exists_by_schema: dict[str, bool], - source_names: tuple[str, ...] | None, -) -> dict[SourceFreshnessIdentity, SourceFreshnessRecord]: - previous_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = {} - state_schema: str - for state_schema in state_schemas: - table_exists: bool = state_table_exists_by_schema.get(state_schema, False) - previous_set: SourceFreshnessSet = read_latest_source_freshness( - connection=connection, - execute=adapter.execute, - table_exists=table_exists, - database=state_database, - schema=state_schema, - render_qualified_name=render_qualified_name, - render_read_latest_sql=adapter.render_read_latest_source_freshness_sql, - source_names=source_names, - ) - previous_records_by_identity = _merged_latest_previous_records( - previous_records_by_identity=previous_records_by_identity, - candidate_records=previous_set.records, - ) - return previous_records_by_identity - - -def _merged_latest_previous_records( - *, - previous_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord], - candidate_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord], -) -> dict[SourceFreshnessIdentity, SourceFreshnessRecord]: - merged: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = dict( - previous_records_by_identity - ) - identity: SourceFreshnessIdentity - candidate_record: SourceFreshnessRecord - for identity, candidate_record in candidate_records.items(): - previous_record: SourceFreshnessRecord | None = merged.get(identity) - if previous_record is None or candidate_record.observed_at > previous_record.observed_at: - merged[identity] = candidate_record - return merged - - -def _source_for_observation(*, adapter: StrictAdapter, source: SourceEntry) -> SourceEntry | None: - if source.freshness is not None: - return source - if ( - source.expression is None - and source.table is not None - and adapter.supports_table_freshness_metadata() - ): - return _source_with_adapter_freshness(source) - return None - - -def _source_with_adapter_freshness(source: SourceEntry) -> SourceEntry: - return SourceEntry( - name=source.name, - database=source.database, - schema=source.schema, - table=source.table, - loader=source.loader, - managed=source.managed, - integration_loader=source.integration_loader, - freshness=SourceFreshnessConfig(strategy=SourceFreshnessStrategy.ADAPTER), - write_strategy=source.write_strategy, - load_batch_size=source.load_batch_size, - cursor_column=source.cursor_column, - unique_key=source.unique_key, - expression=source.expression, - description=source.description, - type_enforcement=source.type_enforcement, - contract=source.contract, - meta=source.meta, - columns=source.columns, - audits=source.audits, - ) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py deleted file mode 100644 index 9368d38745..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Direct source freshness downstream propagation helpers.""" - -from __future__ import annotations - -from sqlbuild._native import build_downstream_model_name_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, - SourceFreshnessIdentity, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus - - -def build_direct_source_freshness_propagation_result( - *, - source_freshness: DirectSourceFreshnessPlanningResult, - scope: PlannerScope, -) -> DirectSourceFreshnessPropagationResult: - """Map changed/unknown source freshness roots to downstream model names.""" - - changed_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = {} - unknown_source_model_names: dict[str, frozenset[str]] = {} - error_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = {} - stale_model_names: set[str] = set() - blocked_model_names: set[str] = set() - - identity: SourceFreshnessIdentity - for identity in source_freshness.changed_identities: - model_names: frozenset[str] = _downstream_model_names( - source_name=identity.source_name, - scope=scope, - ) - changed_source_model_names[identity] = model_names - stale_model_names.update(model_names) - - source_name: str - for source_name in source_freshness.unknown_source_names: - model_names = _downstream_model_names(source_name=source_name, scope=scope) - unknown_source_model_names[source_name] = model_names - stale_model_names.update(model_names) - - for identity, age_status in source_freshness.age_statuses.items(): - if age_status != SourceFreshnessAgeStatus.ERROR: - continue - model_names = _downstream_model_names(source_name=identity.source_name, scope=scope) - error_source_model_names[identity] = model_names - blocked_model_names.update(model_names) - - return DirectSourceFreshnessPropagationResult( - changed_source_model_names=changed_source_model_names, - unknown_source_model_names=unknown_source_model_names, - error_source_model_names=error_source_model_names, - stale_model_names=frozenset(stale_model_names), - blocked_model_names=frozenset(blocked_model_names), - ) - - -def _downstream_model_names(*, source_name: str, scope: PlannerScope) -> frozenset[str]: - source_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SOURCE, - name=source_name, - ) - return build_downstream_model_name_closure( - start_keys=(source_key,), - downstream_deps=scope.downstream_deps, - ) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/sql.py b/src/sqlbuild/compiler/source_freshness/_helpers/sql.py deleted file mode 100644 index e907ae02ca..0000000000 --- a/src/sqlbuild/compiler/source_freshness/_helpers/sql.py +++ /dev/null @@ -1,119 +0,0 @@ -"""SQL generation helpers for direct source freshness storage.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) -from sqlbuild.compiler.source_freshness.constants import ( - COLUMN_DATA_VERSION, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, - COLUMN_RUN_ID, - COLUMN_SOURCE_NAME, - COLUMN_STRATEGY, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, - COLUMN_TARGET_SCHEMA, - COLUMN_VALUE_KIND, - SOURCE_FRESHNESS_COLUMN_TYPES, - SOURCE_FRESHNESS_COLUMNS, - SOURCE_FRESHNESS_TABLE_NAME, -) -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError - -_REQUIRED_SOURCE_FRESHNESS_COLUMNS: frozenset[str] = frozenset( - { - COLUMN_SOURCE_NAME, - COLUMN_RUN_ID, - COLUMN_STRATEGY, - COLUMN_VALUE_KIND, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, - } -) - - -def build_qualified_table_name( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], -) -> str: - """Build the fully qualified source freshness table name for a target schema.""" - - qualified_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=SOURCE_FRESHNESS_TABLE_NAME, - ) - if qualified_name is None: - raise SourceFreshnessInputError("source freshness table requires a target schema") - return qualified_name - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build a CREATE TABLE IF NOT EXISTS statement for source freshness state.""" - - return render_state_table_create_sql( - qualified_name=build_qualified_table_name( - database=database, schema=schema, render_qualified_name=render_qualified_name - ), - columns=SOURCE_FRESHNESS_COLUMNS, - column_types=SOURCE_FRESHNESS_COLUMN_TYPES, - required_columns=_REQUIRED_SOURCE_FRESHNESS_COLUMNS, - render_framework_type=render_framework_type, - ) - - -def build_read_latest_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - """Build a windowed SELECT for the latest source freshness row per identity.""" - - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - selected_columns: str = _source_freshness_select_columns() - return ( - f"SELECT {selected_columns} " - f"FROM (" - f"SELECT {selected_columns}, " - f"ROW_NUMBER() OVER (" - f"PARTITION BY " - f"{COLUMN_SOURCE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME} " - f"ORDER BY {COLUMN_OBSERVED_AT} DESC, {COLUMN_RUN_ID} DESC" - f") AS __sqlbuild_latest_rank " - f"FROM {qualified_name}" - f") AS __sqlbuild_latest_source_freshness " - f"WHERE __sqlbuild_latest_rank = 1" - ) - - -def _source_freshness_select_columns() -> str: - return ( - f"{COLUMN_SOURCE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_STRATEGY}, " - f"{COLUMN_VALUE_KIND}, " - f"{COLUMN_DATA_VERSION}, " - f"{COLUMN_DATA_VERSION_HASH}, " - f"{COLUMN_OBSERVED_AT}" - ) diff --git a/src/sqlbuild/compiler/source_freshness/constants.py b/src/sqlbuild/compiler/source_freshness/constants.py index dda3206687..0cd633a952 100644 --- a/src/sqlbuild/compiler/source_freshness/constants.py +++ b/src/sqlbuild/compiler/source_freshness/constants.py @@ -5,8 +5,6 @@ from sqlbuild.sql_values.types import StateSqlValueType SOURCE_FRESHNESS_TABLE_NAME: str = "_sqlbuild_source_freshness" -PHYSICAL_TABLE_SOURCE_ERROR_FRAGMENT: str = "requires a physical table source" -INCOMPLETE_CONFIGURATION_ERROR_FRAGMENT: str = "incomplete" COLUMN_SOURCE_NAME: str = "source_name" COLUMN_TARGET_DATABASE: str = "target_database" diff --git a/src/sqlbuild/compiler/source_freshness/main/__init__.py b/src/sqlbuild/compiler/source_freshness/main/__init__.py deleted file mode 100644 index 103f169dd0..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Direct source freshness state operations.""" diff --git a/src/sqlbuild/compiler/source_freshness/main/_planning.py b/src/sqlbuild/compiler/source_freshness/main/_planning.py deleted file mode 100644 index 21e835d36b..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/_planning.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Public direct source freshness planning entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness._helpers.planning import ( - build_direct_source_freshness_planning_result as _build_planning_result, -) -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_direct_source_freshness_planning_result( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - state_database: str | None, - state_schemas: tuple[str, ...], - observed_at: datetime, - run_id: str, - render_qualified_name: Callable[..., str | None], - state_table_exists_by_schema: dict[str, bool], - filter_previous_to_sources: bool = False, -) -> DirectSourceFreshnessPlanningResult: - """Observe direct source freshness and compare it to latest append-only state.""" - - return _build_planning_result( - adapter=adapter, - connection=connection, - sources=sources, - state_database=state_database, - state_schemas=state_schemas, - observed_at=observed_at, - run_id=run_id, - render_qualified_name=render_qualified_name, - state_table_exists_by_schema=state_table_exists_by_schema, - filter_previous_to_sources=filter_previous_to_sources, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/_propagation.py b/src/sqlbuild/compiler/source_freshness/main/_propagation.py deleted file mode 100644 index 8a12702483..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/_propagation.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public direct source freshness downstream propagation entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness._helpers import propagation as propagation_helpers -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, -) - - -def build_direct_source_freshness_propagation_result( - *, - source_freshness: DirectSourceFreshnessPlanningResult, - scope: PlannerScope, -) -> DirectSourceFreshnessPropagationResult: - """Map changed/unknown source freshness roots to downstream model names.""" - - return propagation_helpers.build_direct_source_freshness_propagation_result( - source_freshness=source_freshness, - scope=scope, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/adapter_observation.py b/src/sqlbuild/compiler/source_freshness/main/adapter_observation.py deleted file mode 100644 index 91263cf282..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/adapter_observation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public adapter metadata source freshness batch observation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness._helpers.observation import ( - observe_adapter_sources_freshness as _observe_adapter_sources_freshness, -) -from sqlbuild.compiler.source_freshness.models import AdapterSourceFreshnessBatch -from sqlbuild.spec.contracts.models import SourceEntry - - -def observe_adapter_sources_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - observed_at: datetime, -) -> AdapterSourceFreshnessBatch: - """Observe adapter metadata freshness per source, batching the warehouse lookups.""" - - return _observe_adapter_sources_freshness( - adapter=adapter, - connection=connection, - sources=sources, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py b/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py deleted file mode 100644 index cd711ea0b0..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/create_table_sql.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public direct source freshness table DDL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_create_table_sql as _build_create_table_sql, -) - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build source freshness table DDL through the public entrypoint.""" - - return _build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/observation.py b/src/sqlbuild/compiler/source_freshness/main/observation.py deleted file mode 100644 index 3ac5fa2a24..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/observation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public shared source freshness observation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness._helpers.observation import ( - observe_configured_source_freshness as _observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.spec.contracts.models import SourceEntry - - -def observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - """Observe one source freshness config and return a comparable data version.""" - - return _observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/read.py b/src/sqlbuild/compiler/source_freshness/main/read.py deleted file mode 100644 index 58f9a8fd17..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/read.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Direct source freshness read operations.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.source_freshness._helpers.datetime import normalize_presumed_utc_datetime -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_qualified_table_name, -) -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessIdentity, - SourceFreshnessRecord, - SourceFreshnessSet, -) - - -def read_latest_source_freshness( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - table_exists: bool, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_read_latest_sql: Callable[..., str], - source_names: tuple[str, ...] | None = None, -) -> SourceFreshnessSet: - """Read latest source freshness rows, trusting the caller-resolved table existence.""" - - qualified_name: str = build_qualified_table_name( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) - if not table_exists: - return SourceFreshnessSet(schema=schema, records={}) - - read_sql: str = render_read_latest_sql( - database=database, - schema=schema, - ) - if source_names is not None: - if not source_names: - return SourceFreshnessSet(schema=schema, records={}) - literals: str = ", ".join( - "'" + source_name.replace("'", "''") + "'" for source_name in source_names - ) - read_sql = ( - f"SELECT * FROM ({read_sql}) AS __sqlbuild_relevant WHERE source_name IN ({literals})" - ) - try: - result: Any = execute(connection=connection, sql=read_sql) - except Exception as error: - raise SourceFreshnessInputError( - f"Unable to read source freshness records from {qualified_name}. This can happen " - "after upgrading from an older sqlbuild version; delete or rebuild the SQLBuild " - "source freshness table to regenerate source freshness state." - ) from error - rows: list[tuple[Any, ...]] = result.fetchall() - records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = {} - row: tuple[Any, ...] - for row in rows: - record: SourceFreshnessRecord = _row_to_source_freshness_record(row) - records[record.identity] = record - return SourceFreshnessSet(schema=schema, records=records) - - -def _row_to_source_freshness_record(row: tuple[Any, ...]) -> SourceFreshnessRecord: - raw_target_database: Any = row[1] - raw_target_schema: Any = row[2] - raw_target_name: Any = row[3] - raw_data_version: Any = row[7] - raw_observed_at: Any = row[9] - observed_at: datetime = ( - raw_observed_at - if isinstance(raw_observed_at, datetime) - else datetime.fromisoformat(str(raw_observed_at)) - ) - observed_at = normalize_presumed_utc_datetime(value=observed_at) - return SourceFreshnessRecord( - source_name=str(row[0]), - target_database=str(raw_target_database) if raw_target_database is not None else None, - target_schema=str(raw_target_schema) if raw_target_schema is not None else None, - target_name=str(raw_target_name) if raw_target_name is not None else None, - run_id=str(row[4]), - strategy=str(row[5]), - value_kind=str(row[6]), - data_version=str(raw_data_version) if raw_data_version is not None else None, - data_version_hash=str(row[8]), - observed_at=observed_at, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py b/src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py deleted file mode 100644 index 7677518077..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/read_latest_sql.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public direct source freshness latest-read SQL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_read_latest_sql as _build_read_latest_sql, -) - - -def build_read_latest_sql( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - """Build source freshness latest-read SQL through the public entrypoint.""" - - return _build_read_latest_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/compiler/source_freshness/main/write.py b/src/sqlbuild/compiler/source_freshness/main/write.py deleted file mode 100644 index 6014cdabf2..0000000000 --- a/src/sqlbuild/compiler/source_freshness/main/write.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Direct source freshness write operations.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.types import AdapterExecute -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_create_table_sql, -) -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord, - SourceFreshnessRenderers, -) - - -def write_source_freshness_records( - *, - connection: Any, - execute: AdapterExecute[Any, Any], - database: str | None, - schema: str, - records: tuple[SourceFreshnessRecord, ...], - renderers: SourceFreshnessRenderers, -) -> None: - """Append source freshness rows, creating the table once if needed.""" - - if not records: - return - create_sql: str = ( - renderers.render_create_table_sql(database=database, schema=schema) - if renderers.render_create_table_sql is not None - else build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=renderers.render_qualified_name, - render_framework_type=renderers.render_framework_type, - ) - ) - _ = execute(connection=connection, sql=create_sql) - if renderers.render_create_index_sqls is not None: - index_sql: str - for index_sql in renderers.render_create_index_sqls(database=database, schema=schema): - _ = execute(connection=connection, sql=index_sql) - _ = execute( - connection=connection, - sql=renderers.render_insert_records_sql( - database=database, - schema=schema, - records=records, - ), - ) diff --git a/src/sqlbuild/compiler/source_freshness/models.py b/src/sqlbuild/compiler/source_freshness/models.py index 507b047ad1..b12173a8a1 100644 --- a/src/sqlbuild/compiler/source_freshness/models.py +++ b/src/sqlbuild/compiler/source_freshness/models.py @@ -6,8 +6,8 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import require_aware_utc_datetime from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.source_freshness._helpers.datetime import require_aware_utc_datetime from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, SourceFreshnessUnknownReason, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index 8cdfdeb7ca..b30fcfe4c8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -9,10 +9,9 @@ from textwrap import dedent from types import MappingProxyType -from sqlbuild._native import write_fingerprint +from sqlbuild._native import write_fingerprint, write_source_freshness_records from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessRecord, SourceFreshnessRenderers, diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py index c5de3b099e..5e7d43a15e 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py @@ -6,11 +6,10 @@ import pytest +from sqlbuild._native import read_latest_source_freshness, write_source_freshness_records from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index dda7c71329..a9ad4bd0d8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -3,6 +3,7 @@ from datetime import UTC, datetime from pathlib import Path +from sqlbuild._native import build_source_freshness_create_table_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -12,9 +13,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.compiler.source_freshness.main.create_table_sql import ( - build_create_table_sql as build_source_freshness_create_table_sql, -) from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index 1eb327b456..ac86b3eca9 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -5,13 +5,13 @@ import pytest -from sqlbuild._native import normalize_source_freshness_data_version +from sqlbuild._native import ( + normalize_source_freshness_data_version, + observe_configured_source_freshness, +) from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.observation import ( - observe_configured_source_freshness, -) from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index 8ca26d561c..f0ec34eb25 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -5,7 +5,11 @@ from types import MappingProxyType from typing import Any -from sqlbuild._native import source_freshness_data_version_hash +from sqlbuild._native import ( + build_source_freshness_read_latest_sql, + source_freshness_data_version_hash, + write_source_freshness_records, +) from sqlbuild.adapter.contract.models import ( RelationLookup, TableFreshnessMetadata, @@ -16,9 +20,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.source_freshness._helpers.sql import build_read_latest_sql from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, @@ -100,7 +102,7 @@ def render_qualified_name(*, database: str | None, schema: str | None, name: str def render_read_latest_sql(*, database: str | None, schema: str) -> str: - return build_read_latest_sql( + return build_source_freshness_read_latest_sql( database=database, schema=schema, render_qualified_name=render_qualified_name, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py index 08a5bd4908..c49624b344 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py @@ -4,13 +4,12 @@ import pytest +from sqlbuild._native import ( + build_source_freshness_create_table_sql, + build_source_freshness_read_latest_sql, +) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness._helpers.sql import ( - build_create_table_sql, - build_qualified_table_name, - build_read_latest_sql, -) from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from tests.unit.src.sqlbuild.compiler.source_freshness.main.shared._helpers._test_types import ( BuildSourceFreshnessSqlTestCase, @@ -35,7 +34,7 @@ def test_given_schema_when_building_qualified_name_then_returns_expected( test_case: BuildSourceFreshnessSqlTestCase, ) -> None: - result: str = build_qualified_table_name( + result: str = build_source_freshness_read_latest_sql( database=test_case.database, schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, @@ -74,7 +73,7 @@ def test_given_schema_when_building_qualified_name_then_returns_expected( def test_given_schema_when_building_create_table_sql_then_contains_expected_fragments( test_case: BuildSourceFreshnessSqlTestCase, ) -> None: - result: str = build_create_table_sql( + result: str = build_source_freshness_create_table_sql( database=test_case.database, schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, @@ -107,7 +106,7 @@ def test_given_schema_when_building_create_table_sql_then_contains_expected_frag def test_given_schema_when_building_read_latest_sql_then_contains_windowed_latest_query( test_case: BuildSourceFreshnessSqlTestCase, ) -> None: - result: str = build_read_latest_sql( + result: str = build_source_freshness_read_latest_sql( database=test_case.database, schema=test_case.schema, render_qualified_name=RENDER_QUALIFIED_NAME, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index 7d087e4ad2..8c7a31b198 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -5,13 +5,11 @@ import pytest -from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.compiler.source_freshness.main._planning import ( +from sqlbuild._native import ( build_direct_source_freshness_planning_result, -) -from sqlbuild.compiler.source_freshness.main.adapter_observation import ( observe_adapter_sources_freshness, ) +from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.compiler.source_freshness.models import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py index 1532b9780b..b81dbdfdb8 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py @@ -5,11 +5,11 @@ import pytest -from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.main.observation import ( +from sqlbuild._native import ( observe_configured_source_freshness, ) +from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.observability import ( EventDispatcher, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index 6d56c58452..06ec5f6358 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -6,14 +6,14 @@ import pytest -from sqlbuild._native import source_freshness_data_version_hash +from sqlbuild._native import ( + build_direct_source_freshness_planning_result, + source_freshness_data_version_hash, + write_source_freshness_records, +) from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.main._planning import ( - build_direct_source_freshness_planning_result, -) -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessRecord, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py index 9b04c178f6..547a5375f9 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py @@ -2,10 +2,10 @@ import pytest -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness.main._propagation import ( +from sqlbuild._native import ( build_direct_source_freshness_propagation_result, ) +from sqlbuild.compiler.planner.models import PlannerScope from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py index f94c5664be..4f0fd2b953 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py @@ -4,10 +4,9 @@ import pytest +from sqlbuild._native import read_latest_source_freshness, write_source_freshness_records from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError -from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness -from sqlbuild.compiler.source_freshness.main.write import write_source_freshness_records from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, From 8582bc298dd5e0d554bfba0dc24f61dc8cb3a997 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:03:41 +0100 Subject: [PATCH 09/33] refactor: allocate and collect Snowflake run cost natively Run-cost allocation now uses an exact Rust implementation of Python's default decimal context, and Snowflake query-history collection, query-tag parsing, metadata-only statement classification, the statement ledger and the cost context run in Rust. Connection queries, clocks and the telemetry health tracker stay Python callbacks. The cost package keeps its models, types, constants, errors and health tracker. --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/snowflake_cost/_helpers/mod.rs | 1 + .../src/snowflake_cost/_helpers/statements.rs | 43 + .../src/snowflake_cost/constants.rs | 10 + .../main/metadata_only_statement.rs | 19 + .../src/snowflake_cost/main/mod.rs | 2 + .../snowflake_cost/main/sqlbuild_query_tag.rs | 46 + .../src/snowflake_cost/mod.rs | 7 + .../src/snowflake_cost/models.rs | 9 + .../src/snowflake_cost/tests.rs | 4 + .../tests/test_snowflake_cost.rs | 94 ++ .../src/snowflake_cost/tests/test_types.rs | 14 + .../cost_allocation/_helpers/allocation.rs | 119 +++ .../src/cost_allocation/_helpers/decimal.rs | 279 ++++++ .../src/cost_allocation/_helpers/mod.rs | 2 + .../src/cost_allocation/constants.rs | 34 + .../cost_allocation/main/allocate_run_cost.rs | 104 +++ .../src/cost_allocation/main/mod.rs | 2 + .../cost_allocation/main/query_history_sql.rs | 66 ++ .../sqlbuild-core/src/cost_allocation/mod.rs | 8 + .../src/cost_allocation/models.rs | 74 ++ .../src/cost_allocation/tests.rs | 8 + .../src/cost_allocation/tests/helpers.rs | 28 + .../cost_allocation/tests/test_allocation.rs | 45 + .../src/cost_allocation/tests/test_decimal.rs | 86 ++ .../src/cost_allocation/tests/test_types.rs | 14 + .../src/cost_allocation/types.rs | 6 + crates/sqlbuild-core/src/lib.rs | 1 + .../_helpers/audit_command/command.rs | 2 +- .../_helpers/build_command/execution.rs | 2 +- .../src/bindings/_helpers/cli/registration.rs | 9 +- .../_helpers/compile_driver/python_calls.rs | 5 + .../_helpers/cost_telemetry/allocation.rs | 228 +++++ .../_helpers/cost_telemetry/context.rs | 189 ++++ .../_helpers/cost_telemetry/ledger.rs | 268 ++++++ .../bindings/_helpers/cost_telemetry/mod.rs | 6 + .../_helpers/cost_telemetry/registration.rs | 12 + .../_helpers/cost_telemetry/snowflake.rs | 823 ++++++++++++++++++ .../_helpers/cost_telemetry/values.rs | 81 ++ .../src/bindings/_helpers/mod.rs | 1 + fensu.toml | 5 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 93 +- .../snowflake/classes/snowflake_adapter.py | 2 +- .../snowflake/classes/snowflake_cursor.py | 7 +- .../_helpers/freshness/source_freshness.py | 3 +- src/sqlbuild/cost/_helpers/allocation.py | 164 ---- src/sqlbuild/cost/_helpers/context.py | 46 - src/sqlbuild/cost/_helpers/ledger.py | 105 --- src/sqlbuild/cost/_helpers/snowflake.py | 625 ------------- src/sqlbuild/cost/classes/cost_context.py | 68 -- src/sqlbuild/cost/classes/statement_ledger.py | 31 - src/sqlbuild/cost/main/__init__.py | 1 - src/sqlbuild/cost/main/collection.py | 36 - .../executor/auditing/_helpers/execution.py | 2 +- .../executor/build/_helpers/source_node.py | 2 +- .../executor/build/classes/build_scheduler.py | 2 +- src/sqlbuild/executor/load/main/_execute.py | 2 +- .../executor/pipeline/_helpers/auditing.py | 2 +- src/sqlbuild/executor/pipeline/main/run.py | 2 +- .../python_nodes/_helpers/execution.py | 2 +- .../_helpers/ingress_execution.py | 2 +- .../python_nodes/_helpers/python_checks.py | 2 +- .../read_side_python_execution_tracker.py | 2 +- .../executor/run/_helpers/execution/hooks.py | 3 +- .../sqlbuild/adapters/snowflake/test_cost.py | 3 +- .../concurrent/test_microbatch_concurrency.py | 2 +- .../test_snowflake_connection.py | 16 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 3 +- .../_helpers/test_source_freshness.py | 2 +- .../unit/src/sqlbuild/cost/test_allocation.py | 2 +- tests/unit/src/sqlbuild/cost/test_context.py | 2 +- .../unit/src/sqlbuild/cost/test_snowflake.py | 39 +- .../sqlbuild/executor/load/test_execute.py | 2 +- .../pipeline/_helpers/test_auditing.py | 9 +- .../executor/pipeline/main/test_identity.py | 2 +- .../executor/python_nodes/_helpers/helpers.py | 3 +- .../python_nodes/_helpers/test_execution.py | 2 +- .../_helpers/test_python_checks.py | 2 +- .../executor/run/_helpers/test_hooks.py | 2 +- 80 files changed, 2902 insertions(+), 1152 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/constants.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/mod.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/models.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/tests.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs create mode 100644 crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/constants.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/main/mod.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/mod.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/models.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/tests.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/cost_allocation/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs delete mode 100644 src/sqlbuild/cost/_helpers/allocation.py delete mode 100644 src/sqlbuild/cost/_helpers/context.py delete mode 100644 src/sqlbuild/cost/_helpers/ledger.py delete mode 100644 src/sqlbuild/cost/_helpers/snowflake.py delete mode 100644 src/sqlbuild/cost/classes/cost_context.py delete mode 100644 src/sqlbuild/cost/classes/statement_ledger.py delete mode 100644 src/sqlbuild/cost/main/__init__.py delete mode 100644 src/sqlbuild/cost/main/collection.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 79b7b528c6..1b29ef8804 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -38,6 +38,7 @@ pub mod resource_diagnostics; pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; +pub mod snowflake_cost; pub mod sql_test_attachment; pub mod sql_tokens; pub mod target_paths; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs new file mode 100644 index 0000000000..c0d723475b --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod statements; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs new file mode 100644 index 0000000000..7e49d9c53d --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/_helpers/statements.rs @@ -0,0 +1,43 @@ +//! Query-type prefixes, Python whitespace and CREATE statement parsing. + +use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; + +use crate::snowflake_cost::constants::{CREATE_VARIANT_PREFIX, DYNAMIC_TABLE_MODIFIER}; + +pub(crate) fn of_kind(query_type: &str, kind: &str) -> bool { + query_type == kind + || query_type + .strip_prefix(kind) + .is_some_and(|rest| rest.starts_with('_')) +} + +/// Python's `str.isspace` for one character. +pub(crate) fn python_space(character: char) -> bool { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) +} + +/// Whether a CREATE statement's text parses as a definition that needs no warehouse. +pub(crate) fn metadata_only_create(query_text: &str) -> bool { + let options: ParseOptions = ParseOptions { + complexity_guard: None, + }; + let Ok(mut statements) = + Dialect::get(DialectType::Snowflake).parse_with_options(query_text, &options) + else { + return false; + }; + if statements.len() != 1 { + return false; + } + match statements.remove(0) { + Expression::CreateTable(table) => { + !(table.as_select.is_some() + || table.clone_source.is_some() + || table.using_template.is_some() + || table.table_modifier.as_deref().unwrap_or("").to_uppercase() + == DYNAMIC_TABLE_MODIFIER) + } + Expression::CreateView(view) if view.materialized => false, + statement => statement.variant_name().starts_with(CREATE_VARIANT_PREFIX), + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs new file mode 100644 index 0000000000..e941f13be5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -0,0 +1,10 @@ +//! Snowflake query types, table modifiers and the SQLBuild query-tag identity. + +pub(crate) const USE_QUERY_TYPE: &str = "USE"; +pub(crate) const DROP_QUERY_TYPE: &str = "DROP"; +pub(crate) const CREATE_QUERY_TYPE: &str = "CREATE"; +pub(crate) const DYNAMIC_TABLE_MODIFIER: &str = "DYNAMIC"; +pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; +pub(crate) const QUERY_TAG_APP: &str = "sqlbuild"; +pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; +pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs b/crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs new file mode 100644 index 0000000000..05c21b691e --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/main/metadata_only_statement.rs @@ -0,0 +1,19 @@ +//! `metadata_only_statement`: Snowflake statements that run without warehouse compute. + +use crate::snowflake_cost::_helpers::statements::{metadata_only_create, of_kind, python_space}; +use crate::snowflake_cost::constants::{CREATE_QUERY_TYPE, DROP_QUERY_TYPE, USE_QUERY_TYPE}; + +/// Identify statements Snowflake executes without warehouse compute. +pub fn metadata_only_statement(query_type: &str, query_text: Option<&str>) -> bool { + let normalized: String = query_type.to_uppercase().replace(' ', "_"); + if of_kind(&normalized, USE_QUERY_TYPE) || of_kind(&normalized, DROP_QUERY_TYPE) { + return true; + } + if !of_kind(&normalized, CREATE_QUERY_TYPE) { + return false; + } + match query_text { + Some(text) if !text.chars().all(python_space) => metadata_only_create(text), + _ => false, + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs new file mode 100644 index 0000000000..cf473b47aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/main/mod.rs @@ -0,0 +1,2 @@ +pub mod metadata_only_statement; +pub mod sqlbuild_query_tag; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs b/crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs new file mode 100644 index 0000000000..497ffa35c7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/main/sqlbuild_query_tag.rs @@ -0,0 +1,46 @@ +//! `sqlbuild_query_tag`: the run attribution of a SQLBuild Snowflake `QUERY_TAG`. + +use serde_json::Value; + +use crate::snowflake_cost::constants::{QUERY_TAG_APP, QUERY_TAG_APP_KEY, QUERY_TAG_VERSION_KEY}; +use crate::snowflake_cost::models::QueryTag; + +/// Python's `value == 1` for a decoded JSON value. +fn equals_one(value: Option<&Value>) -> bool { + match value { + Some(Value::Bool(flag)) => *flag, + Some(Value::Number(number)) => number + .as_i64() + .map(|integer| integer == 1) + .or_else(|| number.as_u64().map(|integer| integer == 1)) + .unwrap_or_else(|| number.as_f64() == Some(1.0)), + _ => false, + } +} + +fn text(payload: &serde_json::Map, key: &str) -> Option { + match payload.get(key) { + Some(Value::String(value)) if !value.is_empty() => Some(value.clone()), + _ => None, + } +} + +/// The tag's attribution when it is a SQLBuild version-1 JSON object, else `None`. +pub fn sqlbuild_query_tag(value: &str) -> Option { + if value.is_empty() { + return None; + } + let Ok(Value::Object(payload)) = serde_json::from_str::(value) else { + return None; + }; + let app_matches: bool = + matches!(payload.get(QUERY_TAG_APP_KEY), Some(Value::String(app)) if app == QUERY_TAG_APP); + if !app_matches || !equals_one(payload.get(QUERY_TAG_VERSION_KEY)) { + return None; + } + Some(QueryTag { + run_id: text(&payload, "run_id"), + resource_type: text(&payload, "resource_type"), + resource_name: text(&payload, "resource_name"), + }) +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/models.rs b/crates/sqlbuild-analysis/src/snowflake_cost/models.rs new file mode 100644 index 0000000000..a3188938cc --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/models.rs @@ -0,0 +1,9 @@ +//! The run attribution a SQLBuild Snowflake query tag carries. + +/// Non-empty string fields of a SQLBuild query tag. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct QueryTag { + pub run_id: Option, + pub resource_type: Option, + pub resource_name: Option, +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/tests.rs b/crates/sqlbuild-analysis/src/snowflake_cost/tests.rs new file mode 100644 index 0000000000..79d552fda0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_snowflake_cost.rs"] +mod snowflake_cost; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs new file mode 100644 index 0000000000..c373af2531 --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_snowflake_cost.rs @@ -0,0 +1,94 @@ +use crate::snowflake_cost::main::metadata_only_statement::metadata_only_statement; +use crate::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; +use crate::snowflake_cost::models::QueryTag; +use crate::snowflake_cost::tests::test_types::{MetadataOnlyTestCase, QueryTagTestCase}; + +#[test] +fn given_snowflake_statements_when_classifying_then_detects_metadata_only_work() { + let test_cases = [ + MetadataOnlyTestCase { + description: "use statements need no warehouse", + query_type: "USE_DATABASE", + query_text: None, + expected_metadata_only: true, + }, + MetadataOnlyTestCase { + description: "a column-list table is metadata only", + query_type: "CREATE_TABLE", + query_text: Some("CREATE TABLE t (a INT)"), + expected_metadata_only: true, + }, + MetadataOnlyTestCase { + description: "a table built from a query uses the warehouse", + query_type: "CREATE_TABLE", + query_text: Some("CREATE TABLE t AS SELECT 1 AS a"), + expected_metadata_only: false, + }, + MetadataOnlyTestCase { + description: "a materialized view uses the warehouse", + query_type: "CREATE_VIEW", + query_text: Some("CREATE MATERIALIZED VIEW v AS SELECT 1 AS a"), + expected_metadata_only: false, + }, + MetadataOnlyTestCase { + description: "blank create text is not classified", + query_type: "CREATE", + query_text: Some(" "), + expected_metadata_only: false, + }, + MetadataOnlyTestCase { + description: "a create-prefixed type without separator is ordinary work", + query_type: "CREATEX", + query_text: Some("CREATE TABLE t (a INT)"), + expected_metadata_only: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + metadata_only_statement(test_case.query_type, test_case.query_text), + test_case.expected_metadata_only, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_query_tags_when_parsing_then_reads_sqlbuild_attribution() { + let test_cases = [ + QueryTagTestCase { + description: "a version-one tag keeps non-empty strings", + value: r#"{"app": "sqlbuild", "v": 1.0, "run_id": "r", "resource_name": ""}"#, + expected_tag: Some(QueryTag { + run_id: Some("r".to_owned()), + resource_type: None, + resource_name: None, + }), + }, + QueryTagTestCase { + description: "true equals one", + value: r#"{"app": "sqlbuild", "v": true}"#, + expected_tag: Some(QueryTag::default()), + }, + QueryTagTestCase { + description: "other versions are ignored", + value: r#"{"app": "sqlbuild", "v": 2}"#, + expected_tag: None, + }, + QueryTagTestCase { + description: "non-objects are ignored", + value: "[]", + expected_tag: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + sqlbuild_query_tag(test_case.value), + test_case.expected_tag, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs new file mode 100644 index 0000000000..20b8acfeca --- /dev/null +++ b/crates/sqlbuild-analysis/src/snowflake_cost/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::snowflake_cost::models::QueryTag; + +pub(crate) struct MetadataOnlyTestCase { + pub(crate) description: &'static str, + pub(crate) query_type: &'static str, + pub(crate) query_text: Option<&'static str>, + pub(crate) expected_metadata_only: bool, +} + +pub(crate) struct QueryTagTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) expected_tag: Option, +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs new file mode 100644 index 0000000000..7dccb5e13f --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/allocation.rs @@ -0,0 +1,119 @@ +//! Busy-time attribution, warehouse rates, totals and ordering of priced resources. + +use std::cmp::Ordering; +use std::collections::{BTreeSet, HashMap}; + +use crate::cost_allocation::_helpers::decimal::{ + add, compare, divide, from_float, integer, multiply, +}; +use crate::cost_allocation::constants::{CREDITS_PER_HOUR, MICROS_PER_SECOND, SECONDS_PER_HOUR}; +use crate::cost_allocation::models::{CostQuery, PyDecimal, ResourceAllocation}; +use crate::cost_allocation::types::ResourceKey; + +/// Items grouped by key in first-appearance order, as a Python `defaultdict(list)`. +pub(crate) fn grouped<'a, K: Eq + std::hash::Hash + Clone>( + items: impl Iterator, + key: impl Fn(&CostQuery) -> K, +) -> Vec<(K, Vec<&'a CostQuery>)> { + let mut positions: HashMap = HashMap::new(); + let mut groups: Vec<(K, Vec<&'a CostQuery>)> = Vec::new(); + for item in items { + let item_key: K = key(item); + match positions.get(&item_key) { + Some(&position) => groups[position].1.push(item), + None => { + positions.insert(item_key.clone(), groups.len()); + groups.push((item_key, vec![item])); + } + } + } + groups +} + +/// Busy seconds attributed to each run query ID, accumulated in Python's iteration order. +pub(crate) fn attributed_seconds(queries: &[CostQuery]) -> HashMap { + let mut attributed: HashMap = HashMap::new(); + for (_, warehouse_queries) in grouped(queries.iter(), |query| query.warehouse.clone()) { + let boundaries: Vec = warehouse_queries + .iter() + .flat_map(|query| [query.started_micros, query.completed_micros]) + .collect::>() + .into_iter() + .collect(); + for window in boundaries.windows(2) { + let (start, end) = (window[0], window[1]); + let active: Vec<&&CostQuery> = warehouse_queries + .iter() + .filter(|query| query.started_micros < end && query.completed_micros > start) + .collect(); + if active.is_empty() { + continue; + } + let elapsed: f64 = (end - start) as f64 / MICROS_PER_SECOND; + let Some(elapsed) = from_float(elapsed) else { + continue; + }; + let share: PyDecimal = divide(&elapsed, active.len() as u64); + for query in active.into_iter().filter(|query| query.in_run) { + let total: PyDecimal = match attributed.get(&query.query_id) { + Some(total) => add(total, &share), + None => add(&integer(0), &share), + }; + attributed.insert(query.query_id.clone(), total); + } + } + } + attributed +} + +pub(crate) fn credits_per_hour(size: &str) -> Option { + let size: String = size.to_uppercase(); + CREDITS_PER_HOUR + .iter() + .find(|(name, _)| *name == size) + .map(|(_, rate)| *rate) +} + +pub(crate) fn sum<'a>(values: impl Iterator) -> PyDecimal { + values.fold(integer(0), |total, value| add(&total, value)) +} + +/// Python's sort key of a resource cost: most expensive first, then by identity. +pub(crate) fn resource_order(left: &ResourceAllocation, right: &ResourceAllocation) -> Ordering { + let cluster = |resource: &ResourceAllocation| resource.warehouse.cluster.unwrap_or(-1); + compare(&right.estimated_usd, &left.estimated_usd) + .then_with(|| left.resource_name.cmp(&right.resource_name)) + .then_with(|| left.resource_type.cmp(&right.resource_type)) + .then_with(|| left.warehouse.name.cmp(&right.warehouse.name)) + .then_with(|| left.warehouse.size.cmp(&right.warehouse.size)) + .then_with(|| { + left.warehouse + .kind + .as_deref() + .unwrap_or("") + .cmp(right.warehouse.kind.as_deref().unwrap_or("")) + }) + .then_with(|| cluster(left).cmp(&cluster(right))) +} + +pub(crate) fn resource_allocation( + (resource_type, resource_name, warehouse): ResourceKey, + queries: &[&CostQuery], + (attributed, rate, usd_per_credit): (&HashMap, u64, &PyDecimal), +) -> ResourceAllocation { + let zero: PyDecimal = integer(0); + let seconds: PyDecimal = sum(queries + .iter() + .map(|query| attributed.get(&query.query_id).unwrap_or(&zero))); + let credits: PyDecimal = divide(&multiply(&seconds, &integer(rate)), SECONDS_PER_HOUR); + ResourceAllocation { + resource_type, + resource_name, + warehouse, + query_count: queries.len(), + attributed_seconds: seconds, + bytes_scanned: queries.iter().map(|query| query.bytes_scanned).sum(), + estimated_usd: multiply(&credits, usd_per_credit), + estimated_compute_credits: credits, + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs new file mode 100644 index 0000000000..5ce0ff4df4 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/decimal.rs @@ -0,0 +1,279 @@ +//! Python `Decimal` arithmetic in the default context: 28 digits, `ROUND_HALF_EVEN`. + +use std::cmp::Ordering; + +#[cfg(test)] +use crate::cost_allocation::constants::SMALLEST_PLAIN_LEFT_DIGITS; +use crate::cost_allocation::constants::{DECIMAL_PRECISION, HALF_DIGIT, MAX_DIGIT}; +use crate::cost_allocation::models::PyDecimal; +use crate::json::main::python_float_repr::python_float_repr; + +/// Digits without leading zeros; zero is `[0]`. +fn normalized(mut digits: Vec) -> Vec { + let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); + digits.drain(..leading.min(digits.len().saturating_sub(1))); + if digits.is_empty() { + digits.push(0); + } + digits +} + +fn is_zero(digits: &[u8]) -> bool { + digits.iter().all(|digit| *digit == 0) +} + +/// An exact integer as a decimal with exponent zero. +pub(crate) fn integer(value: u64) -> PyDecimal { + PyDecimal { + digits: value.to_string().bytes().map(|byte| byte - b'0').collect(), + exponent: 0, + } +} + +/// A signed decimal exponent such as `+16` or `-06`. +fn exponent_value(text: &str) -> Option { + let (negative, digits): (bool, &str) = match text.as_bytes().first() { + Some(b'-') => (true, &text[1..]), + Some(b'+') => (false, &text[1..]), + _ => (false, text), + }; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let magnitude: i64 = digits.bytes().fold(0_i64, |value, byte| { + value + .saturating_mul(10) + .saturating_add(i64::from(byte - b'0')) + }); + Some(if negative { -magnitude } else { magnitude }) +} + +/// `Decimal(text)` for plain decimal or scientific text of a non-negative finite number. +pub(crate) fn parse(text: &str) -> Option { + let (mantissa, exponent): (&str, i64) = match text.find(['e', 'E']) { + Some(index) => (&text[..index], exponent_value(&text[index + 1..])?), + None => (text, 0), + }; + let (whole, fraction): (&str, &str) = mantissa.split_once('.').unwrap_or((mantissa, "")); + if whole.is_empty() && fraction.is_empty() { + return None; + } + let digits: Vec = whole + .bytes() + .chain(fraction.bytes()) + .map(|byte| byte.is_ascii_digit().then(|| byte - b'0')) + .collect::>()?; + let fraction_length: i64 = i64::try_from(fraction.len()).unwrap_or(i64::MAX); + Some(PyDecimal { + digits: normalized(digits), + exponent: exponent - fraction_length, + }) +} + +/// Round to the context precision with `ROUND_HALF_EVEN`, as `Decimal._fix` does. +pub(crate) fn rounded(digits: Vec, exponent: i64) -> PyDecimal { + let digits: Vec = normalized(digits); + if digits.len() <= DECIMAL_PRECISION { + return PyDecimal { digits, exponent }; + } + let dropped: usize = digits.len() - DECIMAL_PRECISION; + let mut kept: Vec = digits[..DECIMAL_PRECISION].to_vec(); + let rest: &[u8] = &digits[DECIMAL_PRECISION..]; + let round_up: bool = match rest[0].cmp(&HALF_DIGIT) { + Ordering::Greater => true, + Ordering::Less => false, + Ordering::Equal => !is_zero(&rest[1..]) || kept[DECIMAL_PRECISION - 1] % 2 == 1, + }; + let mut exponent: i64 = exponent + i64::try_from(dropped).unwrap_or(i64::MAX); + if round_up { + let mut index: usize = kept.len(); + loop { + if index == 0 { + kept.insert(0, 1); + break; + } + index -= 1; + if kept[index] == MAX_DIGIT { + kept[index] = 0; + } else { + kept[index] += 1; + break; + } + } + if kept.len() > DECIMAL_PRECISION { + kept.pop(); + exponent += 1; + } + } + PyDecimal { + digits: kept, + exponent, + } +} + +fn shifted(value: &PyDecimal, exponent: i64) -> Vec { + let zeros: usize = usize::try_from(value.exponent - exponent).unwrap_or(0); + let mut digits: Vec = value.digits.clone(); + digits.extend(std::iter::repeat_n(0, zeros)); + digits +} + +/// `left + right`. +pub(crate) fn add(left: &PyDecimal, right: &PyDecimal) -> PyDecimal { + let exponent: i64 = left.exponent.min(right.exponent); + let (mut first, mut second) = (shifted(left, exponent), shifted(right, exponent)); + let width: usize = first.len().max(second.len()) + 1; + first.splice(0..0, std::iter::repeat_n(0, width - first.len())); + second.splice(0..0, std::iter::repeat_n(0, width - second.len())); + let mut carry: u8 = 0; + let mut sum: Vec = vec![0; width]; + for index in (0..width).rev() { + let total: u8 = first[index] + second[index] + carry; + sum[index] = total % 10; + carry = total / 10; + } + rounded(sum, exponent) +} + +/// `left * right`. +pub(crate) fn multiply(left: &PyDecimal, right: &PyDecimal) -> PyDecimal { + let mut product: Vec = vec![0; left.digits.len() + right.digits.len()]; + for (left_index, left_digit) in left.digits.iter().enumerate().rev() { + for (right_index, right_digit) in right.digits.iter().enumerate().rev() { + product[left_index + right_index + 1] += + u32::from(*left_digit) * u32::from(*right_digit); + } + } + for index in (1..product.len()).rev() { + product[index - 1] += product[index] / 10; + product[index] %= 10; + } + let digits: Vec = product + .into_iter() + .map(|digit| u8::try_from(digit).unwrap_or(0)) + .collect(); + rounded(digits, left.exponent + right.exponent) +} + +/// Quotient and remainder of a digit string divided by a small positive integer. +fn divide_digits(digits: &[u8], divisor: u64) -> (Vec, bool) { + let mut remainder: u128 = 0; + let mut quotient: Vec = Vec::with_capacity(digits.len()); + for digit in digits { + remainder = remainder * 10 + u128::from(*digit); + quotient.push(u8::try_from(remainder / u128::from(divisor)).unwrap_or(0)); + remainder %= u128::from(divisor); + } + (normalized(quotient), remainder != 0) +} + +fn last_digit(digits: &[u8]) -> u8 { + digits.last().copied().unwrap_or(0) +} + +/// `dividend / Decimal(divisor)` for a positive integer divisor, as `Decimal.__truediv__`. +pub(crate) fn divide(dividend: &PyDecimal, divisor: u64) -> PyDecimal { + if is_zero(÷nd.digits) { + return PyDecimal { + digits: vec![0], + exponent: dividend.exponent, + }; + } + let divisor_length: i64 = i64::try_from(divisor.to_string().len()).unwrap_or(0); + let dividend_length: i64 = i64::try_from(dividend.digits.len()).unwrap_or(0); + let precision: i64 = i64::try_from(DECIMAL_PRECISION).unwrap_or(0); + let shift: i64 = divisor_length - dividend_length + precision + 1; + let mut exponent: i64 = dividend.exponent - shift; + let (mut quotient, inexact) = if shift >= 0 { + let mut digits: Vec = dividend.digits.clone(); + digits.extend(std::iter::repeat_n(0, usize::try_from(shift).unwrap_or(0))); + divide_digits(&digits, divisor) + } else { + let split: usize = dividend.digits.len() - usize::try_from(-shift).unwrap_or(0); + let (high, low) = dividend.digits.split_at(split); + let (quotient, remainder) = divide_digits(high, divisor); + (quotient, remainder || !is_zero(low)) + }; + if inexact { + if last_digit("ient).is_multiple_of(HALF_DIGIT) { + let index: usize = quotient.len() - 1; + quotient[index] += 1; + } + } else { + while exponent < dividend.exponent && quotient.len() > 1 && last_digit("ient) == 0 { + quotient.pop(); + exponent += 1; + } + } + rounded(quotient, exponent) +} + +/// Numeric order of two decimals, ignoring their exponents' representation. +pub(crate) fn compare(left: &PyDecimal, right: &PyDecimal) -> Ordering { + match (is_zero(&left.digits), is_zero(&right.digits)) { + (true, true) => return Ordering::Equal, + (true, false) => return Ordering::Less, + (false, true) => return Ordering::Greater, + (false, false) => {} + } + let adjusted = |value: &PyDecimal| -> i64 { + value.exponent + i64::try_from(value.digits.len()).unwrap_or(0) + }; + adjusted(left).cmp(&adjusted(right)).then_with(|| { + let width: usize = left.digits.len().max(right.digits.len()); + let padded = |digits: &[u8]| -> Vec { + let mut padded: Vec = digits.to_vec(); + padded.resize(width, 0); + padded + }; + padded(&left.digits).cmp(&padded(&right.digits)) + }) +} + +/// `str(Decimal)`: scientific notation when the exponent is positive or the value is tiny. +#[cfg(test)] +pub(crate) fn text(value: &PyDecimal) -> String { + let digits: String = value + .digits + .iter() + .map(|digit| char::from(b'0' + digit)) + .collect(); + let length: i64 = i64::try_from(digits.len()).unwrap_or(0); + let left_digits: i64 = value.exponent + length; + let dot_place: i64 = if value.exponent <= 0 && left_digits > SMALLEST_PLAIN_LEFT_DIGITS { + left_digits + } else { + 1 + }; + let (integer_part, fraction_part): (String, String) = if dot_place <= 0 { + ( + "0".to_owned(), + format!( + ".{}{digits}", + "0".repeat(usize::try_from(-dot_place).unwrap_or(0)) + ), + ) + } else if dot_place >= length { + ( + format!( + "{digits}{}", + "0".repeat(usize::try_from(dot_place - length).unwrap_or(0)) + ), + String::new(), + ) + } else { + let split: usize = usize::try_from(dot_place).unwrap_or(0); + (digits[..split].to_owned(), format!(".{}", &digits[split..])) + }; + let suffix: String = if left_digits == dot_place { + String::new() + } else { + format!("E{:+}", left_digits - dot_place) + }; + format!("{integer_part}{fraction_part}{suffix}") +} + +/// `Decimal(str(value))` for a non-negative finite float; `None` for NaN or infinities. +pub(crate) fn from_float(value: f64) -> Option { + parse(&python_float_repr(value)) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs new file mode 100644 index 0000000000..3b9c8a22a1 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod allocation; +pub(crate) mod decimal; diff --git a/crates/sqlbuild-core/src/cost_allocation/constants.rs b/crates/sqlbuild-core/src/cost_allocation/constants.rs new file mode 100644 index 0000000000..09c13bd940 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/constants.rs @@ -0,0 +1,34 @@ +//! Python `decimal` context limits, Snowflake credit rates and allocation messages. + +/// `decimal.getcontext().prec` of the default context. +pub(crate) const DECIMAL_PRECISION: usize = 28; +pub(crate) const SECONDS_PER_HOUR: u64 = 3_600; +pub(crate) const MICROS_PER_SECOND: f64 = 1_000_000.0; +pub(crate) const STANDARD_WAREHOUSE_TYPE: &str = "STANDARD"; +pub(crate) const UNKNOWN_RESOURCE: &str = "unknown"; +pub(crate) const RESULT_LIMIT_LIMITATION: &str = + "Snowflake query-history result limit was reached."; +/// Credits per hour by uppercased Snowflake warehouse size. +pub(crate) const CREDITS_PER_HOUR: [(&str, u64); 10] = [ + ("X-SMALL", 1), + ("SMALL", 2), + ("MEDIUM", 4), + ("LARGE", 8), + ("X-LARGE", 16), + ("2X-LARGE", 32), + ("3X-LARGE", 64), + ("4X-LARGE", 128), + ("5X-LARGE", 256), + ("6X-LARGE", 512), +]; +/// The default `RESULT_LIMIT` of one Snowflake query-history read. +pub const QUERY_HISTORY_RESULT_LIMIT: usize = 10_000; +pub(crate) const QUERY_HISTORY_FUNCTION: &str = "INFORMATION_SCHEMA.QUERY_HISTORY"; +pub(crate) const NULL_QUERY_TEXT: &str = "NULL AS QUERY_TEXT"; +/// The rounding digit at which `ROUND_HALF_EVEN` looks past the first dropped digit. +pub(crate) const HALF_DIGIT: u8 = 5; +/// `str(Decimal)` switches to exponent form at or below this many left digits. +#[cfg(test)] +pub(crate) const SMALLEST_PLAIN_LEFT_DIGITS: i64 = -6; +/// The largest decimal digit, which carries when rounded up. +pub(crate) const MAX_DIGIT: u8 = 9; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs b/crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs new file mode 100644 index 0000000000..ba5ea60444 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/allocate_run_cost.rs @@ -0,0 +1,104 @@ +//! `allocate_run_cost`: share each warehouse's busy time equally among concurrently active queries. + +use std::collections::{BTreeSet, HashMap}; + +use crate::cost_allocation::_helpers::allocation::{ + attributed_seconds, credits_per_hour, grouped, resource_allocation, resource_order, sum, +}; +use crate::cost_allocation::_helpers::decimal::integer; +use crate::cost_allocation::constants::{ + RESULT_LIMIT_LIMITATION, STANDARD_WAREHOUSE_TYPE, UNKNOWN_RESOURCE, +}; +use crate::cost_allocation::models::{ + AllocationStatus, CostQuery, PyDecimal, ResourceAllocation, RunAllocation, WarehouseKey, +}; + +/// Attribute the run's share of busy warehouse time and price it; pending without run queries. +pub fn allocate_run_cost( + queries: &[CostQuery], + usd_per_credit: &PyDecimal, + result_limit_reached: bool, +) -> RunAllocation { + let run_queries: Vec<&CostQuery> = queries.iter().filter(|query| query.in_run).collect(); + if run_queries.is_empty() { + return RunAllocation { + status: AllocationStatus::Pending, + resources: Vec::new(), + query_count: 0, + attributed_seconds: integer(0), + bytes_scanned: 0, + estimated_compute_credits: integer(0), + estimated_usd: integer(0), + limitations: Vec::new(), + }; + } + let attributed: HashMap = attributed_seconds(queries); + let mut limitations: Vec = Vec::new(); + let mut resources: Vec = Vec::new(); + for (key, group) in grouped(run_queries.iter().copied(), |query| { + ( + query + .resource_type + .clone() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| UNKNOWN_RESOURCE.to_owned()), + query + .resource_name + .clone() + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| UNKNOWN_RESOURCE.to_owned()), + query.warehouse.clone(), + ) + }) { + let warehouse: &WarehouseKey = &key.2; + let rate: Option = credits_per_hour(&warehouse.size); + match &warehouse.kind { + None => limitations.push(format!("Unknown warehouse type for {}", warehouse.name)), + Some(kind) if kind.to_uppercase() != STANDARD_WAREHOUSE_TYPE => { + limitations.push(format!("Unsupported Snowflake warehouse type: {kind}")); + continue; + } + Some(_) => {} + } + if warehouse.cluster.is_none() { + limitations.push(format!("Unknown cluster number for {}", warehouse.name)); + } + let Some(rate) = rate else { + limitations.push(format!( + "Unknown Snowflake warehouse size: {}", + warehouse.size + )); + continue; + }; + resources.push(resource_allocation( + key, + &group, + (&attributed, rate, usd_per_credit), + )); + } + if result_limit_reached { + limitations.push(RESULT_LIMIT_LIMITATION.to_owned()); + } + resources.sort_by(resource_order); + let limitations: Vec = limitations + .into_iter() + .collect::>() + .into_iter() + .collect(); + RunAllocation { + status: if limitations.is_empty() { + AllocationStatus::Complete + } else { + AllocationStatus::Partial + }, + query_count: run_queries.len(), + attributed_seconds: sum(resources.iter().map(|item| &item.attributed_seconds)), + bytes_scanned: resources.iter().map(|item| item.bytes_scanned).sum(), + estimated_compute_credits: sum(resources + .iter() + .map(|item| &item.estimated_compute_credits)), + estimated_usd: sum(resources.iter().map(|item| &item.estimated_usd)), + limitations, + resources, + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/main/mod.rs b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs new file mode 100644 index 0000000000..d4b3364262 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/mod.rs @@ -0,0 +1,2 @@ +pub mod allocate_run_cost; +pub mod query_history_sql; diff --git a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs new file mode 100644 index 0000000000..40b6028249 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs @@ -0,0 +1,66 @@ +//! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window. + +use crate::cost_allocation::constants::{NULL_QUERY_TEXT, QUERY_HISTORY_FUNCTION}; +use crate::cost_allocation::models::QueryHistoryWindow; + +fn query_history_function(database: Option<&str>) -> String { + match database { + None => QUERY_HISTORY_FUNCTION.to_owned(), + Some(database) => format!( + "\"{}\".{QUERY_HISTORY_FUNCTION}", + database.to_uppercase().replace('"', "\"\"") + ), + } +} + +fn query_text_expression(query_ids: &[String]) -> String { + if query_ids.is_empty() { + return NULL_QUERY_TEXT.to_owned(); + } + let mut sorted: Vec<&String> = query_ids.iter().collect(); + sorted.sort(); + sorted.dedup(); + let literals: Vec = sorted + .into_iter() + .map(|query_id| format!("'{}'", query_id.replace('\'', "''"))) + .collect(); + format!( + "CASE WHEN QUERY_ID IN ({}) AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') \ +THEN QUERY_TEXT END AS QUERY_TEXT", + literals.join(", ") + ) +} + +/// Query-history SQL for queries ending after `start`, and before `end` when it is bounded. +pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String { + let end_range: String = match window.end { + Some(end) => format!("\n END_TIME_RANGE_END => TO_TIMESTAMP_LTZ('{end}'),"), + None => String::new(), + }; + format!( + "SELECT + QUERY_ID, + QUERY_TAG, + WAREHOUSE_NAME, + WAREHOUSE_SIZE, + WAREHOUSE_TYPE, + CLUSTER_NUMBER, + START_TIME, + END_TIME, + EXECUTION_TIME, + BYTES_SCANNED, + EXECUTION_STATUS, + QUERY_TYPE, + {}, + CURRENT_TIMESTAMP() +FROM TABLE({}( + END_TIME_RANGE_START => TO_TIMESTAMP_LTZ('{}'),{end_range} + RESULT_LIMIT => {} +)) +ORDER BY START_TIME, QUERY_ID", + query_text_expression(window.classification_query_ids), + query_history_function(window.database), + window.start, + window.result_limit, + ) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/mod.rs b/crates/sqlbuild-core/src/cost_allocation/mod.rs new file mode 100644 index 0000000000..5853224abe --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/cost_allocation/models.rs b/crates/sqlbuild-core/src/cost_allocation/models.rs new file mode 100644 index 0000000000..349f862fbe --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/models.rs @@ -0,0 +1,74 @@ +//! Exact Python decimals, observed queries and their attributed run cost. + +/// A finite non-negative Python `Decimal`: coefficient digits, most significant first, and exponent. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PyDecimal { + pub digits: Vec, + pub exponent: i64, +} + +/// The warehouse identity whose busy time concurrent queries share. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct WarehouseKey { + pub name: String, + pub size: String, + pub kind: Option, + pub cluster: Option, +} + +/// One observed query; instants are microseconds on one consistent clock. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostQuery { + pub query_id: String, + pub warehouse: WarehouseKey, + pub started_micros: i128, + pub completed_micros: i128, + pub in_run: bool, + pub resource_type: Option, + pub resource_name: Option, + pub bytes_scanned: i128, +} + +/// The cost attributed to one resource on one warehouse (`ResourceCost`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceAllocation { + pub resource_type: String, + pub resource_name: String, + pub warehouse: WarehouseKey, + pub query_count: usize, + pub attributed_seconds: PyDecimal, + pub bytes_scanned: i128, + pub estimated_compute_credits: PyDecimal, + pub estimated_usd: PyDecimal, +} + +/// Whether a run's queries were seen and fully priced. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AllocationStatus { + Pending, + Partial, + Complete, +} + +/// The run cost of the selected run's queries (`RunCostSummary` without its rate fields). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunAllocation { + pub status: AllocationStatus, + pub resources: Vec, + pub query_count: usize, + pub attributed_seconds: PyDecimal, + pub bytes_scanned: i128, + pub estimated_compute_credits: PyDecimal, + pub estimated_usd: PyDecimal, + pub limitations: Vec, +} + +/// The time window and classified query IDs of one query-history read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryHistoryWindow<'a> { + pub database: Option<&'a str>, + pub start: &'a str, + pub end: Option<&'a str>, + pub classification_query_ids: &'a [String], + pub result_limit: usize, +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests.rs b/crates/sqlbuild-core/src/cost_allocation/tests.rs new file mode 100644 index 0000000000..6949838ed8 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_allocation.rs"] +mod allocation; +#[path = "tests/test_decimal.rs"] +mod decimal; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs b/crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs new file mode 100644 index 0000000000..ec1a534bf5 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/helpers.rs @@ -0,0 +1,28 @@ +use crate::cost_allocation::_helpers::decimal::{from_float, integer, parse}; +use crate::cost_allocation::models::{CostQuery, PyDecimal, WarehouseKey}; + +pub(crate) fn decimal(text: &str) -> PyDecimal { + parse(text).unwrap_or_else(|| integer(0)) +} + +pub(crate) fn float(value: f64) -> PyDecimal { + from_float(value).unwrap_or_else(|| integer(0)) +} + +pub(crate) fn query(index: u64) -> CostQuery { + CostQuery { + query_id: format!("q{index}"), + warehouse: WarehouseKey { + name: "WH".to_owned(), + size: "X-SMALL".to_owned(), + kind: Some("STANDARD".to_owned()), + cluster: Some(1), + }, + started_micros: 0, + completed_micros: 1_000_000, + in_run: true, + resource_type: Some("model".to_owned()), + resource_name: Some("orders".to_owned()), + bytes_scanned: 1, + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs new file mode 100644 index 0000000000..6d9d540098 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_allocation.rs @@ -0,0 +1,45 @@ +use crate::cost_allocation::_helpers::decimal::text; +use crate::cost_allocation::main::allocate_run_cost::allocate_run_cost; +use crate::cost_allocation::models::{CostQuery, PyDecimal, RunAllocation}; +use crate::cost_allocation::tests::helpers::query; +use crate::cost_allocation::tests::test_types::AllocationTestCase; + +#[test] +fn given_concurrent_queries_when_allocating_then_shares_busy_seconds_equally() { + let test_cases = [ + AllocationTestCase { + description: "one query owns the whole second", + concurrent_queries: 1, + expected_seconds: "1.0", + expected_usd: "0.0008333333333333333333333333334", + }, + AllocationTestCase { + description: "seven concurrent queries round each share", + concurrent_queries: 7, + expected_seconds: "1.000000000000000000000000000", + expected_usd: "0.0008333333333333333333333333334", + }, + ]; + + for test_case in test_cases { + let queries: Vec = (0..test_case.concurrent_queries).map(query).collect(); + let rate: PyDecimal = PyDecimal { + digits: vec![3, 0, 0], + exponent: -2, + }; + let allocation: RunAllocation = allocate_run_cost(&queries, &rate, false); + + assert_eq!( + ( + text(&allocation.attributed_seconds), + text(&allocation.estimated_usd), + ), + ( + test_case.expected_seconds.to_owned(), + test_case.expected_usd.to_owned(), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs new file mode 100644 index 0000000000..356b9ce1ca --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_decimal.rs @@ -0,0 +1,86 @@ +use crate::cost_allocation::_helpers::decimal::{add, divide, integer, multiply, text}; +use crate::cost_allocation::tests::helpers::{decimal, float}; +use crate::cost_allocation::tests::test_types::DecimalOperationTestCase; + +#[test] +fn given_python_decimals_when_operating_then_rounds_like_the_default_context() { + let test_cases = [ + DecimalOperationTestCase { + description: "inexact division rounds to 28 digits", + result: divide(&decimal("1.0"), 3), + expected_text: "0.3333333333333333333333333333", + }, + DecimalOperationTestCase { + description: "tiny quotients switch to scientific text", + result: divide(&decimal("0.000005"), 7), + expected_text: "7.142857142857142857142857143E-7", + }, + DecimalOperationTestCase { + description: "exact division keeps the ideal exponent", + result: divide(&decimal("12"), 4), + expected_text: "3", + }, + DecimalOperationTestCase { + description: "exact division keeps needed fraction digits", + result: divide(&decimal("7.5"), 3), + expected_text: "2.5", + }, + DecimalOperationTestCase { + description: "zero division keeps the dividend exponent", + result: divide(&decimal("0.000000"), 3_600), + expected_text: "0.000000", + }, + DecimalOperationTestCase { + description: "adding zero aligns a large exponent and rounds", + result: add(&decimal("1E+30"), &integer(0)), + expected_text: "1.000000000000000000000000000E+30", + }, + DecimalOperationTestCase { + description: "a half above an odd coefficient rounds up with carry", + result: add(&decimal("9999999999999999999999999999"), &decimal("0.5")), + expected_text: "1.000000000000000000000000000E+28", + }, + DecimalOperationTestCase { + description: "a half above an even coefficient rounds to even", + result: add(&decimal("9999999999999999999999999998"), &decimal("0.5")), + expected_text: "9999999999999999999999999998", + }, + DecimalOperationTestCase { + description: "long products round to 28 digits", + result: multiply( + &decimal("3.123456789012345678901234567890123"), + &decimal("1.234"), + ), + expected_text: "3.854345677641234567764123457", + }, + DecimalOperationTestCase { + description: "zero products keep the summed exponent", + result: multiply(&integer(0), &decimal("3.00")), + expected_text: "0.00", + }, + DecimalOperationTestCase { + description: "integral floats keep repr's trailing zero", + result: float(3_600.0), + expected_text: "3600.0", + }, + DecimalOperationTestCase { + description: "small floats read repr's exponent form", + result: float(5e-6), + expected_text: "0.000005", + }, + DecimalOperationTestCase { + description: "large floats read repr's exponent form", + result: float(1e16), + expected_text: "1E+16", + }, + ]; + + for test_case in test_cases { + assert_eq!( + text(&test_case.result), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs new file mode 100644 index 0000000000..cefbc507fe --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::cost_allocation::models::PyDecimal; + +pub(crate) struct DecimalOperationTestCase { + pub(crate) description: &'static str, + pub(crate) result: PyDecimal, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct AllocationTestCase { + pub(crate) description: &'static str, + pub(crate) concurrent_queries: u64, + pub(crate) expected_seconds: &'static str, + pub(crate) expected_usd: &'static str, +} diff --git a/crates/sqlbuild-core/src/cost_allocation/types.rs b/crates/sqlbuild-core/src/cost_allocation/types.rs new file mode 100644 index 0000000000..2936fd82bc --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/types.rs @@ -0,0 +1,6 @@ +//! Grouping keys of run-cost allocation. + +use crate::cost_allocation::models::WarehouseKey; + +/// Resource type, resource name and warehouse of one priced resource. +pub(crate) type ResourceKey = (String, String, WarehouseKey); diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index a363d58d5c..2e9cf4e654 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -10,6 +10,7 @@ pub mod cli_text; pub mod clone_text; pub mod constants; pub mod contract_command; +pub mod cost_allocation; pub mod cost_command; pub mod cost_report; pub mod dbt_cli; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 5a4d0ed19c..c7b6027704 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -20,7 +20,7 @@ const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const AUDIT_PLAN_MODULE: &str = "sqlbuild._native"; -const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; +const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const PROJECTION_MODULE: &str = "sqlbuild.executor.auditing.main._current_audit_result_projection"; const EXECUTION_COMMAND: &str = "sqb audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 9ce0cd9904..a35915276a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -27,7 +27,7 @@ const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; -const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; +const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index b49ba9c7b5..aa0308a836 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -8,10 +8,10 @@ use crate::bindings::_helpers::cli::{ }; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, - cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, - format_command, freshness, init_command, janitor, load, plan_command, presentation_text, - query_command, result_output, scenario_command, seed_command, sql_test_command, - workspace_commands, + cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, dbt_interop, debug_command, + diff_report, format_command, freshness, init_command, janitor, load, plan_command, + presentation_text, query_command, result_output, scenario_command, seed_command, + sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -30,6 +30,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { load::registration::register(module)?; result_output::registration::register(module)?; cost_report::registration::register(module)?; + cost_telemetry::registration::register(module)?; debug_command::registration::register(module)?; dbt_events::registration::register(module)?; dbt_interop::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs index 22c3e7f7b1..1a6c91e380 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs @@ -109,6 +109,11 @@ impl<'py> KeywordArguments<'py> { Ok(Self { values }) } + /// Whether the argument `name` was passed. + pub(crate) fn contains(&self, name: &str) -> PyResult { + self.values.contains(name) + } + /// The argument `name`, or `None` when it was not passed. pub(crate) fn value(&self, name: &str) -> PyResult> { let py: Python<'py> = self.values.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs new file mode 100644 index 0000000000..99f5bc2578 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs @@ -0,0 +1,228 @@ +//! `allocate_run_cost`: price one run's share of observed Snowflake warehouse busy time. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_allocation::main::allocate_run_cost::allocate_run_cost as allocate; +use sqlbuild_core::cost_allocation::models::{ + AllocationStatus, CostQuery, PyDecimal, ResourceAllocation, RunAllocation, WarehouseKey, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cost_telemetry::values::{ + COST_MODELS, COST_TYPES, instant_micros, native_decimal, python_decimal, +}; + +const PENDING_MESSAGE: &str = "Snowflake query history is not complete yet."; + +/// One `QueryCostObservation` as a native query, in the run when its run ID matches. +pub(crate) fn cost_query(observation: &Bound<'_, PyAny>, run_id: &str) -> PyResult { + let text = |name: &str| -> PyResult> { observation.getattr(name)?.extract() }; + Ok(CostQuery { + query_id: observation.getattr("query_id")?.extract()?, + warehouse: WarehouseKey { + name: observation.getattr("warehouse_name")?.extract()?, + size: observation.getattr("warehouse_size")?.extract()?, + kind: text("warehouse_type")?, + cluster: observation.getattr("cluster_number")?.extract()?, + }, + started_micros: instant_micros(&observation.getattr("started_at")?)?, + completed_micros: instant_micros(&observation.getattr("completed_at")?)?, + in_run: text("run_id")?.as_deref() == Some(run_id), + resource_type: text("resource_type")?, + resource_name: text("resource_name")?, + bytes_scanned: observation.getattr("bytes_scanned")?.extract()?, + }) +} + +/// The `CostStatus` member of a native allocation status. +pub(crate) fn status_member(status: AllocationStatus) -> &'static str { + match status { + AllocationStatus::Pending => "PENDING", + AllocationStatus::Partial => "PARTIAL", + AllocationStatus::Complete => "COMPLETE", + } +} + +/// Summary fields that collection derives on top of an allocation. +pub(crate) struct SummaryState { + pub(crate) status: &'static str, + pub(crate) message: Option, + pub(crate) limitations: Vec, + /// Expected, observed, missing and skipped statement counts. + pub(crate) counts: [usize; 4], +} + +fn resource_cost<'py>( + py: Python<'py>, + resource: &ResourceAllocation, +) -> PyResult> { + let decimal = |value: &PyDecimal| python_decimal(py, value); + call_with( + &imported(py, COST_MODELS, "ResourceCost")?, + &[ + ( + "resource_type", + resource.resource_type.clone().into_bound_py_any(py)?, + ), + ( + "resource_name", + resource.resource_name.clone().into_bound_py_any(py)?, + ), + ( + "warehouse_name", + resource.warehouse.name.clone().into_bound_py_any(py)?, + ), + ( + "warehouse_size", + resource.warehouse.size.clone().into_bound_py_any(py)?, + ), + ( + "warehouse_type", + resource.warehouse.kind.clone().into_bound_py_any(py)?, + ), + ( + "cluster_number", + resource.warehouse.cluster.into_bound_py_any(py)?, + ), + ("query_count", resource.query_count.into_bound_py_any(py)?), + ("attributed_seconds", decimal(&resource.attributed_seconds)?), + ( + "bytes_scanned", + resource.bytes_scanned.into_bound_py_any(py)?, + ), + ( + "estimated_compute_credits", + decimal(&resource.estimated_compute_credits)?, + ), + ("estimated_usd", decimal(&resource.estimated_usd)?), + ], + ) +} + +/// A `RunCostSummary`; without an allocation its totals keep their defaults. +pub(crate) fn summary_object<'py>( + allocation: Option<&RunAllocation>, + state: &SummaryState, + (usd_per_credit, rate_source): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = usd_per_credit.py(); + let [expected, observed, missing, skipped] = state.counts; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "status", + imported(py, COST_TYPES, "CostStatus")?.getattr(state.status)?, + ), + ("usd_per_credit", usd_per_credit.clone()), + ("rate_source", rate_source.clone()), + ("message", state.message.clone().into_bound_py_any(py)?), + ( + "limitations", + PyTuple::new(py, &state.limitations)?.into_any(), + ), + ("expected_statement_count", expected.into_bound_py_any(py)?), + ("observed_statement_count", observed.into_bound_py_any(py)?), + ("missing_statement_count", missing.into_bound_py_any(py)?), + ("skipped_statement_count", skipped.into_bound_py_any(py)?), + ]; + if let Some(allocation) = allocation { + let resources: Vec> = allocation + .resources + .iter() + .map(|resource| resource_cost(py, resource)) + .collect::>()?; + fields.extend([ + ("resources", PyTuple::new(py, resources)?.into_any()), + ("query_count", allocation.query_count.into_bound_py_any(py)?), + ( + "attributed_seconds", + python_decimal(py, &allocation.attributed_seconds)?, + ), + ( + "bytes_scanned", + allocation.bytes_scanned.into_bound_py_any(py)?, + ), + ( + "estimated_compute_credits", + python_decimal(py, &allocation.estimated_compute_credits)?, + ), + ( + "estimated_usd", + python_decimal(py, &allocation.estimated_usd)?, + ), + ]); + } + call_with(&imported(py, COST_MODELS, "RunCostSummary")?, &fields) +} + +/// The allocation that prices queries, or `None` while the run's history is pending. +pub(crate) fn priced(allocation: RunAllocation) -> (Option, SummaryState) { + let pending: bool = allocation.status == AllocationStatus::Pending; + let state: SummaryState = SummaryState { + status: status_member(allocation.status), + message: pending.then(|| PENDING_MESSAGE.to_owned()), + limitations: allocation.limitations.clone(), + counts: [0; 4], + }; + ((!pending).then_some(allocation), state) +} + +/// Native queries and the allocation of one run over observations. +pub(crate) fn allocate_observations( + observations: &[Bound<'_, PyAny>], + (run_id, usd_per_credit): (&str, &Bound<'_, PyAny>), + result_limit_reached: bool, +) -> PyResult { + let queries: Vec = observations + .iter() + .map(|observation| cost_query(observation, run_id)) + .collect::>()?; + Ok(allocate( + &queries, + &native_decimal(usd_per_credit)?, + result_limit_reached, + )) +} + +/// Concurrency-safe Snowflake busy-compute attribution for one run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn allocate_run_cost<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "allocate_run_cost", + options, + ( + &["observations", "run_id", "usd_per_credit", "rate_source"], + &["result_limit_reached"], + ), + )?; + let observations: Vec> = arguments + .value("observations")? + .try_iter()? + .collect::>()?; + let run_id: String = arguments.value("run_id")?.extract()?; + let usd_per_credit: Bound<'py, PyAny> = arguments.value("usd_per_credit")?; + let allocation: RunAllocation = allocate_observations( + &observations, + (&run_id, &usd_per_credit), + arguments.value("result_limit_reached")?.is_truthy()?, + )?; + let (allocation, state) = priced(allocation); + summary_object( + allocation.as_ref(), + &state, + (&usd_per_credit, &arguments.value("rate_source")?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(allocate_run_cost, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs new file mode 100644 index 0000000000..01c70906eb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs @@ -0,0 +1,189 @@ +//! `CostContext`: the per-resource cost attribution context, held in a Python `ContextVar`. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cost_telemetry::values::{COST_MODELS, process_object}; + +const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_cost_resource_context"; +const DEFAULT_PHASE: &str = "execute"; +const DEFAULT_ATTEMPT: i64 = 1; + +static CURRENT: PyOnceLock> = PyOnceLock::new(); + +fn current_variable(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + process_object(&CURRENT, py, || { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")?.call((CONTEXT_VARIABLE_NAME,), Some(&options)) + }) +} + +/// The active `CostResourceContext`, or `None`. +pub(crate) fn current_context(py: Python<'_>) -> PyResult> { + current_variable(py)?.call_method0("get") +} + +/// A `with` scope that makes one resource context current, or does nothing without one. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct CostScope { + context: Option>, + token: Option>, +} + +#[pymethods] +impl CostScope { + fn __enter__(&mut self, py: Python<'_>) -> PyResult<()> { + if let Some(context) = &self.context { + self.token = Some( + current_variable(py)? + .call_method1("set", (context.bind(py),))? + .unbind(), + ); + } + Ok(()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&mut self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { + if let Some(token) = self.token.take() { + current_variable(py)?.call_method1("reset", (token.bind(py),))?; + } + Ok(()) + } +} + +fn or_current<'py>( + value: Bound<'py, PyAny>, + current: &Bound<'py, PyAny>, + field: &str, +) -> PyResult> { + if !value.is_none() || current.is_none() { + return Ok(value); + } + current.getattr(field) +} + +fn scope_of(context: Bound<'_, PyAny>) -> CostScope { + CostScope { + context: Some(context.unbind()), + token: None, + } +} + +/// Run-cost execution context entry points. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CostContext; + +#[pymethods] +impl CostContext { + #[staticmethod] + fn current(py: Python<'_>) -> PyResult> { + current_context(py) + } + + #[staticmethod] + #[pyo3(signature = (**options))] + fn scope(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "scope", + options, + ( + &["run_id", "resource_type", "resource_name"], + &["ledger_path", "phase", "attempt", "on_statement_complete"], + ), + )?; + let current: Bound<'_, PyAny> = current_context(py)?; + let defaulted = |name: &str| -> PyResult> { + if arguments.contains(name)? { + return arguments.value(name); + } + match name { + "attempt" => DEFAULT_ATTEMPT.into_bound_py_any(py), + _ => DEFAULT_PHASE.into_bound_py_any(py), + } + }; + let context: Bound<'_, PyAny> = call_with( + &imported(py, COST_MODELS, "CostResourceContext")?, + &[ + ("run_id", arguments.value("run_id")?), + ("resource_type", arguments.value("resource_type")?), + ("resource_name", arguments.value("resource_name")?), + ( + "ledger_path", + or_current(arguments.value("ledger_path")?, ¤t, "ledger_path")?, + ), + ("phase", defaulted("phase")?), + ("attempt", defaulted("attempt")?), + ( + "on_statement_complete", + or_current( + arguments.value("on_statement_complete")?, + ¤t, + "on_statement_complete", + )?, + ), + ], + )?; + Ok(scope_of(context)) + } + + #[staticmethod] + #[pyo3(signature = (**options))] + fn resource_scope(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "resource_scope", + options, + (&["resource_type", "resource_name"], &["phase", "attempt"]), + )?; + let current: Bound<'_, PyAny> = current_context(py)?; + if current.is_none() { + return Ok(CostScope { + context: None, + token: None, + }); + } + let phase: Bound<'_, PyAny> = arguments.value("phase")?; + let attempt: Bound<'_, PyAny> = if arguments.contains("attempt")? { + arguments.value("attempt")? + } else { + DEFAULT_ATTEMPT.into_bound_py_any(py)? + }; + let context: Bound<'_, PyAny> = call_with( + &imported(py, COST_MODELS, "CostResourceContext")?, + &[ + ("run_id", current.getattr("run_id")?), + ("resource_type", arguments.value("resource_type")?), + ("resource_name", arguments.value("resource_name")?), + ("ledger_path", current.getattr("ledger_path")?), + ( + "phase", + if phase.is_none() { + current.getattr("phase")? + } else { + phase + }, + ), + ("attempt", attempt), + ( + "on_statement_complete", + current.getattr("on_statement_complete")?, + ), + ], + )?; + Ok(scope_of(context)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs new file mode 100644 index 0000000000..db37db2b86 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -0,0 +1,268 @@ +//! The best-effort, append-only statement ledger that cost collection reads back. + +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBytes, PyDict, PyString, PyTuple, PyTypeMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::cost_telemetry::values::{ + COST_CONSTANTS, COST_EXCEPTIONS, COST_MODELS, process_object, +}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; + +static LEDGER_LOCK: PyOnceLock> = PyOnceLock::new(); + +/// The process-wide `threading.Lock` that orders ledger appends and snapshot reads. +fn ledger_lock(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + process_object(&LEDGER_LOCK, py, || { + imported(py, "threading", "Lock")?.call0() + }) +} + +fn health(py: Python<'_>) -> PyResult> { + imported(py, COST_CONSTANTS, "COST_TELEMETRY_HEALTH") +} + +fn mark_failure(run_id: &Bound<'_, PyAny>, error: &Bound<'_, PyAny>) -> PyResult<()> { + call_with( + &health(run_id.py())?.getattr("mark_ledger_failure")?, + &[("run_id", run_id.clone()), ("error", error.clone())], + )?; + Ok(()) +} + +fn optional_text(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, JsonValue::String) +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(JsonValue::String(value.extract()?)) +} + +/// The ledger line of one statement: compact, key-sorted JSON. +fn ledger_line(arguments: &KeywordArguments<'_>, context: &Bound<'_, PyAny>) -> PyResult { + let query_id: Bound<'_, PyAny> = arguments.value("query_id")?; + let query_id: Option = + if query_id.is_instance_of::() && query_id.is_truthy()? { + Some(query_id.extract()?) + } else { + None + }; + let sql: String = arguments.value("sql")?.extract()?; + let error: Bound<'_, PyAny> = arguments.value("error")?; + let error_type: Option = if error.is_none() { + None + } else { + Some(error.get_type().name()?.extract()?) + }; + let attempt: String = context.getattr("attempt")?.extract::()?.to_string(); + let isoformat = |name: &str| -> PyResult { + text(&arguments.value(name)?.call_method0("isoformat")?) + }; + let mut fields: Vec<(String, JsonValue)> = vec![ + ("run_id".to_owned(), text(&context.getattr("run_id")?)?), + ( + "statement_id".to_owned(), + text(&arguments.value("statement_id")?)?, + ), + ( + "resource_type".to_owned(), + text(&context.getattr("resource_type")?)?, + ), + ( + "resource_name".to_owned(), + text(&context.getattr("resource_name")?)?, + ), + ("phase".to_owned(), text(&context.getattr("phase")?)?), + ( + "attempt".to_owned(), + JsonValue::Integer(JsonInteger::parse(&attempt).ok_or_else(|| { + PyValueError::new_err("statement ledger attempt must be an integer") + })?), + ), + ("query_id".to_owned(), optional_text(query_id)), + ("status".to_owned(), text(&arguments.value("status")?)?), + ("started_at".to_owned(), isoformat("started_at")?), + ("completed_at".to_owned(), isoformat("completed_at")?), + ( + "sql_sha256".to_owned(), + JsonValue::String(sha256_hex(sql.as_bytes())), + ), + ("error_type".to_owned(), optional_text(error_type)), + ]; + fields.sort_by(|left, right| left.0.cmp(&right.0)); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + dumps(&JsonValue::Object(fields), &JsonDialect::Stdlib(options)) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +fn append_line(path: &Path, line: &str) -> std::io::Result<()> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent)?; + } + let mut file = OpenOptions::new().append(true).create(true).open(path)?; + file.write_all(format!("{line}\n").as_bytes())?; + file.flush() +} + +/// Append statement identity without allowing telemetry I/O to affect execution. +#[pyfunction] +#[pyo3(signature = (**options))] +fn record_statement(py: Python<'_>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "record_statement", + options, + ( + &[ + "context", + "statement_id", + "sql", + "query_id", + "status", + "started_at", + "completed_at", + ], + &["error"], + ), + )?; + let context: Bound<'_, PyAny> = arguments.value("context")?; + let path: Bound<'_, PyAny> = context.getattr("ledger_path")?; + if path.is_none() { + return Ok(()); + } + let appended: PyResult<()> = (|| { + let line: String = ledger_line(&arguments, &context)?; + let path: PathBuf = path.extract()?; + within(ledger_lock(py)?, |_| { + append_line(&path, &line).map_err(PyErr::from) + }) + })(); + match appended { + Ok(()) => Ok(()), + Err(error) if error.is_instance_of::(py) => { + mark_failure(&context.getattr("run_id")?, error.value(py).as_any()) + } + Err(error) => Err(error), + } +} + +/// One ledger line as a `StatementLedgerEntry`, raising the error Python's parsing raises. +fn ledger_entry<'py>(line: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = line.py(); + let payload: Bound<'py, PyAny> = imported(py, "json", "loads")?.call1((line,))?; + let query_id: Bound<'py, PyAny> = payload.call_method1("get", ("query_id",))?; + let field = |name: &str| -> PyResult> { + Ok(payload.get_item(name)?.str()?.into_any()) + }; + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let parsed = |name: &str| -> PyResult> { + datetime.call_method1("fromisoformat", (field(name)?,)) + }; + let entry_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("statement_id", field("statement_id")?), + ("run_id", field("run_id")?), + ("resource_type", field("resource_type")?), + ("resource_name", field("resource_name")?), + ("phase", field("phase")?), + ( + "attempt", + py.get_type::() + .call1((payload.get_item("attempt")?,))?, + ), + ( + "query_id", + if query_id.is_instance_of::() && query_id.is_truthy()? { + query_id + } else { + py.None().into_bound(py) + }, + ), + ("status", field("status")?), + ("started_at", parsed("started_at")?), + ("completed_at", parsed("completed_at")?), + ]; + call_with( + &imported(py, COST_MODELS, "StatementLedgerEntry")?, + &entry_fields, + ) +} + +fn entry_error(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) + || error.is_instance_of::(py) + || error.is_instance_of::(py) +} + +/// Valid ledger entries of a stable snapshot, marking invalid lines and read failures. +pub(crate) fn read_ledger<'py>( + path: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = path.py(); + if !path.call_method0("is_file")?.is_truthy()? { + return Ok(Vec::new()); + } + let native_path: PathBuf = path.extract()?; + let read: std::io::Result> = within(ledger_lock(py)?, |_| Ok(fs::read(&native_path)))?; + let bytes: Vec = match read { + Ok(bytes) => bytes, + Err(error) => { + mark_failure(run_id, PyErr::from(error).value(py).as_any())?; + return Ok(Vec::new()); + } + }; + let content: Bound<'py, PyAny> = PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",))?; + let mut entries: Vec> = Vec::new(); + for line in content.call_method0("splitlines")?.try_iter()? { + match ledger_entry(&line?) { + Ok(entry) => entries.push(entry), + Err(error) if entry_error(py, &error) => { + let message: String = format!( + "invalid statement ledger entry ({})", + error.get_type(py).name()? + ); + let artifact_error: Bound<'py, PyAny> = + imported(py, COST_EXCEPTIONS, "CostArtifactError")?.call1((message,))?; + mark_failure(run_id, &artifact_error)?; + } + Err(error) => return Err(error), + } + } + Ok(entries) +} + +/// Read a stable snapshot of valid statement ledger entries. +#[pyfunction] +#[pyo3(signature = (*, path, run_id))] +fn read_statement_ledger<'py>( + path: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + PyTuple::new(path.py(), read_ledger(path, run_id)?) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(record_statement, module)?)?; + module.add_function(wrap_pyfunction!(read_statement_ledger, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs new file mode 100644 index 0000000000..4bc1ed6824 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod allocation; +pub(crate) mod context; +pub(crate) mod ledger; +pub(crate) mod registration; +pub(crate) mod snowflake; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs new file mode 100644 index 0000000000..20c54783fe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs @@ -0,0 +1,12 @@ +//! Register native run-cost allocation, Snowflake cost collection and statement telemetry. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cost_telemetry::{allocation, context, ledger, snowflake}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + allocation::register(module)?; + context::register(module)?; + ledger::register(module)?; + snowflake::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs new file mode 100644 index 0000000000..742f1c5f47 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -0,0 +1,823 @@ +//! Snowflake query-history cost collection for one run, attributed through the statement ledger. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use pyo3::exceptions::{PyTimeoutError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; +use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; +use sqlbuild_analysis::snowflake_cost::models::QueryTag; +use sqlbuild_core::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; +use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; +use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::cost_telemetry::allocation::{ + SummaryState, allocate_observations, priced, summary_object, +}; +use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; +use crate::bindings::_helpers::cost_telemetry::values::{COST_CONSTANTS, COST_MODELS}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; + +const DEADLINE_SECONDS: f64 = 10.0; +const QUERY_TIMEOUT_SECONDS: &str = "5"; +const RUNNING_STATUS: &str = "RUNNING"; +const HOOK_RESOURCE_TYPE: &str = "hook"; +const ROW_WIDTH: usize = 14; +const NO_WAREHOUSE_COMPUTE: &str = "no_warehouse_compute"; +const MISSING_WAREHOUSE_METADATA: &str = "missing_warehouse_metadata"; +const INCOMPLETE_HISTORY: &str = "incomplete_history"; +const DEADLINE_MESSAGE: &str = "Snowflake cost collection deadline exceeded"; +const VISIBLE_QUERY_LIMITATION: &str = + "Concurrency sharing includes only queries visible to the executing Snowflake role."; +const RESULT_LIMIT_LIMITATION: &str = "Snowflake query-history result limit was reached."; +const LEDGER_FAILURE_PREFIX: &str = "Statement-ledger persistence failed ("; +const PENDING_STATUS: &str = "PENDING"; +const PARTIAL_STATUS: &str = "PARTIAL"; +const COMPLETE_STATUS: &str = "COMPLETE"; +const UNAVAILABLE_STATUS: &str = "UNAVAILABLE"; + +/// The connection, window inputs and deadline every query-history read shares. +struct History<'py> { + connection: Bound<'py, PyAny>, + database: Option, + classification_query_ids: Vec, + deadline: f64, + result_limit: usize, +} + +/// One query-history snapshot: observations, observed IDs, skip reasons and saturation. +struct Observed<'py> { + observations: Vec>, + observed: HashSet, + skipped: Vec<(String, &'static str)>, + saturated: bool, +} + +fn monotonic(py: Python<'_>) -> PyResult { + imported(py, "time", "monotonic")?.call0()?.extract() +} + +fn timedelta<'py>(py: Python<'py>, (unit, amount): (&str, i64)) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item(unit, amount)?; + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options)) +} + +fn check_deadline(history: &History<'_>, py: Python<'_>) -> PyResult<()> { + if monotonic(py)? >= history.deadline { + return Err(PyTimeoutError::new_err(DEADLINE_MESSAGE)); + } + Ok(()) +} + +fn history_rows<'py>( + history: &History<'py>, + (start, end): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), +) -> PyResult>> { + let py: Python<'py> = start.py(); + check_deadline(history, py)?; + let start_text: String = start.call_method0("isoformat")?.extract()?; + let end_text: Option = match end { + Some(end) => Some(end.call_method0("isoformat")?.extract()?), + None => None, + }; + let sql: String = query_history_sql(&QueryHistoryWindow { + database: history.database.as_deref(), + start: &start_text, + end: end_text.as_deref(), + classification_query_ids: &history.classification_query_ids, + result_limit: history.result_limit, + }); + let parameters: Bound<'py, PyDict> = PyDict::new(py); + parameters.set_item("STATEMENT_TIMEOUT_IN_SECONDS", QUERY_TIMEOUT_SECONDS)?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("statement_params", parameters)?; + let cursor: Bound<'py, PyAny> = + history + .connection + .call_method("execute", (sql,), Some(&options))?; + check_deadline(history, py)?; + let rows: Vec> = cursor + .call_method0("fetchall")? + .try_iter()? + .map(|row| { + Ok(PyTuple::new(py, row?.try_iter()?.collect::>>()?)?.into_any()) + }) + .collect::>()?; + check_deadline(history, py)?; + Ok(rows) +} + +fn strings(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +fn query_id(row: &Bound<'_, PyAny>) -> PyResult { + row.get_item(0)?.str()?.extract() +} + +/// Rows keyed by query ID, later rows replacing earlier ones, in query-ID order. +fn merged_rows<'py>( + rows: impl Iterator>, +) -> PyResult>> { + let mut by_query_id: BTreeMap> = BTreeMap::new(); + for row in rows { + by_query_id.insert(query_id(&row)?, row); + } + Ok(by_query_id.into_values().collect()) +} + +/// End-time-bounded rows, halving the window while a read reaches the result limit. +fn bounded_rows<'py>( + history: &History<'py>, + (start, end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Vec>, bool)> { + let py: Python<'py> = start.py(); + let rows: Vec> = history_rows(history, (start, Some(end)))?; + if rows.len() < history.result_limit { + return Ok((rows, false)); + } + let width: Bound<'py, PyAny> = end.sub(start)?; + if width.le(timedelta(py, ("milliseconds", 1))?)? { + return Ok((rows, true)); + } + let midpoint: Bound<'py, PyAny> = start.add(width.div(2)?)?; + let (left, left_saturated) = bounded_rows(history, (start, &midpoint))?; + let (right, right_saturated) = bounded_rows(history, (&midpoint, end))?; + Ok(( + merged_rows(left.into_iter().chain(right))?, + left_saturated || right_saturated, + )) +} + +fn running(end_time: &Bound<'_, PyAny>, status: &Bound<'_, PyAny>) -> PyResult { + Ok(end_time.is_none() || status.str()?.extract::()?.to_uppercase() == RUNNING_STATUS) +} + +fn unpacked<'py>(row: &Bound<'py, PyAny>) -> PyResult>> { + let fields: Vec> = row.try_iter()?.collect::>()?; + match fields.len() { + ROW_WIDTH => Ok(fields), + width if width > ROW_WIDTH => Err(PyValueError::new_err(format!( + "too many values to unpack (expected {ROW_WIDTH})" + ))), + width => Err(PyValueError::new_err(format!( + "not enough values to unpack (expected {ROW_WIDTH}, got {width})" + ))), + } +} + +fn python_int<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let value: Bound<'py, PyAny> = if value.is_truthy()? { + value.clone() + } else { + 0_i64.into_bound_py_any(py)? + }; + py.get_type::().call1((value,)) +} + +fn optional_str<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + Ok(value.str()?.into_any()) +} + +/// A history row as an observation, or the reason it carries no attributable compute. +fn row_observation<'py>( + row: &Bound<'py, PyAny>, + require_complete: bool, +) -> PyResult, &'static str>> { + let py: Python<'py> = row.py(); + let fields: Vec> = unpacked(row)?; + let [ + query_id, + query_tag, + warehouse_name, + warehouse_size, + warehouse_type, + cluster_number, + start_time, + end_time, + execution_ms, + bytes_scanned, + execution_status, + query_type, + query_text, + observed_at, + ] = <[Bound<'py, PyAny>; ROW_WIDTH]>::try_from(fields) + .map_err(|_| PyValueError::new_err("query history row width changed"))?; + let mut milliseconds: Bound<'py, PyAny> = python_int(&execution_ms)?; + if milliseconds.lt(0)? { + milliseconds = 0_i64.into_bound_py_any(py)?; + } + let query_type: String = if query_type.is_truthy()? { + query_type.str()?.extract()? + } else { + String::new() + }; + let query_text: Option = if query_text.is_instance_of::() { + Some(query_text.extract()?) + } else { + None + }; + if metadata_only_statement(&query_type, query_text.as_deref()) { + return Ok(Err(NO_WAREHOUSE_COMPUTE)); + } + let running: bool = running(&end_time, &execution_status)?; + if require_complete && running { + return Ok(Err(INCOMPLETE_HISTORY)); + } + if milliseconds.eq(0)? || warehouse_name.is_none() || warehouse_size.is_none() { + return Ok(Err(MISSING_WAREHOUSE_METADATA)); + } + let tag: Option = if query_tag.is_instance_of::() { + sqlbuild_query_tag(&query_tag.extract::()?) + } else { + None + }; + let effective_end: Bound<'py, PyAny> = if running { observed_at } else { end_time }; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("milliseconds", &milliseconds)?; + let busy_start: Bound<'py, PyAny> = + effective_end.sub(imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?)?; + let active_start: Bound<'py, PyAny> = if busy_start.gt(&start_time)? { + busy_start + } else { + start_time + }; + if effective_end.le(&active_start)? { + return Ok(Err(NO_WAREHOUSE_COMPUTE)); + } + let tag: QueryTag = tag.unwrap_or_default(); + let observation: Bound<'py, PyAny> = call_with( + &imported(py, COST_MODELS, "QueryCostObservation")?, + &[ + ("query_id", query_id.str()?.into_any()), + ("warehouse_name", warehouse_name.str()?.into_any()), + ("warehouse_size", warehouse_size.str()?.into_any()), + ("warehouse_type", optional_str(&warehouse_type)?), + ( + "cluster_number", + if cluster_number.is_none() { + cluster_number + } else { + py.get_type::() + .call1((cluster_number,))? + }, + ), + ("started_at", active_start), + ("completed_at", effective_end), + ("execution_ms", milliseconds), + ("bytes_scanned", python_int(&bytes_scanned)?), + ("execution_status", execution_status.str()?.into_any()), + ("run_id", tag.run_id.into_bound_py_any(py)?), + ("resource_type", tag.resource_type.into_bound_py_any(py)?), + ("resource_name", tag.resource_name.into_bound_py_any(py)?), + ], + )?; + Ok(Ok(observation)) +} + +/// Observations of the run window: open-ended history, bounded windows when saturated. +fn query_observations<'py>( + history: &History<'py>, + (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + expected: &HashSet, +) -> PyResult> { + let py: Python<'py> = started_at.py(); + let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; + let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; + let open_rows: Vec> = history_rows(history, (&window_start, None))?; + let mut saturated: bool = open_rows.len() >= history.result_limit; + let rows: Vec> = if saturated { + let (bounded, bounded_saturated) = + bounded_rows(history, (&window_start, &completed_at.add(&skew)?))?; + let mut still_running: Vec> = Vec::new(); + for row in open_rows { + if running(&row.get_item(7)?, &row.get_item(10)?)? { + still_running.push(row); + } + } + saturated = bounded_saturated; + merged_rows(still_running.into_iter().chain(bounded))? + } else { + open_rows + }; + let observations: Bound<'py, PyList> = PyList::empty(py); + let mut observed: HashSet = HashSet::new(); + let mut skipped: Vec<(String, &'static str)> = Vec::new(); + for row in rows { + let id: String = query_id(&row)?; + observed.insert(id.clone()); + match row_observation(&row, expected.contains(&id))? { + Ok(observation) => observations.append(observation)?, + Err(reason) => match skipped.iter_mut().find(|(skipped_id, _)| *skipped_id == id) { + Some(entry) => entry.1 = reason, + None => skipped.push((id, reason)), + }, + } + } + let key: Bound<'py, PyAny> = + imported(py, "operator", "attrgetter")?.call1(("started_at", "query_id"))?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("key", key)?; + observations.call_method("sort", (), Some(&options))?; + Ok(Observed { + observations: observations.iter().collect(), + observed, + skipped, + saturated, + }) +} + +/// A collected summary: the allocation it prices, if any, and its derived fields. +struct Summary { + allocation: Option, + state: SummaryState, +} + +/// Sorted unique limitations after adding `added`. +fn with_limitations( + limitations: &[String], + added: impl IntoIterator, +) -> Vec { + let limitations: BTreeSet = limitations.iter().cloned().chain(added).collect(); + limitations.into_iter().collect() +} + +fn partial(message: &str) -> Summary { + Summary { + allocation: None, + state: SummaryState { + status: PARTIAL_STATUS, + message: Some(message.to_owned()), + limitations: Vec::new(), + counts: [0; 4], + }, + } +} + +fn with_ledger_failure(mut summary: Summary, failure: Option<&str>) -> Summary { + let Some(failure) = failure else { + return summary; + }; + summary.state.status = PARTIAL_STATUS; + summary.state.limitations = with_limitations( + &summary.state.limitations, + [format!("{LEDGER_FAILURE_PREFIX}{failure}).")], + ); + if summary.state.message.is_none() { + summary.state.message = Some("Cost telemetry was only partially persisted.".to_owned()); + } + summary +} + +/// Ledger expectations a summary is checked against. +struct Expected<'a> { + statement_count: usize, + query_ids: &'a HashSet, +} + +fn with_observation_gaps( + mut summary: Summary, + expected: &Expected<'_>, + observed: &Observed<'_>, +) -> Summary { + let relevant: Vec<&'static str> = observed + .skipped + .iter() + .filter(|(id, _)| expected.query_ids.contains(id)) + .map(|(_, reason)| *reason) + .collect(); + let missing_metadata: bool = relevant.contains(&MISSING_WAREHOUSE_METADATA); + let incomplete: bool = relevant.contains(&INCOMPLETE_HISTORY); + let mut added: Vec = Vec::new(); + if missing_metadata { + added.push( + "One or more run statements had missing Snowflake warehouse metadata.".to_owned(), + ); + } + if incomplete { + added.push("One or more run statements were still incomplete in query history.".to_owned()); + } + summary.state.limitations = with_limitations(&summary.state.limitations, added); + let query_count: usize = summary + .allocation + .as_ref() + .map_or(0, |allocation| allocation.query_count); + if missing_metadata || (incomplete && query_count > 0) { + summary.state.status = PARTIAL_STATUS; + } + let visible: usize = expected.query_ids.intersection(&observed.observed).count(); + summary.state.counts = [ + expected.statement_count, + visible, + expected.statement_count - visible, + relevant.len(), + ]; + summary +} + +fn incomplete_observation(expected: &HashSet, observed: &Observed<'_>) -> bool { + observed.skipped.iter().any(|(id, reason)| { + expected.contains(id) + && (*reason == MISSING_WAREHOUSE_METADATA || *reason == INCOMPLETE_HISTORY) + }) +} + +/// `replace(observation, run_id=..., resource_type=..., resource_name=...)` from the ledger entry. +fn ledger_attributed<'py>( + observation: &Bound<'py, PyAny>, + ledger: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = observation.py(); + let entry: Option> = + pyo3::types::PyDictMethods::get_item(ledger, observation.getattr("query_id")?)?; + let field = |name: &str| -> PyResult> { + match &entry { + Some(entry) => entry.getattr(name), + None => Ok(py.None().into_bound(py)), + } + }; + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("run_id", field("run_id")?)?; + changes.set_item("resource_type", field("resource_type")?)?; + changes.set_item("resource_name", field("resource_name")?)?; + imported(py, "dataclasses", "replace")?.call((observation,), Some(&changes)) +} + +/// Collection inputs read from the keyword arguments. +struct Collection<'py> { + run_id: Bound<'py, PyAny>, + started_at: Bound<'py, PyAny>, + completed_at: Bound<'py, PyAny>, + usd_per_credit: Bound<'py, PyAny>, + rate_source: Bound<'py, PyAny>, + attempts: i64, + retry_delay_seconds: f64, +} + +fn collect<'py>( + collection: &Collection<'py>, + mut history: History<'py>, + ledger_path: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = ledger_path.py(); + history.deadline = monotonic(py)? + DEADLINE_SECONDS; + let run_id: String = collection.run_id.extract()?; + let ledger: Vec> = read_ledger(ledger_path, &collection.run_id)?; + let failure: Option = call_with( + &imported(py, COST_CONSTANTS, "COST_TELEMETRY_HEALTH")? + .getattr("consume_ledger_failure")?, + &[("run_id", collection.run_id.clone())], + )? + .extract()?; + let by_query_id: Bound<'py, PyDict> = PyDict::new(py); + for entry in &ledger { + let id: Bound<'py, PyAny> = entry.getattr("query_id")?; + if !id.is_none() { + by_query_id.set_item(id, entry)?; + } + } + let mut expected_ids: HashSet = HashSet::new(); + let mut classification: Vec = Vec::new(); + for (id, entry) in pyo3::types::PyDictMethods::iter(&by_query_id) { + let id: String = id.extract()?; + if !entry.getattr("resource_type")?.eq(HOOK_RESOURCE_TYPE)? { + classification.push(id.clone()); + } + expected_ids.insert(id); + } + history.classification_query_ids = classification; + if !ledger.is_empty() && expected_ids.is_empty() { + let mut summary: Summary = with_ledger_failure( + partial("No Snowflake query IDs were available in the statement ledger."), + failure.as_deref(), + ); + summary.state.counts = [ledger.len(), 0, ledger.len(), 0]; + return Ok(summary); + } + let expected: Expected<'_> = Expected { + statement_count: ledger.len(), + query_ids: &expected_ids, + }; + let mut summary: Option = None; + let mut observed: Observed<'py> = Observed { + observations: Vec::new(), + observed: HashSet::new(), + skipped: Vec::new(), + saturated: false, + }; + for attempt in 0..collection.attempts { + observed = query_observations( + &history, + (&collection.started_at, &collection.completed_at), + &expected_ids, + )?; + let attributed: Vec> = observed + .observations + .iter() + .map(|observation| ledger_attributed(observation, &by_query_id)) + .collect::>()?; + let allocation: RunAllocation = allocate_observations( + &attributed, + (&run_id, &collection.usd_per_credit), + observed.saturated, + )?; + let (allocation, mut state) = priced(allocation); + state.limitations = + with_limitations(&state.limitations, [VISIBLE_QUERY_LIMITATION.to_owned()]); + let current: Summary = Summary { allocation, state }; + if expected_ids.is_subset(&observed.observed) + && !incomplete_observation(&expected_ids, &observed) + { + let mut complete: Summary = with_observation_gaps(current, &expected, &observed); + if complete.state.status == PENDING_STATUS { + complete.state.status = COMPLETE_STATUS; + complete.state.message = + Some("No cost-bearing Snowflake warehouse queries were observed.".to_owned()); + } + return Ok(with_ledger_failure(complete, failure.as_deref())); + } + summary = Some(current); + if attempt + 1 < collection.attempts { + if history.deadline - monotonic(py)? <= collection.retry_delay_seconds { + break; + } + imported(py, "time", "sleep")?.call1((collection.retry_delay_seconds,))?; + } + } + let mut summary: Summary = + summary.unwrap_or_else(|| partial("Snowflake query history could not be collected.")); + let missing: usize = expected_ids.difference(&observed.observed).count(); + let mut added: Vec = Vec::new(); + if missing > 0 { + added.push(format!( + "{missing} statement ledger query ID(s) were not visible in query history." + )); + } + if observed.saturated { + added.push(RESULT_LIMIT_LIMITATION.to_owned()); + } + summary.state.limitations = with_limitations(&summary.state.limitations, added); + let visible: bool = expected_ids + .intersection(&observed.observed) + .next() + .is_some(); + let now: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")? + .call_method1("now", (imported(py, DATETIME_MODULE, "UTC")?,))?; + let expired: bool = now + .sub(&collection.completed_at)? + .ge(timedelta(py, ("days", 7))?)?; + if expired && !visible { + summary.state.status = UNAVAILABLE_STATUS; + summary.state.message = + Some("Snowflake query history is no longer available for this run.".to_owned()); + } else if !visible && !observed.saturated { + summary.state.status = PENDING_STATUS; + summary.state.message = Some( + "Snowflake query history is not complete yet. Run `sqb cost latest` to refresh." + .to_owned(), + ); + } else { + summary.state.status = PARTIAL_STATUS; + if summary.state.message.is_none() { + summary.state.message = + Some("Snowflake query history was only partially available.".to_owned()); + } + } + let summary: Summary = with_observation_gaps(summary, &expected, &observed); + Ok(with_ledger_failure(summary, failure.as_deref())) +} + +/// Collect visible history and attribute exact ledger query IDs to one run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn collect_snowflake_cost<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "collect_snowflake_cost", + options, + ( + &[ + "connection", + "run_id", + "started_at", + "completed_at", + "statement_ledger_path", + "usd_per_credit", + "rate_source", + ], + &[ + "database", + "attempts", + "retry_delay_seconds", + "result_limit", + ], + ), + )?; + let optional = |name: &str, default: Bound<'py, PyAny>| -> PyResult> { + Ok(if arguments.contains(name)? { + arguments.value(name)? + } else { + default + }) + }; + let collection: Collection<'py> = Collection { + run_id: arguments.value("run_id")?, + started_at: arguments.value("started_at")?, + completed_at: arguments.value("completed_at")?, + usd_per_credit: arguments.value("usd_per_credit")?, + rate_source: arguments.value("rate_source")?, + attempts: optional("attempts", 3_i64.into_bound_py_any(py)?)?.extract()?, + retry_delay_seconds: optional("retry_delay_seconds", 1.0_f64.into_bound_py_any(py)?)? + .extract()?, + }; + let history: History<'py> = History { + connection: arguments.value("connection")?, + database: arguments.value("database")?.extract()?, + classification_query_ids: Vec::new(), + deadline: 0.0, + result_limit: optional( + "result_limit", + QUERY_HISTORY_RESULT_LIMIT.into_bound_py_any(py)?, + )? + .extract()?, + }; + let summary: Summary = collect( + &collection, + history, + &arguments.value("statement_ledger_path")?, + )?; + summary_object( + summary.allocation.as_ref(), + &summary.state, + (&collection.usd_per_credit, &collection.rate_source), + ) +} + +/// Query-history observations of one window, as cost collection reads them. +#[pyfunction] +#[pyo3(signature = (**options))] +fn snowflake_query_observations<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "snowflake_query_observations", + options, + ( + &[ + "connection", + "started_at", + "completed_at", + "expected_query_ids", + "classification_query_ids", + "deadline", + ], + &["database", "result_limit"], + ), + )?; + let result_limit: Bound<'py, PyAny> = arguments.value("result_limit")?; + let history: History<'py> = History { + connection: arguments.value("connection")?, + database: arguments.value("database")?.extract()?, + classification_query_ids: strings(&arguments.value("classification_query_ids")?)?, + deadline: arguments.value("deadline")?.extract()?, + result_limit: if result_limit.is_none() { + QUERY_HISTORY_RESULT_LIMIT + } else { + result_limit.extract()? + }, + }; + let expected: HashSet = strings(&arguments.value("expected_query_ids")?)? + .into_iter() + .collect(); + let observed: Observed<'py> = query_observations( + &history, + ( + &arguments.value("started_at")?, + &arguments.value("completed_at")?, + ), + &expected, + )?; + let skipped: Bound<'py, PyDict> = PyDict::new(py); + for (id, reason) in &observed.skipped { + skipped.set_item(id, reason)?; + } + PyTuple::new( + py, + [ + PyTuple::new(py, &observed.observations)?.into_any(), + pyo3::types::PyFrozenSet::new(py, &observed.observed)?.into_any(), + skipped.into_any(), + observed.saturated.into_bound_py_any(py)?, + ], + ) +} + +/// The query-history SQL of one window; an absent end reads every query ending after the start. +#[pyfunction] +#[pyo3(signature = (**options))] +fn render_snowflake_query_history_sql( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "render_snowflake_query_history_sql", + options, + ( + &["started_at", "classification_query_ids"], + &["database", "completed_at", "result_limit"], + ), + )?; + let start: String = arguments + .value("started_at")? + .call_method0("isoformat")? + .extract()?; + let completed_at: Bound<'_, PyAny> = arguments.value("completed_at")?; + let end: Option = if completed_at.is_none() { + None + } else { + Some(completed_at.call_method0("isoformat")?.extract()?) + }; + let result_limit: Bound<'_, PyAny> = arguments.value("result_limit")?; + let database: Option = arguments.value("database")?.extract()?; + Ok(query_history_sql(&QueryHistoryWindow { + database: database.as_deref(), + start: &start, + end: end.as_deref(), + classification_query_ids: &strings(&arguments.value("classification_query_ids")?)?, + result_limit: if result_limit.is_none() { + QUERY_HISTORY_RESULT_LIMIT + } else { + result_limit.extract()? + }, + })) +} + +/// Whether Snowflake runs a statement of this type and text without warehouse compute. +#[pyfunction] +#[pyo3(signature = (*, query_type, query_text))] +fn snowflake_statement_is_metadata_only( + query_type: &Bound<'_, PyAny>, + query_text: &Bound<'_, PyAny>, +) -> PyResult { + let query_type: String = if query_type.is_truthy()? { + query_type.str()?.extract()? + } else { + String::new() + }; + let query_text: Option = if query_text.is_instance_of::() { + Some(query_text.extract()?) + } else { + None + }; + Ok(metadata_only_statement(&query_type, query_text.as_deref())) +} + +/// The run attribution of a SQLBuild query tag, or `None` for any other tag. +#[pyfunction] +fn parse_sqlbuild_query_tag<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = value.py(); + if !value.is_instance_of::() { + return Ok(None); + } + let Some(tag) = sqlbuild_query_tag(&value.extract::()?) else { + return Ok(None); + }; + let attribution: Bound<'py, PyDict> = PyDict::new(py); + attribution.set_item("run_id", tag.run_id)?; + attribution.set_item("resource_type", tag.resource_type)?; + attribution.set_item("resource_name", tag.resource_name)?; + Ok(Some(attribution)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(collect_snowflake_cost, module)?)?; + module.add_function(wrap_pyfunction!(snowflake_query_observations, module)?)?; + module.add_function(wrap_pyfunction!( + render_snowflake_query_history_sql, + module + )?)?; + module.add_function(wrap_pyfunction!( + snowflake_statement_is_metadata_only, + module + )?)?; + module.add_function(wrap_pyfunction!(parse_sqlbuild_query_tag, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs new file mode 100644 index 0000000000..b3a3a431be --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs @@ -0,0 +1,81 @@ +//! Python `Decimal` and `datetime` values as exact native decimals and microsecond instants. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_core::cost_allocation::models::PyDecimal; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::state_rows::timestamps::offset_aware; + +pub(crate) const COST_MODELS: &str = "sqlbuild.cost.models"; +pub(crate) const COST_TYPES: &str = "sqlbuild.cost.types"; +pub(crate) const COST_CONSTANTS: &str = "sqlbuild.cost.constants"; +pub(crate) const COST_EXCEPTIONS: &str = "sqlbuild.cost.exceptions"; +const DECIMAL_MODULE: &str = "decimal"; + +/// A finite non-negative `Decimal` as its exact digits and exponent. +pub(crate) fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { + let (sign, digits, exponent): (u8, Vec, Bound<'_, PyAny>) = + value.call_method0("as_tuple")?.extract()?; + let Ok(exponent) = exponent.extract::() else { + return Err(PyValueError::new_err("cost decimals must be finite")); + }; + if sign != 0 && digits.iter().any(|digit| *digit != 0) { + return Err(PyValueError::new_err("cost decimals must not be negative")); + } + let leading: usize = digits.iter().take_while(|digit| **digit == 0).count(); + let digits: Vec = if leading == digits.len() { + vec![0] + } else { + digits[leading..].to_vec() + }; + Ok(PyDecimal { digits, exponent }) +} + +/// `Decimal((0, digits, exponent))`. +pub(crate) fn python_decimal<'py>( + py: Python<'py>, + value: &PyDecimal, +) -> PyResult> { + let parts: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + 0_i64.into_bound_py_any(py)?, + PyTuple::new(py, &value.digits)?.into_any(), + value.exponent.into_bound_py_any(py)?, + ], + )?; + imported(py, DECIMAL_MODULE, "Decimal")?.call1((parts,)) +} + +/// Microseconds since the epoch: UTC for aware values, the wall clock for naive ones. +pub(crate) fn instant_micros(value: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value.py(); + let datetime: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + let epoch: Bound<'_, PyAny> = if offset_aware(value)? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("tzinfo", imported(py, DATETIME_MODULE, "UTC")?)?; + datetime.call((1970, 1, 1), Some(&options))? + } else { + datetime.call1((1970, 1, 1))? + }; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("microseconds", 1)?; + let micro: Bound<'_, PyAny> = + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options))?; + value.sub(epoch)?.floor_div(micro)?.extract() +} + +/// The process-wide Python object in `cell`, built by `build` on first use. +pub(crate) fn process_object<'py>( + cell: &'static PyOnceLock>, + py: Python<'py>, + build: impl FnOnce() -> PyResult>, +) -> PyResult<&'py Bound<'py, PyAny>> { + cell.get_or_try_init(py, || Ok::, PyErr>(build()?.unbind())) + .map(|object| object.bind(py)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 7499911e21..33931214d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod contract_command; pub(crate) mod contracts; pub(crate) mod cost_command; pub(crate) mod cost_report; +pub(crate) mod cost_telemetry; pub(crate) mod cursor; pub(crate) mod dag_artifact; pub(crate) mod dbt_command; diff --git a/fensu.toml b/fensu.toml index 9601450583..e54c77f4fd 100644 --- a/fensu.toml +++ b/fensu.toml @@ -68,6 +68,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/source_freshness/__init__.py" reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types, constants and errors they build and raise." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cost/constants.py" +reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types, constants, errors and the telemetry health tracker they use." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index bdf5fc8a58..7de02ac944 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 16308 +remaining_lines = 15231 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index beb3e065ab..27050ee081 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,6 +5,7 @@ from concurrent.futures import Future from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime +from decimal import Decimal from pathlib import Path, PurePath from typing import Any, Literal, TextIO, TypedDict @@ -367,7 +368,14 @@ from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestPlanningRequest, ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow -from sqlbuild.cost.models import CostRunRecord, StatementExecutionTelemetry +from sqlbuild.cost.models import ( + CostResourceContext, + CostRunRecord, + QueryCostObservation, + RunCostSummary, + StatementExecutionTelemetry, + StatementLedgerEntry, +) from sqlbuild.cursor_algebra.models import ( AlignedInterval, DateValue, @@ -6099,3 +6107,86 @@ def list_source_candidates( database: str | None, entries: tuple[SourceEntry, ...], ) -> tuple[RelationInfo, ...]: ... + +class CostScope: + def __enter__(self) -> None: ... + def __exit__(self, *exception: object) -> None: ... + +class CostContext: + @staticmethod + def current() -> CostResourceContext | None: ... + @staticmethod + def scope( + *, + run_id: str, + resource_type: str, + resource_name: str, + ledger_path: Path | None = None, + phase: str = "execute", + attempt: int = 1, + on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None, + ) -> CostScope: ... + @staticmethod + def resource_scope( + *, + resource_type: str, + resource_name: str, + phase: str | None = None, + attempt: int = 1, + ) -> CostScope: ... + +def allocate_run_cost( + *, + observations: Iterable[QueryCostObservation], + run_id: str, + usd_per_credit: Decimal, + rate_source: str, + result_limit_reached: bool = False, +) -> RunCostSummary: ... +def record_statement( + *, + context: CostResourceContext, + statement_id: str, + sql: str, + query_id: object, + status: str, + started_at: datetime, + completed_at: datetime, + error: BaseException | None = None, +) -> None: ... +def read_statement_ledger(*, path: Path, run_id: str) -> tuple[StatementLedgerEntry, ...]: ... +def collect_snowflake_cost( + *, + connection: Any, + run_id: str, + started_at: datetime, + completed_at: datetime, + statement_ledger_path: Path, + usd_per_credit: Decimal, + rate_source: str, + database: str | None = None, + attempts: int = 3, + retry_delay_seconds: float = 1.0, + result_limit: int = 10000, +) -> RunCostSummary: ... +def snowflake_query_observations( + *, + connection: Any, + started_at: datetime, + completed_at: datetime, + expected_query_ids: Iterable[str], + classification_query_ids: Iterable[str], + deadline: float, + database: str | None = None, + result_limit: int | None = None, +) -> tuple[tuple[QueryCostObservation, ...], frozenset[str], dict[str, str], bool]: ... +def render_snowflake_query_history_sql( + *, + started_at: datetime, + classification_query_ids: Iterable[str], + database: str | None = None, + completed_at: datetime | None = None, + result_limit: int = 10000, +) -> str: ... +def snowflake_statement_is_metadata_only(*, query_type: object, query_text: object) -> bool: ... +def parse_sqlbuild_query_tag(value: object) -> dict[str, str | None] | None: ... diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 6938184a72..c20afa3e80 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import collect_snowflake_cost from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -152,7 +153,6 @@ from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.cost.main.collection import collect_snowflake_cost from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py index 712fe1b1f6..8b5d0fd682 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py @@ -12,8 +12,7 @@ from typing import Any from uuid import uuid4 -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.classes.statement_ledger import StatementLedger +from sqlbuild._native import CostContext, record_statement from sqlbuild.cost.constants import SQLBUILD_QUERY_TAG_APP from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry from sqlbuild.runtime.execution_limits.main.current_execution_deadline import ( @@ -224,7 +223,7 @@ def _record_failure( cursor=self.raw_cursor, previous_query_id=previous_query_id ) if context is not None: - StatementLedger.record( + record_statement( context=context, statement_id=statement_id, sql=sql, @@ -258,7 +257,7 @@ def _record_success( cursor=self.raw_cursor, previous_query_id=previous_query_id ) if context is not None: - StatementLedger.record( + record_statement( context=context, statement_id=statement_id, sql=sql, diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 354e97f2ce..3741f1a35f 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -5,7 +5,7 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import write_source_freshness_records +from sqlbuild._native import CostContext, write_source_freshness_records from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.source_freshness.models import ( @@ -14,7 +14,6 @@ SourceFreshnessRecord, SourceFreshnessRenderers, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/cost/_helpers/allocation.py b/src/sqlbuild/cost/_helpers/allocation.py deleted file mode 100644 index 8729a369df..0000000000 --- a/src/sqlbuild/cost/_helpers/allocation.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Concurrency-safe Snowflake busy-compute attribution.""" - -from __future__ import annotations - -from collections import defaultdict -from datetime import datetime -from decimal import Decimal - -from sqlbuild.cost.constants import STANDARD_WAREHOUSE_TYPE -from sqlbuild.cost.models import QueryCostObservation, ResourceCost, RunCostSummary -from sqlbuild.cost.types import CostStatus - -_CREDITS_PER_HOUR: dict[str, Decimal] = { - "X-SMALL": Decimal(1), - "SMALL": Decimal(2), - "MEDIUM": Decimal(4), - "LARGE": Decimal(8), - "X-LARGE": Decimal(16), - "2X-LARGE": Decimal(32), - "3X-LARGE": Decimal(64), - "4X-LARGE": Decimal(128), - "5X-LARGE": Decimal(256), - "6X-LARGE": Decimal(512), -} - - -def allocate_run_cost( - *, - observations: tuple[QueryCostObservation, ...], - run_id: str, - usd_per_credit: Decimal, - rate_source: str, - result_limit_reached: bool = False, -) -> RunCostSummary: - run_queries: tuple[QueryCostObservation, ...] = tuple( - observation for observation in observations if observation.run_id == run_id - ) - if not run_queries: - return RunCostSummary( - status=CostStatus.PENDING, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message="Snowflake query history is not complete yet.", - ) - - attributed: dict[str, Decimal] = defaultdict(Decimal) - by_warehouse: dict[tuple[str, str, str | None, int | None], list[QueryCostObservation]] = ( - defaultdict(list) - ) - for observation in observations: - by_warehouse[ - ( - observation.warehouse_name, - observation.warehouse_size, - observation.warehouse_type, - observation.cluster_number, - ) - ].append(observation) - - for warehouse_queries in by_warehouse.values(): - boundaries: list[datetime] = _query_boundaries(queries=warehouse_queries) - for start, end in zip(boundaries, boundaries[1:], strict=False): - if end <= start: - continue - active: tuple[QueryCostObservation, ...] = tuple( - query - for query in warehouse_queries - if query.started_at < end and query.completed_at > start - ) - if not active: - continue - seconds: Decimal = Decimal(str((end - start).total_seconds())) / Decimal(len(active)) - for query in active: - if query.run_id == run_id: - attributed[query.query_id] += seconds - - grouped: dict[tuple[str, str, str, str, str | None, int | None], list[QueryCostObservation]] = ( - defaultdict(list) - ) - for query in run_queries: - grouped[ - ( - query.resource_type or "unknown", - query.resource_name or "unknown", - query.warehouse_name, - query.warehouse_size, - query.warehouse_type, - query.cluster_number, - ) - ].append(query) - - limitations: list[str] = [] - resources: list[ResourceCost] = [] - for ( - resource_type, - resource_name, - warehouse_name, - warehouse_size, - warehouse_type, - cluster_number, - ), queries in grouped.items(): - rate: Decimal | None = _CREDITS_PER_HOUR.get(warehouse_size.upper()) - if warehouse_type is None: - limitations.append(f"Unknown warehouse type for {warehouse_name}") - elif warehouse_type.upper() != STANDARD_WAREHOUSE_TYPE: - limitations.append(f"Unsupported Snowflake warehouse type: {warehouse_type}") - continue - if cluster_number is None: - limitations.append(f"Unknown cluster number for {warehouse_name}") - if rate is None: - limitations.append(f"Unknown Snowflake warehouse size: {warehouse_size}") - continue - seconds = sum((attributed[query.query_id] for query in queries), Decimal(0)) - credits: Decimal = seconds * rate / Decimal(3600) - resources.append( - ResourceCost( - resource_type=resource_type, - resource_name=resource_name, - warehouse_name=warehouse_name, - warehouse_size=warehouse_size, - warehouse_type=warehouse_type, - cluster_number=cluster_number, - query_count=len(queries), - attributed_seconds=seconds, - bytes_scanned=sum(query.bytes_scanned for query in queries), - estimated_compute_credits=credits, - estimated_usd=credits * usd_per_credit, - ) - ) - if result_limit_reached: - limitations.append("Snowflake query-history result limit was reached.") - resources.sort( - key=lambda item: ( - -item.estimated_usd, - item.resource_name, - item.resource_type, - item.warehouse_name, - item.warehouse_size, - item.warehouse_type or "", - -1 if item.cluster_number is None else item.cluster_number, - ) - ) - return RunCostSummary( - status=CostStatus.PARTIAL if limitations else CostStatus.COMPLETE, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - resources=tuple(resources), - query_count=len(run_queries), - attributed_seconds=sum((item.attributed_seconds for item in resources), Decimal(0)), - bytes_scanned=sum(item.bytes_scanned for item in resources), - estimated_compute_credits=sum( - (item.estimated_compute_credits for item in resources), Decimal(0) - ), - estimated_usd=sum((item.estimated_usd for item in resources), Decimal(0)), - limitations=tuple(sorted(set(limitations))), - ) - - -def _query_boundaries(*, queries: list[QueryCostObservation]) -> list[datetime]: - boundaries: set[datetime] = set() - for query in queries: - boundaries.add(query.started_at) - boundaries.add(query.completed_at) - return sorted(boundaries) diff --git a/src/sqlbuild/cost/_helpers/context.py b/src/sqlbuild/cost/_helpers/context.py deleted file mode 100644 index a3db6f4c2f..0000000000 --- a/src/sqlbuild/cost/_helpers/context.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Per-resource cost attribution context.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token -from pathlib import Path - -from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry - -_CURRENT: ContextVar[CostResourceContext | None] = ContextVar( - "sqlbuild_cost_resource_context", default=None -) - - -def current_cost_resource_context() -> CostResourceContext | None: - return _CURRENT.get() - - -@contextmanager -def cost_resource_context( - *, - run_id: str, - resource_type: str, - resource_name: str, - ledger_path: Path | None = None, - phase: str = "execute", - attempt: int = 1, - on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None, -) -> Iterator[None]: - token: Token[CostResourceContext | None] = _CURRENT.set( - CostResourceContext( - run_id=run_id, - resource_type=resource_type, - resource_name=resource_name, - ledger_path=ledger_path, - phase=phase, - attempt=attempt, - on_statement_complete=on_statement_complete, - ) - ) - try: - yield - finally: - _CURRENT.reset(token) diff --git a/src/sqlbuild/cost/_helpers/ledger.py b/src/sqlbuild/cost/_helpers/ledger.py deleted file mode 100644 index ef75a02edb..0000000000 --- a/src/sqlbuild/cost/_helpers/ledger.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Best-effort append-only statement telemetry persistence.""" - -from __future__ import annotations - -import hashlib -import json -import threading -from datetime import datetime -from pathlib import Path -from typing import Any - -from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH -from sqlbuild.cost.exceptions import CostArtifactError -from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry - -_LOCK: threading.Lock = threading.Lock() - - -def record_statement( - *, - context: CostResourceContext, - statement_id: str, - sql: str, - query_id: object, - status: str, - started_at: datetime, - completed_at: datetime, - error: Exception | None = None, -) -> None: - """Append statement identity without allowing telemetry I/O to affect execution.""" - - if context.ledger_path is None: - return - try: - _append_statement( - path=context.ledger_path, - payload={ - "run_id": context.run_id, - "statement_id": statement_id, - "resource_type": context.resource_type, - "resource_name": context.resource_name, - "phase": context.phase, - "attempt": context.attempt, - "query_id": query_id if isinstance(query_id, str) and query_id else None, - "status": status, - "started_at": started_at.isoformat(), - "completed_at": completed_at.isoformat(), - "sql_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(), - "error_type": None if error is None else type(error).__name__, - }, - ) - except Exception as error: - COST_TELEMETRY_HEALTH.mark_ledger_failure(run_id=context.run_id, error=error) - return - - -def read_statement_ledger(*, path: Path, run_id: str) -> tuple[StatementLedgerEntry, ...]: - """Read a stable snapshot of valid statement ledger entries.""" - - if not path.is_file(): - return () - entries: list[StatementLedgerEntry] = [] - with _LOCK: - try: - lines: list[str] = path.read_text(encoding="utf-8").splitlines() - except OSError as error: - COST_TELEMETRY_HEALTH.mark_ledger_failure(run_id=run_id, error=error) - return () - for line in lines: - try: - payload: dict[str, Any] = json.loads(line) - query_id_value: object = payload.get("query_id") - entries.append( - StatementLedgerEntry( - statement_id=str(payload["statement_id"]), - run_id=str(payload["run_id"]), - resource_type=str(payload["resource_type"]), - resource_name=str(payload["resource_name"]), - phase=str(payload["phase"]), - attempt=int(payload["attempt"]), - query_id=( - query_id_value - if isinstance(query_id_value, str) and query_id_value - else None - ), - status=str(payload["status"]), - started_at=datetime.fromisoformat(str(payload["started_at"])), - completed_at=datetime.fromisoformat(str(payload["completed_at"])), - ) - ) - except (KeyError, TypeError, ValueError, json.JSONDecodeError) as error: - COST_TELEMETRY_HEALTH.mark_ledger_failure( - run_id=run_id, - error=CostArtifactError(f"invalid statement ledger entry ({type(error).__name__})"), - ) - continue - return tuple(entries) - - -def _append_statement(*, path: Path, payload: dict[str, object]) -> None: - with _LOCK: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("a", encoding="utf-8") as file: - file.write(json.dumps(payload, separators=(",", ":"), sort_keys=True) + "\n") - file.flush() diff --git a/src/sqlbuild/cost/_helpers/snowflake.py b/src/sqlbuild/cost/_helpers/snowflake.py deleted file mode 100644 index a04833cc78..0000000000 --- a/src/sqlbuild/cost/_helpers/snowflake.py +++ /dev/null @@ -1,625 +0,0 @@ -"""Snowflake query-history cost collection.""" - -from __future__ import annotations - -import json -import time -from dataclasses import replace -from datetime import UTC, datetime, timedelta -from decimal import Decimal -from pathlib import Path -from typing import Any - -from polyglot_sql import CreateTable, CreateView, ParseError, parse_one - -from sqlbuild.cost._helpers.allocation import allocate_run_cost -from sqlbuild.cost._helpers.ledger import read_statement_ledger -from sqlbuild.cost.constants import ( - COST_TELEMETRY_HEALTH, - INCOMPLETE_HISTORY_REASON, - LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX, - MISSING_WAREHOUSE_METADATA_REASON, - NO_WAREHOUSE_COMPUTE_REASON, - RUNNING_EXECUTION_STATUS, - SQLBUILD_QUERY_TAG_APP, -) -from sqlbuild.cost.models import ( - QueryCostObservation, - RunCostSummary, - StatementLedgerEntry, -) -from sqlbuild.cost.types import CostStatus - -_RESULT_LIMIT: int = 10_000 -_MINIMUM_SPLIT_WINDOW: timedelta = timedelta(milliseconds=1) -_COLLECTION_DEADLINE_SECONDS: float = 10.0 -_QUERY_TIMEOUT_SECONDS: str = "5" -_QUERY_HISTORY_RETENTION: timedelta = timedelta(days=7) -_MAXIMUM_CLOCK_SKEW: timedelta = timedelta(minutes=10) -_USE_QUERY_TYPE: str = "USE" -_DROP_QUERY_TYPE: str = "DROP" -_CREATE_QUERY_TYPE: str = "CREATE" -_DYNAMIC_TABLE_MODIFIER: str = "DYNAMIC" -_HOOK_RESOURCE_TYPE: str = "hook" -_VISIBLE_QUERY_LIMITATION: str = ( - "Concurrency sharing includes only queries visible to the executing Snowflake role." -) -_INCOMPLETE_OBSERVATION_REASONS: frozenset[str] = frozenset( - {MISSING_WAREHOUSE_METADATA_REASON, INCOMPLETE_HISTORY_REASON} -) - - -def collect_snowflake_run_cost( - *, - connection: Any, - database: str | None = None, - run_id: str, - started_at: datetime, - completed_at: datetime, - statement_ledger_path: Path, - usd_per_credit: Decimal, - rate_source: str, - attempts: int = 3, - retry_delay_seconds: float = 1.0, -) -> RunCostSummary: - """Collect visible history and attribute exact ledger query IDs to one run.""" - - deadline: float = time.monotonic() + _COLLECTION_DEADLINE_SECONDS - ledger: tuple[StatementLedgerEntry, ...] = read_statement_ledger( - path=statement_ledger_path, - run_id=run_id, - ) - ledger_failure: str | None = COST_TELEMETRY_HEALTH.consume_ledger_failure(run_id=run_id) - ledger_by_query_id: dict[str, StatementLedgerEntry] = { - entry.query_id: entry for entry in ledger if entry.query_id is not None - } - expected_query_ids: frozenset[str] = frozenset(ledger_by_query_id) - classification_query_ids: frozenset[str] = frozenset( - query_id - for query_id, entry in ledger_by_query_id.items() - if entry.resource_type != _HOOK_RESOURCE_TYPE - ) - if ledger and not expected_query_ids: - no_query_id_summary: RunCostSummary = _apply_ledger_failure( - summary=_partial_summary( - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message="No Snowflake query IDs were available in the statement ledger.", - ), - ledger_failure=ledger_failure, - ) - return replace( - no_query_id_summary, - expected_statement_count=len(ledger), - missing_statement_count=len(ledger), - ) - - summary: RunCostSummary | None = None - observed_query_ids: frozenset[str] = frozenset() - skipped_reasons: dict[str, str] = {} - saturated = False - for attempt in range(attempts): - observations, observed_query_ids, skipped_reasons, saturated = _query_observations( - connection=connection, - database=database, - started_at=started_at, - completed_at=completed_at, - expected_query_ids=expected_query_ids, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - attributed_observations: tuple[QueryCostObservation, ...] = tuple( - _attribute_from_ledger(observation=observation, ledger=ledger_by_query_id) - for observation in observations - ) - summary = allocate_run_cost( - observations=attributed_observations, - run_id=run_id, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - result_limit_reached=saturated, - ) - summary = replace( - summary, - limitations=tuple(sorted({*summary.limitations, _VISIBLE_QUERY_LIMITATION})), - ) - if expected_query_ids.issubset(observed_query_ids) and not _has_incomplete_observation( - expected_query_ids=expected_query_ids, - skipped_reasons=skipped_reasons, - ): - summary = _apply_observation_gaps( - summary=summary, - expected_statement_count=len(ledger), - expected_query_ids=expected_query_ids, - observed_query_ids=observed_query_ids, - skipped_reasons=skipped_reasons, - ) - if summary.status == CostStatus.PENDING: - summary = replace( - summary, - status=CostStatus.COMPLETE, - message="No cost-bearing Snowflake warehouse queries were observed.", - ) - return _apply_ledger_failure(summary=summary, ledger_failure=ledger_failure) - if attempt + 1 < attempts: - remaining_seconds: float = deadline - time.monotonic() - if remaining_seconds <= retry_delay_seconds: - break - time.sleep(retry_delay_seconds) - - if summary is None: - summary = _partial_summary( - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message="Snowflake query history could not be collected.", - ) - missing_count: int = len(expected_query_ids.difference(observed_query_ids)) - limitations: set[str] = set(summary.limitations) - if missing_count: - limitations.add( - f"{missing_count} statement ledger query ID(s) were not visible in query history." - ) - if saturated: - limitations.add("Snowflake query-history result limit was reached.") - visible_expected_query_ids: frozenset[str] = expected_query_ids.intersection(observed_query_ids) - history_expired: bool = datetime.now(UTC) - completed_at >= _QUERY_HISTORY_RETENTION - final_status: CostStatus - if history_expired and not visible_expected_query_ids: - final_status = CostStatus.UNAVAILABLE - message = "Snowflake query history is no longer available for this run." - elif not visible_expected_query_ids and not saturated: - final_status = CostStatus.PENDING - message = "Snowflake query history is not complete yet. Run `sqb cost latest` to refresh." - else: - final_status = CostStatus.PARTIAL - message = summary.message or "Snowflake query history was only partially available." - summary = _apply_observation_gaps( - summary=replace( - summary, - status=final_status, - limitations=tuple(sorted(limitations)), - message=message, - ), - expected_statement_count=len(ledger), - expected_query_ids=expected_query_ids, - observed_query_ids=observed_query_ids, - skipped_reasons=skipped_reasons, - ) - return _apply_ledger_failure( - summary=summary, - ledger_failure=ledger_failure, - ) - - -def _query_observations( - *, - connection: Any, - database: str | None = None, - started_at: datetime, - completed_at: datetime, - expected_query_ids: frozenset[str], - classification_query_ids: frozenset[str], - deadline: float, -) -> tuple[ - tuple[QueryCostObservation, ...], - frozenset[str], - dict[str, str], - bool, -]: - open_rows: tuple[tuple[Any, ...], ...] = _query_open_history_rows( - connection=connection, - database=database, - started_at=started_at - _MAXIMUM_CLOCK_SKEW, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - saturated: bool = len(open_rows) >= _RESULT_LIMIT - if saturated: - bounded_rows, bounded_saturated = _query_history_rows( - connection=connection, - database=database, - started_at=started_at - _MAXIMUM_CLOCK_SKEW, - completed_at=completed_at + _MAXIMUM_CLOCK_SKEW, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - rows_by_query_id: dict[str, tuple[Any, ...]] = { - str(row[0]): row for row in open_rows if _is_running_history_row(row=row) - } - rows_by_query_id.update({str(row[0]): row for row in bounded_rows}) - rows: tuple[tuple[Any, ...], ...] = tuple( - rows_by_query_id[key] for key in sorted(rows_by_query_id) - ) - saturated = bounded_saturated - else: - rows = open_rows - observations: list[QueryCostObservation] = [] - observed_query_ids: set[str] = set() - skipped_reasons: dict[str, str] = {} - for row in rows: - query_id: str = str(row[0]) - observed_query_ids.add(query_id) - observation, skip_reason = _row_to_observation( - row=row, - require_complete=query_id in expected_query_ids, - ) - if observation is not None: - observations.append(observation) - elif skip_reason is not None: - skipped_reasons[query_id] = skip_reason - observations.sort(key=lambda item: (item.started_at, item.query_id)) - return tuple(observations), frozenset(observed_query_ids), skipped_reasons, saturated - - -def _is_running_history_row(*, row: tuple[Any, ...]) -> bool: - return row[7] is None or str(row[10]).upper() == RUNNING_EXECUTION_STATUS - - -def _query_open_history_rows( - *, - connection: Any, - database: str | None = None, - started_at: datetime, - classification_query_ids: frozenset[str], - deadline: float, -) -> tuple[tuple[Any, ...], ...]: - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - cursor: Any = connection.execute( - _render_open_query_history_sql( - database=database, - started_at=started_at, - classification_query_ids=classification_query_ids, - ), - statement_params={"STATEMENT_TIMEOUT_IN_SECONDS": _QUERY_TIMEOUT_SECONDS}, - ) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - rows: tuple[tuple[Any, ...], ...] = tuple(tuple(row) for row in cursor.fetchall()) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - return rows - - -def _query_history_rows( - *, - connection: Any, - database: str | None = None, - started_at: datetime, - completed_at: datetime, - classification_query_ids: frozenset[str], - deadline: float, -) -> tuple[tuple[tuple[Any, ...], ...], bool]: - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - cursor: Any = connection.execute( - _render_query_history_sql( - database=database, - started_at=started_at, - completed_at=completed_at, - classification_query_ids=classification_query_ids, - ), - statement_params={"STATEMENT_TIMEOUT_IN_SECONDS": _QUERY_TIMEOUT_SECONDS}, - ) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - rows: tuple[tuple[Any, ...], ...] = tuple(tuple(row) for row in cursor.fetchall()) - if time.monotonic() >= deadline: - raise TimeoutError("Snowflake cost collection deadline exceeded") - if len(rows) < _RESULT_LIMIT: - return rows, False - if completed_at - started_at <= _MINIMUM_SPLIT_WINDOW: - return rows, True - - midpoint: datetime = started_at + (completed_at - started_at) / 2 - left_rows, left_saturated = _query_history_rows( - connection=connection, - database=database, - started_at=started_at, - completed_at=midpoint, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - right_rows, right_saturated = _query_history_rows( - connection=connection, - database=database, - started_at=midpoint, - completed_at=completed_at, - classification_query_ids=classification_query_ids, - deadline=deadline, - ) - rows_by_query_id: dict[str, tuple[Any, ...]] = { - str(row[0]): row for row in (*left_rows, *right_rows) - } - merged_rows: tuple[tuple[Any, ...], ...] = tuple( - rows_by_query_id[key] for key in sorted(rows_by_query_id) - ) - return merged_rows, left_saturated or right_saturated - - -def _row_to_observation( - *, - row: tuple[Any, ...], - require_complete: bool, -) -> tuple[QueryCostObservation | None, str | None]: - ( - query_id, - query_tag, - warehouse_name, - warehouse_size, - warehouse_type, - cluster_number, - start_time, - end_time, - execution_ms, - bytes_scanned, - execution_status, - query_type, - query_text, - observed_at, - ) = row - milliseconds: int = max(0, int(execution_ms or 0)) - if _is_metadata_only_statement(query_type=query_type, query_text=query_text): - return None, NO_WAREHOUSE_COMPUTE_REASON - running: bool = end_time is None or str(execution_status).upper() == RUNNING_EXECUTION_STATUS - if require_complete and running: - return None, INCOMPLETE_HISTORY_REASON - if milliseconds == 0 or warehouse_name is None or warehouse_size is None: - return None, MISSING_WAREHOUSE_METADATA_REASON - tag: dict[str, object] | None = _parse_sqlbuild_tag(query_tag) - effective_end: datetime = observed_at if running else end_time - active_start: datetime = max( - start_time, - effective_end - timedelta(milliseconds=milliseconds), - ) - if effective_end <= active_start: - return None, NO_WAREHOUSE_COMPUTE_REASON - return ( - QueryCostObservation( - query_id=str(query_id), - warehouse_name=str(warehouse_name), - warehouse_size=str(warehouse_size), - warehouse_type=None if warehouse_type is None else str(warehouse_type), - cluster_number=None if cluster_number is None else int(cluster_number), - started_at=active_start, - completed_at=effective_end, - execution_ms=milliseconds, - bytes_scanned=int(bytes_scanned or 0), - execution_status=str(execution_status), - run_id=None if tag is None else _optional_string(tag.get("run_id")), - resource_type=None if tag is None else _optional_string(tag.get("resource_type")), - resource_name=None if tag is None else _optional_string(tag.get("resource_name")), - ), - None, - ) - - -def _attribute_from_ledger( - *, - observation: QueryCostObservation, - ledger: dict[str, StatementLedgerEntry], -) -> QueryCostObservation: - entry: StatementLedgerEntry | None = ledger.get(observation.query_id) - if entry is None: - return replace( - observation, - run_id=None, - resource_type=None, - resource_name=None, - ) - return replace( - observation, - run_id=entry.run_id, - resource_type=entry.resource_type, - resource_name=entry.resource_name, - ) - - -def _render_query_history_sql( - *, - database: str | None = None, - started_at: datetime, - completed_at: datetime, - classification_query_ids: frozenset[str], -) -> str: - start: str = started_at.isoformat() - end: str = completed_at.isoformat() - query_text_expression: str = _render_query_text_expression( - classification_query_ids=classification_query_ids - ) - query_history_function: str = _render_query_history_function(database=database) - return f""" -SELECT - QUERY_ID, - QUERY_TAG, - WAREHOUSE_NAME, - WAREHOUSE_SIZE, - WAREHOUSE_TYPE, - CLUSTER_NUMBER, - START_TIME, - END_TIME, - EXECUTION_TIME, - BYTES_SCANNED, - EXECUTION_STATUS, - QUERY_TYPE, - {query_text_expression}, - CURRENT_TIMESTAMP() -FROM TABLE({query_history_function}( - END_TIME_RANGE_START => TO_TIMESTAMP_LTZ('{start}'), - END_TIME_RANGE_END => TO_TIMESTAMP_LTZ('{end}'), - RESULT_LIMIT => {_RESULT_LIMIT} -)) -ORDER BY START_TIME, QUERY_ID -""".strip() - - -def _render_open_query_history_sql( - *, - database: str | None = None, - started_at: datetime, - classification_query_ids: frozenset[str], -) -> str: - start: str = started_at.isoformat() - query_text_expression: str = _render_query_text_expression( - classification_query_ids=classification_query_ids - ) - query_history_function: str = _render_query_history_function(database=database) - return f""" -SELECT - QUERY_ID, - QUERY_TAG, - WAREHOUSE_NAME, - WAREHOUSE_SIZE, - WAREHOUSE_TYPE, - CLUSTER_NUMBER, - START_TIME, - END_TIME, - EXECUTION_TIME, - BYTES_SCANNED, - EXECUTION_STATUS, - QUERY_TYPE, - {query_text_expression}, - CURRENT_TIMESTAMP() -FROM TABLE({query_history_function}( - END_TIME_RANGE_START => TO_TIMESTAMP_LTZ('{start}'), - RESULT_LIMIT => {_RESULT_LIMIT} -)) -ORDER BY START_TIME, QUERY_ID -""".strip() - - -def _render_query_history_function(*, database: str | None) -> str: - if database is None: - return "INFORMATION_SCHEMA.QUERY_HISTORY" - quoted_database: str = database.upper().replace('"', '""') - return f'"{quoted_database}".INFORMATION_SCHEMA.QUERY_HISTORY' - - -def _render_query_text_expression(*, classification_query_ids: frozenset[str]) -> str: - if not classification_query_ids: - return "NULL AS QUERY_TEXT" - query_ids: str = ", ".join( - f"'{query_id.replace(chr(39), chr(39) * 2)}'" - for query_id in sorted(classification_query_ids) - ) - return ( - f"CASE WHEN QUERY_ID IN ({query_ids}) " - "AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') " - "THEN QUERY_TEXT END AS QUERY_TEXT" - ) - - -def _parse_sqlbuild_tag(value: object) -> dict[str, object] | None: - if not isinstance(value, str) or not value: - return None - try: - payload: object = json.loads(value) - except json.JSONDecodeError: - return None - if not isinstance(payload, dict): - return None - if payload.get("app") != SQLBUILD_QUERY_TAG_APP or payload.get("v") != 1: - return None - return payload - - -def _is_metadata_only_statement(*, query_type: object, query_text: object) -> bool: - """Identify statements Snowflake executes without warehouse compute.""" - - normalized_type: str = str(query_type or "").upper().replace(" ", "_") - if normalized_type == _USE_QUERY_TYPE or normalized_type.startswith(f"{_USE_QUERY_TYPE}_"): - return True - if normalized_type == _DROP_QUERY_TYPE or normalized_type.startswith(f"{_DROP_QUERY_TYPE}_"): - return True - if normalized_type != _CREATE_QUERY_TYPE and not normalized_type.startswith( - f"{_CREATE_QUERY_TYPE}_" - ): - return False - if not isinstance(query_text, str) or not query_text.strip(): - return False - try: - statement: Any = parse_one(query_text, dialect="snowflake") - except ParseError: - return False - if isinstance(statement, CreateTable): - return not any( - ( - statement.args.get("as_select") is not None, - statement.args.get("clone_source") is not None, - statement.args.get("using_template") is not None, - str(statement.args.get("table_modifier") or "").upper() == _DYNAMIC_TABLE_MODIFIER, - ) - ) - if isinstance(statement, CreateView) and bool(statement.args.get("materialized")): - return False - return type(statement).__name__.startswith("Create") - - -def _optional_string(value: object) -> str | None: - return value if isinstance(value, str) and value else None - - -def _partial_summary(*, usd_per_credit: Decimal, rate_source: str, message: str) -> RunCostSummary: - return RunCostSummary( - status=CostStatus.PARTIAL, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - message=message, - ) - - -def _apply_ledger_failure(*, summary: RunCostSummary, ledger_failure: str | None) -> RunCostSummary: - if ledger_failure is None: - return summary - limitation: str = f"{LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX}{ledger_failure})." - return replace( - summary, - status=CostStatus.PARTIAL, - limitations=tuple(sorted({*summary.limitations, limitation})), - message=summary.message or "Cost telemetry was only partially persisted.", - ) - - -def _apply_observation_gaps( - *, - summary: RunCostSummary, - expected_statement_count: int, - expected_query_ids: frozenset[str], - observed_query_ids: frozenset[str], - skipped_reasons: dict[str, str], -) -> RunCostSummary: - relevant_skips: dict[str, str] = { - query_id: reason - for query_id, reason in skipped_reasons.items() - if query_id in expected_query_ids - } - limitations: set[str] = set(summary.limitations) - if MISSING_WAREHOUSE_METADATA_REASON in relevant_skips.values(): - limitations.add("One or more run statements had missing Snowflake warehouse metadata.") - if INCOMPLETE_HISTORY_REASON in relevant_skips.values(): - limitations.add("One or more run statements were still incomplete in query history.") - status: CostStatus = summary.status - if MISSING_WAREHOUSE_METADATA_REASON in relevant_skips.values(): - status = CostStatus.PARTIAL - elif INCOMPLETE_HISTORY_REASON in relevant_skips.values() and summary.query_count: - status = CostStatus.PARTIAL - return replace( - summary, - status=status, - limitations=tuple(sorted(limitations)), - expected_statement_count=expected_statement_count, - observed_statement_count=len(expected_query_ids.intersection(observed_query_ids)), - missing_statement_count=( - expected_statement_count - len(expected_query_ids.intersection(observed_query_ids)) - ), - skipped_statement_count=len(relevant_skips), - ) - - -def _has_incomplete_observation( - *, expected_query_ids: frozenset[str], skipped_reasons: dict[str, str] -) -> bool: - return any( - query_id in expected_query_ids and reason in _INCOMPLETE_OBSERVATION_REASONS - for query_id, reason in skipped_reasons.items() - ) diff --git a/src/sqlbuild/cost/classes/cost_context.py b/src/sqlbuild/cost/classes/cost_context.py deleted file mode 100644 index 0b769e2785..0000000000 --- a/src/sqlbuild/cost/classes/cost_context.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Run-cost execution context.""" - -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext -from pathlib import Path - -from sqlbuild.cost._helpers.context import ( - cost_resource_context, - current_cost_resource_context, -) -from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry - - -class CostContext: - @staticmethod - def current() -> CostResourceContext | None: - return current_cost_resource_context() - - @staticmethod - def scope( - *, - run_id: str, - resource_type: str, - resource_name: str, - ledger_path: Path | None = None, - phase: str = "execute", - attempt: int = 1, - on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None, - ) -> AbstractContextManager[None]: - current: CostResourceContext | None = CostContext.current() - return cost_resource_context( - run_id=run_id, - resource_type=resource_type, - resource_name=resource_name, - ledger_path=( - ledger_path - if ledger_path is not None - else (None if current is None else current.ledger_path) - ), - phase=phase, - attempt=attempt, - on_statement_complete=( - on_statement_complete - if on_statement_complete is not None - else (None if current is None else current.on_statement_complete) - ), - ) - - @staticmethod - def resource_scope( - *, - resource_type: str, - resource_name: str, - phase: str | None = None, - attempt: int = 1, - ) -> AbstractContextManager[None]: - current: CostResourceContext | None = CostContext.current() - if current is None: - return nullcontext() - return CostContext.scope( - run_id=current.run_id, - resource_type=resource_type, - resource_name=resource_name, - ledger_path=current.ledger_path, - phase=current.phase if phase is None else phase, - attempt=attempt, - on_statement_complete=current.on_statement_complete, - ) diff --git a/src/sqlbuild/cost/classes/statement_ledger.py b/src/sqlbuild/cost/classes/statement_ledger.py deleted file mode 100644 index 057df118eb..0000000000 --- a/src/sqlbuild/cost/classes/statement_ledger.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Best-effort statement ledger.""" - -from datetime import datetime - -from sqlbuild.cost._helpers.ledger import record_statement -from sqlbuild.cost.models import CostResourceContext - - -class StatementLedger: - @staticmethod - def record( - *, - context: CostResourceContext, - statement_id: str, - sql: str, - query_id: object, - status: str, - started_at: datetime, - completed_at: datetime, - error: Exception | None = None, - ) -> None: - record_statement( - context=context, - statement_id=statement_id, - sql=sql, - query_id=query_id, - status=status, - started_at=started_at, - completed_at=completed_at, - error=error, - ) diff --git a/src/sqlbuild/cost/main/__init__.py b/src/sqlbuild/cost/main/__init__.py deleted file mode 100644 index 9b431179fa..0000000000 --- a/src/sqlbuild/cost/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public cost observability entry points.""" diff --git a/src/sqlbuild/cost/main/collection.py b/src/sqlbuild/cost/main/collection.py deleted file mode 100644 index 392bba226d..0000000000 --- a/src/sqlbuild/cost/main/collection.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Snowflake cost collection entry point.""" - -from datetime import datetime -from decimal import Decimal -from pathlib import Path -from typing import Any - -from sqlbuild.cost._helpers.snowflake import collect_snowflake_run_cost -from sqlbuild.cost.models import RunCostSummary - - -def collect_snowflake_cost( - *, - connection: Any, - database: str | None = None, - run_id: str, - started_at: datetime, - completed_at: datetime, - statement_ledger_path: Path, - usd_per_credit: Decimal, - rate_source: str, - attempts: int = 3, - retry_delay_seconds: float = 1.0, -) -> RunCostSummary: - return collect_snowflake_run_cost( - connection=connection, - database=database, - run_id=run_id, - started_at=started_at, - completed_at=completed_at, - statement_ledger_path=statement_ledger_path, - usd_per_credit=usd_per_credit, - rate_source=rate_source, - attempts=attempts, - retry_delay_seconds=retry_delay_seconds, - ) diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index e24ee76f7f..6b16e9c4a3 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -10,6 +10,7 @@ from typing import Any, cast from sqlbuild._native import ( + CostContext, assert_no_unresolved_sql_markers, evaluate_measurement, render_audit_sql, @@ -22,7 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index ce74485e38..028b2af56b 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -7,12 +7,12 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 220e1bf020..892158d337 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import RelationInfo @@ -37,7 +38,6 @@ IncrementalStrategy, PlanAction, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index ea10e1add6..4db2a54708 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -7,13 +7,13 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.cursors import ( diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index f645564581..aee09c8380 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -15,10 +15,10 @@ from typing import Any, cast from uuid import uuid4 +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index 8ed47a4f25..3c6faeaf0d 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -6,9 +6,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.build.main._execute import execute_build_plan diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 0a291eb538..1c36b1e9dd 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -8,6 +8,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction from sqlbuild.compiler.python_nodes.types import ( @@ -15,7 +16,6 @@ PythonNodeKind, PythonNodeStatus, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.models import NodeResultRecord diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 64d67e6450..3faa8e9f0e 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -8,6 +8,7 @@ from functools import partial from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, @@ -20,7 +21,6 @@ PythonSqlRunLifecyclePlan, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.main._resource_kind import load_resource_kind diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 86f89689f4..fe5a64ac54 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction @@ -15,7 +16,6 @@ PythonNodeIdentity, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index 82c964c868..bed168a6ee 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -5,10 +5,10 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index 0d60c85c5d..e5cd6aa0b1 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import render_source_relation +from sqlbuild._native import CostContext, render_source_relation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( @@ -22,7 +22,6 @@ from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py index 27698afa10..8a93a03435 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_cost.py @@ -7,9 +7,8 @@ import pytest +from sqlbuild._native import CostContext, collect_snowflake_cost from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.main.collection import collect_snowflake_cost from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py index 272150769b..ed4d0986b8 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py @@ -9,8 +9,8 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from tests.integration.src.sqlbuild.executor.build.concurrent._test_types import ( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py b/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py index 39ce775b47..dae2f53662 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py @@ -8,9 +8,9 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.adapters.snowflake.classes.snowflake_cursor import _SnowflakeCursor -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.observability import ( @@ -512,13 +512,9 @@ def test_given_ledger_write_failure_when_executing_then_statement_result_is_unch ) -> None: cursor: _Cursor = _Cursor() connection: _SnowflakeConnection = _SnowflakeConnection(_RawConnection(cursor)) - ledger_path: Path = tmp_path / "statements.jsonl" - - def fail_append(**kwargs: Any) -> None: - del kwargs - raise OSError("disk unavailable") - - monkeypatch.setattr("sqlbuild.cost._helpers.ledger._append_statement", fail_append) + blocker: Path = tmp_path / "blocker" + blocker.write_text("not a directory", encoding="utf-8") + ledger_path: Path = blocker / "statements.jsonl" with CostContext.scope( run_id="ledger-failure-run", resource_type="model", @@ -531,7 +527,9 @@ def fail_append(**kwargs: Any) -> None: assert cursor.sfqid == test_case.expected_query_id assert test_case.expected_status == "success" assert not ledger_path.exists() - assert COST_TELEMETRY_HEALTH.consume_ledger_failure(run_id="ledger-failure-run") == "OSError" + assert COST_TELEMETRY_HEALTH.consume_ledger_failure(run_id="ledger-failure-run") == ( + "FileExistsError" + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index a9ad4bd0d8..5054627b09 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -3,7 +3,7 @@ from datetime import UTC, datetime from pathlib import Path -from sqlbuild._native import build_source_freshness_create_table_sql +from sqlbuild._native import CostContext, build_source_freshness_create_table_sql from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -17,7 +17,6 @@ SourceFreshnessIdentity, SourceFreshnessRecord, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index aa3eebe902..7360cdc360 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( append_eligible_direct_source_freshness_records, @@ -15,7 +16,6 @@ DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, ) -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/cost/test_allocation.py b/tests/unit/src/sqlbuild/cost/test_allocation.py index 9193b89449..b6e2c87032 100644 --- a/tests/unit/src/sqlbuild/cost/test_allocation.py +++ b/tests/unit/src/sqlbuild/cost/test_allocation.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cost._helpers.allocation import allocate_run_cost +from sqlbuild._native import allocate_run_cost from sqlbuild.cost.models import QueryCostObservation, RunCostSummary from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import AllocationTestCase diff --git a/tests/unit/src/sqlbuild/cost/test_context.py b/tests/unit/src/sqlbuild/cost/test_context.py index daff103809..28ad5accbb 100644 --- a/tests/unit/src/sqlbuild/cost/test_context.py +++ b/tests/unit/src/sqlbuild/cost/test_context.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.cost.classes.cost_context import CostContext +from sqlbuild._native import CostContext from sqlbuild.cost.models import CostResourceContext from tests.unit.src.sqlbuild.cost._test_types import CostResourceScopeTestCase diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index 173fdadbc6..7b6355c511 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -9,15 +9,14 @@ import pytest -from sqlbuild.cost._helpers.snowflake import ( - _is_metadata_only_statement, - _parse_sqlbuild_tag, - _query_observations, - _render_open_query_history_sql, - _render_query_history_sql, +from sqlbuild._native import ( + collect_snowflake_cost, + parse_sqlbuild_query_tag, + render_snowflake_query_history_sql, + snowflake_query_observations, + snowflake_statement_is_metadata_only, ) from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH -from sqlbuild.cost.main.collection import collect_snowflake_cost from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import ( @@ -88,7 +87,7 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_table_function( test_case: RenderQueryHistorySqlTestCase, ) -> None: - sql: str = _render_query_history_sql( + sql: str = render_snowflake_query_history_sql( database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), completed_at=datetime(2026, 8, 23, 10, 5, tzinfo=UTC), @@ -123,7 +122,7 @@ def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_t def test_given_start_time_when_rendering_open_history_then_running_queries_are_requested( test_case: RenderQueryHistorySqlTestCase, ) -> None: - sql: str = _render_open_query_history_sql( + sql: str = render_snowflake_query_history_sql( database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), classification_query_ids=frozenset({"q-create"}), @@ -148,7 +147,7 @@ def test_given_start_time_when_rendering_open_history_then_running_queries_are_r def test_given_no_classifiable_create_ids_when_rendering_history_then_query_text_is_masked( test_case: RenderQueryHistorySqlTestCase, ) -> None: - sql: str = _render_open_query_history_sql( + sql: str = render_snowflake_query_history_sql( started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), classification_query_ids=frozenset(), ) @@ -224,7 +223,6 @@ def test_given_exact_create_hook_when_collecting_then_hook_query_text_is_not_req ) def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not_saturated( test_case: SnowflakeSaturationTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) row: tuple[Any, ...] = ( @@ -247,9 +245,9 @@ def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not connection: _SequencedConnection = _SequencedConnection( ((row, ("q-open-2", *row[1:])), (bounded_row,)) ) - monkeypatch.setattr("sqlbuild.cost._helpers.snowflake._RESULT_LIMIT", 2) - _, observed_query_ids, _, saturated = _query_observations( + _, observed_query_ids, _, saturated = snowflake_query_observations( + result_limit=2, connection=connection, started_at=origin, completed_at=origin + timedelta(seconds=1), @@ -277,8 +275,6 @@ def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not } ), expected_payload={ - "app": "sqlbuild", - "v": 1, "run_id": "run-1", "resource_type": "model", "resource_name": "orders", @@ -290,7 +286,7 @@ def test_given_saturated_open_history_when_bounded_fallback_is_complete_then_not def test_given_sqlbuild_query_tag_when_parsing_then_context_is_returned( test_case: ParseQueryTagTestCase, ) -> None: - parsed: dict[str, Any] | None = _parse_sqlbuild_tag(test_case.query_tag) + parsed: dict[str, Any] | None = parse_sqlbuild_query_tag(test_case.query_tag) assert parsed == test_case.expected_payload @@ -319,7 +315,7 @@ def test_given_sqlbuild_query_tag_when_parsing_then_context_is_returned( def test_given_unrelated_or_invalid_query_tag_when_parsing_then_none_is_returned( test_case: ParseQueryTagTestCase, ) -> None: - assert _parse_sqlbuild_tag(test_case.query_tag) == test_case.expected_payload + assert parse_sqlbuild_query_tag(test_case.query_tag) == test_case.expected_payload @pytest.mark.parametrize( @@ -386,7 +382,7 @@ def test_given_snowflake_statement_when_classifying_then_compute_capability_is_c test_case: SnowflakeStatementClassificationTestCase, ) -> None: assert ( - _is_metadata_only_statement( + snowflake_statement_is_metadata_only( query_type=test_case.query_type, query_text=test_case.query_text, ) @@ -871,11 +867,11 @@ def test_given_retry_delay_exceeds_remaining_deadline_when_collecting_then_parti ) monotonic_values: Iterator[float] = iter((0.0, 0.0, 0.0, 0.0, 9.5)) monkeypatch.setattr( - "sqlbuild.cost._helpers.snowflake.time.monotonic", + "time.monotonic", lambda: next(monotonic_values), ) monkeypatch.setattr( - "sqlbuild.cost._helpers.snowflake.time.sleep", + "time.sleep", lambda seconds: pytest.fail(f"unexpected sleep for {seconds} seconds"), ) @@ -1168,7 +1164,6 @@ def test_given_visible_running_competitor_when_collecting_then_busy_time_is_shar def test_given_saturated_open_history_when_bounded_fallback_completes_then_running_share_is_retained( test_case: CollectSnowflakeCostTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: origin: datetime = datetime(2026, 8, 23, 10, tzinfo=UTC) ledger_path: Path = tmp_path / "statements.jsonl" @@ -1225,10 +1220,10 @@ def test_given_saturated_open_history_when_bounded_fallback_completes_then_runni connection: _SequencedConnection = _SequencedConnection( ((completed, running_competitor), (completed,)) ) - monkeypatch.setattr("sqlbuild.cost._helpers.snowflake._RESULT_LIMIT", 2) summary: RunCostSummary = collect_snowflake_cost( connection=connection, + result_limit=2, run_id="run-saturated-share", started_at=origin, completed_at=origin + timedelta(seconds=1), diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index 56e7a4cd4c..9e5203a0d5 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -9,10 +9,10 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index e3cb4245ec..21daaa917a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -18,7 +18,12 @@ import pytest -from sqlbuild._native import AuditProgressReporter +from sqlbuild._native import ( + AuditProgressReporter, + CostContext, + read_statement_ledger, + record_statement, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -30,8 +35,6 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost._helpers.ledger import read_statement_ledger, record_statement -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py index 631389fa10..bcf529cd61 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py @@ -7,9 +7,9 @@ import pytest import sqlbuild.executor.pipeline.main.run as run_module +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 5647bb96ac..df86a5e314 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import build_python_sql_run_lifecycle +from sqlbuild._native import CostContext, build_python_sql_run_lifecycle from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.assets import AssetContext, asset @@ -31,7 +31,6 @@ PythonSqlRunSelection, ) from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index 012182863a..e50de69cc8 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index 321a6da001..dc5da97064 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -9,6 +9,7 @@ import pytest from sqlbuild._native import ( + CostContext, ExecutionEventWriter, TerminalEventIndex, format_check_json, @@ -17,7 +18,6 @@ from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.models import NodeResultEnvelope from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py index 97043e06f8..934ce4aedc 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py @@ -7,12 +7,12 @@ import pytest +from sqlbuild._native import CostContext from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks From 5777c9132ec9117391102e403dc9448ea517d48b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:04:17 +0100 Subject: [PATCH 10/33] docs: describe native source freshness and run-cost modules --- crates/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/README.md b/crates/README.md index 3bfdbb0b5f..54922e3765 100644 --- a/crates/README.md +++ b/crates/README.md @@ -6,9 +6,9 @@ that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | |---|---| -| `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, and a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte. | +| `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. | | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. | -| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence and age policy, byte-compatible with the rows Python wrote. | +| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. | | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. | | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. | | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | @@ -18,7 +18,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-model-config` | MODEL configuration, columns and audits, template expansion over caller values, and dynamic-column name patterns using Rust regex syntax. | | `sqlbuild-render` | Macro call scanning, argument evaluation, expansion, splicing, memoisation and persistent call-store keys. Python executes user macro bodies at the extension boundary. | | `sqlbuild-attachments` | Native attachment of tests, audits, sources, functions, scenarios and seeds, including argument merging, validation and audit policy resolution. Adapter value rendering remains an extension boundary. | -| `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, and SQL-test/scenario extraction and compiler-side assembly. | +| `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, SQL-test/scenario extraction and compiler-side assembly, and Snowflake cost statement classification and query-tag parsing. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | From f7f3317a227667b8a247c68978d371389d256366 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:55:35 +0100 Subject: [PATCH 11/33] refactor: delete unused CLI, cost, dbt and value constants and models --- src/sqlbuild/cli/commands/constants.py | 34 ---------------------- src/sqlbuild/cost/constants.py | 8 ----- src/sqlbuild/cursor_algebra/constants.py | 8 ----- src/sqlbuild/integrations/dbt/constants.py | 20 ------------- src/sqlbuild/integrations/dbt/models.py | 12 -------- src/sqlbuild/integrations/dbt/types.py | 11 ------- src/sqlbuild/presentation/constants.py | 2 -- src/sqlbuild/sql_values/constants.py | 3 -- 8 files changed, 98 deletions(-) delete mode 100644 src/sqlbuild/integrations/dbt/constants.py diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index a0d40b595d..2be1351002 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -5,41 +5,10 @@ from sqlbuild.cli.commands.types import CliCommand from sqlbuild.spec.contracts.types import WarehouseGroup -C0_CONTROL_CODE_LIMIT: int = 32 -C1_CONTROL_CODE_START: int = 127 -C1_CONTROL_CODE_LIMIT: int = 160 EMPTY_DAG_PATH: str = "" TARGET_DIRECTORY_NAME: str = "target" SQLBUILD_CONCURRENCY_ENV_VAR: str = "SQLBUILD_CONCURRENCY" -NO_COLOR_OPTION: str = "--no-color" -DBT_INIT_COMMAND: str = "init" -SCENARIO_TEST_COMMAND: str = "test" -SCENARIO_CAPTURE_COMMAND: str = "capture" -JSON_OUTPUT_FORMAT: str = "json" -CSV_OUTPUT_FORMAT: str = "csv" -DEBUG_COMMAND: str = "debug" -LINEAGE_COMMAND: str = "lineage" -QUERY_COMMAND: str = "query" -RULES_LIST_ACTION: str = "list" -RULES_SHOW_ACTION: str = "show" -RULES_SKILLS_ACTION: str = "skills" -SQL_FILE_SUFFIX: str = ".sql" -SUCCESS_STATUS: str = "success" -FAILED_STATUS: str = "failed" -SCENARIO_CLI_MISSING_SUBCOMMAND: str = "C450" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" -COST_NARROW_TERMINAL_WIDTH: int = 120 -COST_HISTORY_NUMERIC_COLUMNS: frozenset[int] = frozenset({4, 5}) -COST_RESOURCE_NUMERIC_START_COLUMN: int = 2 -BYTE_SCALE: int = 1024 -BYTE_UNIT: str = "B" -TERABYTE_UNIT: str = "TiB" -SCOPE_PATH_RELATIVE: str = "relative" -SCOPE_PATH_COMPACT: str = "compact" -SCOPE_PATH_NONE: str = "none" -SCOPE_GLOBAL_SUMMARY: str = "summary" -SCOPE_GLOBAL_ALL: str = "all" -SCOPE_DEFAULT_PAGE_SIZE: int = 100 MISSING_ORIGIN_BUILD_HELP: str = ( "Pass --allow-missing-migration-origin to confirm in non-interactive runs." ) @@ -74,9 +43,6 @@ ) EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." EXECUTION_LIMIT_DBT_NOTE: str = "dbt work has already run; no SQLBuild models were built." -CLI_WAREHOUSE_SOURCE: str = "--warehouse" -CONNECTION_WAREHOUSE_SOURCE: str = "connection warehouse" -CONNECTION_DEFAULT_WAREHOUSE_SOURCE: str = "connection default" COMMAND_WAREHOUSE_GROUPS: MappingProxyType[CliCommand, WarehouseGroup | None] = MappingProxyType( { CliCommand.AUDIT: WarehouseGroup.BUILD, diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py index c426667edb..d0277f87c6 100644 --- a/src/sqlbuild/cost/constants.py +++ b/src/sqlbuild/cost/constants.py @@ -6,14 +6,6 @@ DEFAULT_USD_PER_CREDIT: Decimal = Decimal("3.00") DEFAULT_RATE_SOURCE: str = "default" -STANDARD_WAREHOUSE_TYPE: str = "STANDARD" -RUNNING_EXECUTION_STATUS: str = "RUNNING" SQLBUILD_QUERY_TAG_APP: str = "sqlbuild" -USD_PER_CREDIT_CONFIG_KEY: str = "usd_per_credit" -LATEST_COST_RUN_SELECTOR: str = "latest" -RUNNING_BUILD_STATUS: str = "running" -NO_WAREHOUSE_COMPUTE_REASON: str = "no_warehouse_compute" -MISSING_WAREHOUSE_METADATA_REASON: str = "missing_warehouse_metadata" -INCOMPLETE_HISTORY_REASON: str = "incomplete_history" LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX: str = "Statement-ledger persistence failed (" COST_TELEMETRY_HEALTH: CostTelemetryHealth = CostTelemetryHealth() diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index f3dac3ee9a..9d38c029db 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -11,14 +11,6 @@ CursorGrain.YEAR: 5, } -GRAIN_BATCH_SIZE: dict[CursorGrain, str] = { - CursorGrain.SECOND: "1s", - CursorGrain.MINUTE: "1m", - CursorGrain.HOUR: "1h", - CursorGrain.DAY: "1d", - CursorGrain.MONTH: "1mo", - CursorGrain.YEAR: "1y", -} DURATION_YEAR_UNIT: str = "y" DURATION_MONTH_UNIT: str = "mo" diff --git a/src/sqlbuild/integrations/dbt/constants.py b/src/sqlbuild/integrations/dbt/constants.py deleted file mode 100644 index e6b2f3d9d6..0000000000 --- a/src/sqlbuild/integrations/dbt/constants.py +++ /dev/null @@ -1,20 +0,0 @@ -"""dbt integration constants.""" - -from sqlbuild.integrations.dbt.types import DbtInteropCommand - -DBT_EXECUTION_COMMANDS: frozenset[DbtInteropCommand] = frozenset( - ( - DbtInteropCommand.PLAN, - DbtInteropCommand.RUN, - DbtInteropCommand.BUILD, - ) -) -DBT_EXECUTION_DISPLAY_FLAGS: frozenset[str] = frozenset(("--json", "--verbose", "-v")) - -DBT_MANIFEST_CONFIG_KEY: str = "config" -DBT_MANIFEST_MATERIALIZED_KEY: str = "materialized" - - -DBT_DEFER_FLAG: str = "--defer" -DBT_FULL_REFRESH_FLAG: str = "--full-refresh" -DBT_UNIQUE_ID_SEPARATOR: str = "." diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 6635c9274a..742e719653 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -165,18 +165,6 @@ class NormalizedDbtProfileConnection: warnings: tuple[str, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DbtProfileConnectionRequest: - """Request to resolve one dbt-profile-backed SQLBuild connection.""" - - sqlbuild_project_dir: Path - dbt_project_dir: Path | None - profiles_dir: Path | None - profile_name: str | None - target_name: str | None - cli_vars: dict[str, object] = field(default_factory=dict) - - @dataclass(frozen=True) class DbtInitProgressCallbacks: """Optional progress hooks for `sqb dbt init` phases.""" diff --git a/src/sqlbuild/integrations/dbt/types.py b/src/sqlbuild/integrations/dbt/types.py index c9558180ba..d1de84b07e 100644 --- a/src/sqlbuild/integrations/dbt/types.py +++ b/src/sqlbuild/integrations/dbt/types.py @@ -13,17 +13,6 @@ class DbtInvoker(Protocol): def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... -class DbtSupportedResourceType(StrEnum): - """dbt resource types SQLBuild currently handles with dedicated behavior.""" - - MODEL = "model" - SEED = "seed" - SNAPSHOT = "snapshot" - SOURCE = "source" - TEST = "test" - UNIT_TEST = "unit_test" - - class DbtInteropCommand(StrEnum): """dbt interop commands with SQLBuild participation.""" diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index a06117ca75..b31cf9d1f9 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -4,8 +4,6 @@ DEFAULT_MAX_DISPLAY_ENTRIES: int = 20 ERROR_STATUS_LABELS: frozenset[str] = frozenset({"ERROR", "FAIL", "FAILED"}) -PHASE_FAIL_GLYPH: str = "\u2717" -PHASE_OK_GLYPH: str = "\u2713" SKIPPED_STATUS_LABELS: frozenset[str] = frozenset({"SKIP", "SKIPPED"}) SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" diff --git a/src/sqlbuild/sql_values/constants.py b/src/sqlbuild/sql_values/constants.py index 472af39e48..6fee16de1b 100644 --- a/src/sqlbuild/sql_values/constants.py +++ b/src/sqlbuild/sql_values/constants.py @@ -1,8 +1,5 @@ """Constants for typed SQL value validation.""" -MIN_SIGNED_64_BIT_INTEGER: int = -(2**63) -MAX_SIGNED_64_BIT_INTEGER: int = 2**63 - 1 DEFAULT_MAX_SQL_VALUE_DEPTH: int = 32 DEFAULT_MAX_SQL_VALUE_ELEMENTS: int = 10_000 DEFAULT_MAX_SQL_VALUE_SIZE: int = 1_000_000 -MISSING_SQL_VALUE: object = object() From 1bd9c3e068ff44c43bd68d190f8bb67d16334da4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:09:14 +0100 Subject: [PATCH 12/33] refactor: own compile reuse rules and notices as native constants --- .../_helpers/compile_command/constants.rs | 69 +++++++++ .../bindings/_helpers/compile_command/mod.rs | 1 + .../_helpers/compile_command/reuse_attempt.rs | 146 +++++++++--------- .../compile_command/reuse_recording.rs | 77 ++++----- src/sqlbuild/_native.pyi | 4 + src/sqlbuild/cli/compile_reuse/constants.py | 76 +-------- tests/conftest.py | 6 +- .../main/compile/fast_lineage/helpers.py | 4 +- .../cli/commands/main/compile/helpers.py | 22 +-- .../main/compile/semantic_checks/helpers.py | 4 +- .../cli/commands/main/scenario/helpers.py | 4 +- .../commands/main/compile_outputs/helpers.py | 11 +- .../main/test_startup_import_footprint.py | 4 +- .../src/sqlbuild/cli/entry/helpers.py | 4 +- .../_helpers/test_module_paths.py | 1 + .../cli/compile_reuse/main/helpers.py | 4 +- 16 files changed, 214 insertions(+), 223 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs new file mode 100644 index 0000000000..a72ed97f9f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -0,0 +1,69 @@ +//! Compile reuse storage, fingerprint, environment and notice constants. + +/// The directory under the compiler cache directory holding the native reuse store. +pub(crate) const NATIVE_REUSE_DIRECTORY_NAME: &str = "project-reuse-v2"; +/// The reuse directory an older release kept stored compiles in. +pub(crate) const RETIRED_REUSE_DIRECTORY_NAME: &str = "project-reuse-v1"; +pub(crate) const REUSE_MAX_ENTRY_BYTES: i64 = 512 * 1024 * 1024; +pub(crate) const REUSE_MAX_STORED_ENTRIES: i64 = 8; +pub(crate) const REUSE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_REUSE"; +pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; +pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; +pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; +/// Sorted; tracked-prefix variables that never invalidate a stored compile. +pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = [ + "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR", + "SQLBUILD_DISABLE_COMPILE_REUSE", +]; +pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; +pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; +pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = + ["sqlbuild_project.toml", "sqlbuild_project.yml"]; +/// The `sqlbuild._native` attribute the reuse check reads its racy window from. +pub(crate) const RACY_WINDOW_ATTRIBUTE: &str = "COMPILE_REUSE_RACY_WINDOW_NS"; +/// The `sqlbuild._native` attribute the reuse check reads its excluded root directories from. +pub(crate) const EXCLUDED_ROOT_ATTRIBUTE: &str = "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES"; +pub(crate) const RACY_WINDOW_NS: i64 = 2_000_000_000; +pub(crate) const MISSING_PATH_MTIME_NS: i64 = -1; +pub(crate) const PROJECT_ROOT_PATH_MTIME_NS: i64 = -2; +pub(crate) const TARGET_DIRECTORY_NAME: &str = "target"; +pub(crate) const COMPILED_DIRECTORY_NAME: &str = "compiled"; +pub(crate) const MILLISECONDS_PER_SECOND: f64 = 1000.0; +/// Sorted; top-level project directories the reuse snapshot skips. +pub(crate) const EXCLUDED_ROOT_DIRECTORIES: [&str; 16] = [ + ".cache", + ".fensu", + ".hg", + ".idea", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".sqlbuild", + ".svn", + ".tox", + ".venv", + ".vscode", + "logs", + "target", + "venv", +]; +/// Sorted; directories the reuse snapshot skips at any depth. +pub(crate) const EXCLUDED_DIRECTORIES: [&str; 2] = [".git", "__pycache__"]; +pub(crate) const PRESENCE_ONLY_FILE_SUFFIXES: [&str; 2] = [".duckdb", ".duckdb.wal"]; +pub(crate) const REUSE_HIT_TIMING: &str = "project_reuse_hits"; +pub(crate) const REUSE_MISS_TIMING: &str = "project_reuse_misses"; +pub(crate) const REUSE_BYPASS_TIMING: &str = "project_reuse_bypasses"; +pub(crate) const REUSE_CHECK_TIMING: &str = "project_reuse_check_ms"; +pub(crate) const TOTAL_TIMING: &str = "total_ms"; +/// The reuse hit notice, split around its elapsed seconds. +pub(crate) const REUSE_HIT_MESSAGE: [&str; 2] = + ["Inputs unchanged; reused the previous compile (", " s)"]; +pub(crate) const REUSE_STORE_NOTICE_BYTES: u64 = 64 * 1024 * 1024; +pub(crate) const REUSE_STORE_NOTICE_PATHS: usize = 100_000; +pub(crate) const BYTES_PER_MEBIBYTE: f64 = 1024.0 * 1024.0; +pub(crate) const REUSE_STORE_DONE_PREFIX: &str = "Recorded compile for reuse ("; +pub(crate) const REUSE_STORE_SKIPPED_PREFIX: &str = + "Did not record compile for reuse: files changed while it ran ("; +pub(crate) const COMPUTE_LOG_DIRECTORY_NAME: &str = "logs"; +pub(crate) const COMPILE_COMMAND_NAME: &str = "compile"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index a89f776184..2e471ccf9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod artifact_batch; +pub(crate) mod constants; pub(crate) mod prepared_artifacts; pub(crate) mod registration; pub(crate) mod report; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 30ba913b70..01c9ce461e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -6,18 +6,30 @@ use pyo3::exceptions::PyOSError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; +use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; + use crate::bindings::_helpers::cli::launch_state::launch_record_path; +use crate::bindings::_helpers::compile_command::constants::{ + COMPILE_COMMAND_NAME, COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, + EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_ATTRIBUTE, EXCLUDED_ROOT_DIRECTORIES, + MILLISECONDS_PER_SECOND, MISSING_ENVIRONMENT_VALUE, MISSING_FILE_DIGEST, MISSING_PATH_MTIME_NS, + NATIVE_REUSE_DIRECTORY_NAME, PRESENCE_ONLY_FILE_SUFFIXES, PROJECT_CONFIG_FILENAMES, + PROJECT_ROOT_PATH_MTIME_NS, RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS, + RETIRED_REUSE_DIRECTORY_NAME, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_DISABLE_ENV_VAR, + REUSE_DISABLE_VALUE, REUSE_HIT_MESSAGE, REUSE_HIT_TIMING, REUSE_LOGGER_NAME, + REUSE_MAX_ENTRY_BYTES, REUSE_MAX_STORED_ENTRIES, REUSE_MISS_TIMING, TARGET_DIRECTORY_NAME, + TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, +}; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REUSE_CONSTANTS: &str = "sqlbuild.cli.compile_reuse.constants"; const REUSE_MODELS: &str = "sqlbuild.cli.compile_reuse.models"; const REUSE_TYPES: &str = "sqlbuild.cli.compile_reuse.types"; const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; @@ -26,40 +38,35 @@ const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; -const SECONDS_FIELD: &str = "{seconds:.1f}"; const HIT: &str = "HIT"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; const MISS: &str = "MISS"; const BYPASS: &str = "BYPASS"; -fn constant<'py>(py: Python<'py>, name: &str) -> PyResult> { - imported(py, REUSE_CONSTANTS, name) -} - -fn sorted<'py>(value: Bound<'py, PyAny>) -> PyResult> { - imported(value.py(), "builtins", "sorted")?.call1((value,)) -} - fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) } fn reuse_logger(py: Python<'_>) -> PyResult> { - imported(py, "logging", "getLogger")?.call1((constant(py, "REUSE_LOGGER_NAME")?,)) + imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } /// The reuse constants the native check, the stored compile and the launch record share. pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult> { let rules: Bound<'_, PyDict> = PyDict::new(py); - rules.set_item("excluded", sorted(constant(py, "EXCLUDED_DIRECTORIES")?)?)?; + rules.set_item("excluded", PyList::new(py, EXCLUDED_DIRECTORIES)?)?; rules.set_item( "excluded_root", - sorted(constant(py, "EXCLUDED_ROOT_DIRECTORIES")?)?, + imported(py, "builtins", "sorted")?.call1((imported( + py, + NATIVE_MODULE, + EXCLUDED_ROOT_ATTRIBUTE, + )?,))?, )?; rules.set_item( "presence_suffixes", - listed(constant(py, "PRESENCE_ONLY_FILE_SUFFIXES")?)?, + PyList::new(py, PRESENCE_ONLY_FILE_SUFFIXES)?, )?; let output_files: Bound<'_, PyAny> = imported(py, RUNTIME_IDENTITY_MODULE, "redirected_output_files")?.call0()?; @@ -68,11 +75,11 @@ pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult, with_output_files: bool) -> PyResult( .is_instance(&imported(py, COMPUTE_LOG_TEE_MODULE, "TextComputeLogTee")?)? { let logs: Bound<'py, PyAny> = resolved - .call_method1( - "__truediv__", - (constant(py, "COMPUTE_LOG_DIRECTORY_NAME")?,), - )? + .call_method1("__truediv__", (COMPUTE_LOG_DIRECTORY_NAME,))? .call_method0("resolve")?; let capture: Bound<'py, PyDict> = PyDict::new(py); capture.set_item("root", logs.str()?)?; - capture.set_item("command", constant(py, "COMPILE_COMMAND_NAME")?)?; + capture.set_item("command", COMPILE_COMMAND_NAME)?; capture.set_item("project_dir", resolved.str()?)?; capture.set_item("target", attempt.getattr("selected_target")?)?; capture.into_any() } else { py.None().into_bound(py) }; - let hit_message: Bound<'py, PyAny> = - constant(py, "REUSE_HIT_MESSAGE")?.call_method1("split", (SECONDS_FIELD,))?; let timings: Bound<'py, PyDict> = PyDict::new(py); for (key, name) in [ - ("hit", "REUSE_HIT_TIMING"), - ("miss", "REUSE_MISS_TIMING"), - ("bypass", "REUSE_BYPASS_TIMING"), - ("check", "REUSE_CHECK_TIMING"), - ("total", "TOTAL_TIMING"), + ("hit", REUSE_HIT_TIMING), + ("miss", REUSE_MISS_TIMING), + ("bypass", REUSE_BYPASS_TIMING), + ("check", REUSE_CHECK_TIMING), + ("total", TOTAL_TIMING), ] { - timings.set_item(key, constant(py, name)?)?; + timings.set_item(key, name)?; } let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item("format", constant(py, "LAUNCH_RECORD_FORMAT")?)?; + payload.set_item("format", LAUNCH_RECORD_FORMAT)?; payload.set_item("key", key)?; payload.set_item("project_dir", project_dir.str()?)?; payload.set_item("store_path", native.call_method0("store_path")?)?; @@ -180,7 +185,7 @@ fn launch_payload<'py>( payload.set_item("json_output", attempt.getattr("json_output")?)?; payload.set_item("rules", reuse_rules(py, false)?)?; payload.set_item("capture", capture)?; - payload.set_item("hit_message", listed(hit_message)?)?; + payload.set_item("hit_message", PyList::new(py, REUSE_HIT_MESSAGE)?)?; payload.set_item("timings", timings)?; Ok(payload) } @@ -213,8 +218,8 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let mut directory: Bound<'_, PyAny> = attempt.getattr("project_dir")?; - for part in constant(py, "LAUNCH_RECORD_DIRECTORY")?.try_iter()? { - directory = directory.call_method1("__truediv__", (part?,))?; + for part in LAUNCH_RECORD_DIRECTORY { + directory = directory.call_method1("__truediv__", (part,))?; } if path .getattr("parent")? @@ -265,10 +270,7 @@ fn reuse_check<'py>( "store_directory", imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? .call1((imported(py, "pathlib", "Path")?.call1((project_dir,))?,))? - .call_method1( - "__truediv__", - (constant(py, "NATIVE_REUSE_DIRECTORY_NAME")?,), - )?, + .call_method1("__truediv__", (NATIVE_REUSE_DIRECTORY_NAME,))?, )?; check.set_item("selected_target", request.getattr("selected_target")?)?; check.set_item( @@ -330,16 +332,13 @@ fn replayed<'py>( ) -> PyResult> { let py: Python<'py> = native.py(); let check_ms: i64 = elapsed_ms(py, started)?; - let mut timings: Vec<(Bound<'py, PyAny>, i64)> = Vec::new(); - for (name, counted) in [ - ("REUSE_HIT_TIMING", 1), - ("REUSE_MISS_TIMING", 0), - ("REUSE_BYPASS_TIMING", 0), - ("REUSE_CHECK_TIMING", check_ms), - ] { - timings.push((constant(py, name)?, counted)); - } - timings.push((constant(py, "TOTAL_TIMING")?, elapsed_ms(py, started)?)); + let timings: Vec<(&str, i64)> = vec![ + (REUSE_HIT_TIMING, 1), + (REUSE_MISS_TIMING, 0), + (REUSE_BYPASS_TIMING, 0), + (REUSE_CHECK_TIMING, check_ms), + (TOTAL_TIMING, elapsed_ms(py, started)?), + ]; let rows: Bound<'py, PyList> = if json_output { PyList::new(py, timings)? } else { @@ -357,16 +356,11 @@ fn replayed<'py>( Bound<'py, PyAny>, Bound<'py, PyAny>, ) = replay.extract()?; - let seconds: Bound<'py, PyAny> = check_ms - .into_bound_py_any(py)? - .call_method1("__truediv__", (constant(py, "MILLISECONDS_PER_SECOND")?,))?; + let [prefix, suffix] = REUSE_HIT_MESSAGE; + let seconds: f64 = check_ms as f64 / MILLISECONDS_PER_SECOND; print_stderr( py, - &constant(py, "REUSE_HIT_MESSAGE")?.call_method( - "format", - (), - Some(&keywords(py, &[("seconds", seconds)])?), - )?, + &format!("{prefix}{seconds:.1}{suffix}").into_bound_py_any(py)?, )?; for line in stderr_lines.try_iter()? { print_stderr(py, &line?)?; @@ -503,6 +497,12 @@ fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult> } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + module.add(RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS)?; + module.add( + EXCLUDED_ROOT_ATTRIBUTE, + PyFrozenSet::new(py, EXCLUDED_ROOT_DIRECTORIES)?, + )?; module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?)?; module.add_function(wrap_pyfunction!(run_compile, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 9908bd9144..c0f5edb951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -8,13 +8,17 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::bindings::_helpers::compile_command::constants::{ + BYTES_PER_MEBIBYTE, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_HIT_TIMING, + REUSE_LOGGER_NAME, REUSE_MISS_TIMING, REUSE_STORE_DONE_PREFIX, REUSE_STORE_NOTICE_BYTES, + REUSE_STORE_NOTICE_PATHS, REUSE_STORE_SKIPPED_PREFIX, +}; use crate::bindings::_helpers::compile_command::reuse_attempt::{reuse_rules, write_launch_record}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::compile_outputs::reuse::artifact_digest; -const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; const PROVIDER_SETTINGS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.provider_settings"; const MACRO_STORE_PATHS_MODULE: &str = "sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"; @@ -260,28 +264,18 @@ pub(crate) fn compile_reuse_timings<'py>( let outcome: String = attempt.getattr("outcome")?.str()?.to_string(); let timings: Bound<'py, PyDict> = PyDict::new(py); for (name, counted) in [ - ("REUSE_HIT_TIMING", HIT_OUTCOME), - ("REUSE_MISS_TIMING", MISS_OUTCOME), - ("REUSE_BYPASS_TIMING", BYPASS_OUTCOME), + (REUSE_HIT_TIMING, HIT_OUTCOME), + (REUSE_MISS_TIMING, MISS_OUTCOME), + (REUSE_BYPASS_TIMING, BYPASS_OUTCOME), ] { - timings.set_item( - imported(py, REUSE_CONSTANTS_MODULE, name)?, - i64::from(outcome == counted), - )?; + timings.set_item(name, i64::from(outcome == counted))?; } - timings.set_item( - imported(py, REUSE_CONSTANTS_MODULE, "REUSE_CHECK_TIMING")?, - attempt.getattr("check_ms")?, - )?; + timings.set_item(REUSE_CHECK_TIMING, attempt.getattr("check_ms")?)?; Ok(timings) } fn reuse_logger(py: Python<'_>) -> PyResult> { - imported(py, "logging", "getLogger")?.call1((imported( - py, - REUSE_CONSTANTS_MODULE, - "REUSE_LOGGER_NAME", - )?,)) + imported(py, "logging", "getLogger")?.call1((REUSE_LOGGER_NAME,)) } /// Loaded module files, then the macro store's extra module paths, as Python lists them. @@ -315,46 +309,29 @@ fn module_paths(py: Python<'_>) -> PyResult> { /// The notice a long reuse record prints first, with when it started. fn start_notice(py: Python<'_>, native: &Bound<'_, PyAny>) -> PyResult> { - let pending: i64 = native.call_method0("pending_digest_bytes")?.extract()?; - let paths: i64 = native.call_method0("snapshot_paths")?.extract()?; - let constant = |name: &str| imported(py, REUSE_CONSTANTS_MODULE, name); - if pending < constant("REUSE_STORE_NOTICE_BYTES")?.extract::()? - && paths < constant("REUSE_STORE_NOTICE_PATHS")?.extract::()? - { + let pending: u64 = native.call_method0("pending_digest_bytes")?.extract()?; + let paths: usize = native.call_method0("snapshot_paths")?.extract()?; + if pending < REUSE_STORE_NOTICE_BYTES && paths < REUSE_STORE_NOTICE_PATHS { return Ok(None); } - let mebibytes: Bound<'_, PyAny> = pending - .into_bound_py_any(py)? - .call_method1("__truediv__", (constant("BYTES_PER_MEBIBYTE")?,))?; - let message: Bound<'_, PyAny> = constant("REUSE_STORE_START_MESSAGE")?.call_method( - "format", - (), - Some(&keywords( - py, - &[ - ("paths", paths.into_bound_py_any(py)?), - ("mebibytes", mebibytes), - ], - )?), - )?; - print_to(py, "stderr", &message)?; + let mebibytes: f64 = pending as f64 / BYTES_PER_MEBIBYTE; + let message: String = + format!("Recording compile for reuse ({paths} paths, {mebibytes:.0} MiB to hash)..."); + print_to(py, "stderr", &message.into_bound_py_any(py)?)?; Ok(Some(imported(py, "time", "monotonic")?.call0()?.extract()?)) } -fn finish_notice(py: Python<'_>, started: Option, message: &str) -> PyResult<()> { +fn finish_notice(py: Python<'_>, started: Option, prefix: &str) -> PyResult<()> { let Some(started) = started else { return Ok(()); }; let now: f64 = imported(py, "time", "monotonic")?.call0()?.extract()?; - let text: Bound<'_, PyAny> = imported(py, REUSE_CONSTANTS_MODULE, message)?.call_method( - "format", - (), - Some(&keywords( - py, - &[("seconds", (now - started).into_bound_py_any(py)?)], - )?), - )?; - print_to(py, "stderr", &text) + let seconds: f64 = now - started; + print_to( + py, + "stderr", + &format!("{prefix}{seconds:.1} s)").into_bound_py_any(py)?, + ) } /// What `write_reusable_compile` records about one full compile. @@ -478,9 +455,9 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { py, notice_started, if outcome == SKIPPED { - "REUSE_STORE_SKIPPED_MESSAGE" + REUSE_STORE_SKIPPED_PREFIX } else { - "REUSE_STORE_DONE_MESSAGE" + REUSE_STORE_DONE_PREFIX }, )?; Ok(outcome == STORED) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 27050ee081..acb9a5c99c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1654,6 +1654,10 @@ class NativeReuseAttempt: def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... def run_compile(request: CompileCommandRequest) -> int: ... + +COMPILE_REUSE_RACY_WINDOW_NS: int +COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES: frozenset[str] + def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ... class CompileArtifactWrites: diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index a6c9d8cca0..c115ceb35f 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -1,80 +1,6 @@ -"""Compile reuse storage, fingerprint, and environment constants.""" +"""Compile reuse runtime identity constants.""" from __future__ import annotations -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, -) - REUSE_FORMAT_VERSION: int = 1 -RETIRED_REUSE_DIRECTORY_NAME: str = "project-reuse-v1" -NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" -NATIVE_REUSE_SUFFIX: str = ".store" -REUSE_MAX_ENTRY_BYTES: int = 512 * 1024 * 1024 -REUSE_MAX_STORED_ENTRIES: int = 8 -REUSE_DISABLE_ENV_VAR: str = COMPILE_REUSE_DISABLE_ENV_VAR -REUSE_DISABLE_VALUE: str = "1" -REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" -TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") -UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - { - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, - } -) -MISSING_ENVIRONMENT_VALUE: str = "\0missing" -MISSING_FILE_DIGEST: str = "\0missing" -PROJECT_CONFIG_FILENAMES: tuple[str, ...] = ("sqlbuild_project.toml", "sqlbuild_project.yml") - -RACY_WINDOW_NS: int = 2_000_000_000 -MISSING_PATH_MTIME_NS: int = -1 -PROJECT_ROOT_PATH_MTIME_NS: int = -2 OUTPUT_FILE_DESCRIPTORS: tuple[int, ...] = (1, 2) - -TARGET_DIRECTORY_NAME: str = "target" -COMPILED_DIRECTORY_NAME: str = "compiled" -EXCLUDED_ROOT_DIRECTORIES: frozenset[str] = frozenset( - { - "logs", - "target", - "venv", - ".cache", - ".fensu", - ".hg", - ".idea", - ".mypy_cache", - ".nox", - ".pytest_cache", - ".ruff_cache", - ".sqlbuild", - ".svn", - ".tox", - ".venv", - ".vscode", - } -) -EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".git", "__pycache__"}) -PRESENCE_ONLY_FILE_SUFFIXES: tuple[str, ...] = (".duckdb", ".duckdb.wal") - -REUSE_HIT_TIMING: str = "project_reuse_hits" -REUSE_MISS_TIMING: str = "project_reuse_misses" -REUSE_BYPASS_TIMING: str = "project_reuse_bypasses" -REUSE_CHECK_TIMING: str = "project_reuse_check_ms" -TOTAL_TIMING: str = "total_ms" -REUSE_HIT_MESSAGE: str = "Inputs unchanged; reused the previous compile ({seconds:.1f} s)" -MILLISECONDS_PER_SECOND: int = 1000 -REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 -REUSE_STORE_NOTICE_PATHS: int = 100_000 -BYTES_PER_MEBIBYTE: int = 1024 * 1024 -REUSE_STORE_START_MESSAGE: str = ( - "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." -) -REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" -REUSE_STORE_SKIPPED_MESSAGE: str = ( - "Did not record compile for reuse: files changed while it ran ({seconds:.1f} s)" -) -LAUNCH_RECORD_FORMAT: int = 1 -LAUNCH_RECORD_DIRECTORY: tuple[str, ...] = (TARGET_DIRECTORY_NAME, "cache", "launcher-v1") -COMPUTE_LOG_DIRECTORY_NAME: str = "logs" -COMPILE_COMMAND_NAME: str = "compile" diff --git a/tests/conftest.py b/tests/conftest.py index 2427d418f2..d1b10120dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,11 +6,13 @@ import pytest -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) def pytest_configure(config: pytest.Config) -> None: """Exercise full compiles by default; compile reuse tests opt back in explicitly.""" del config - os.environ.setdefault(REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE) + os.environ.setdefault(REUSE_DISABLE_ENV_VAR, "1") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index 0ec74c34da..542142b45c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -7,7 +7,9 @@ from pathlib import Path from typing import NamedTuple, cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb _TIMING: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 90d7db2484..9d8d4409f8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -33,7 +33,6 @@ import sqlbuild._native as plan_artifacts_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings -import sqlbuild.cli.compile_reuse.constants as reuse_constants import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class @@ -45,14 +44,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.cli.compile_reuse.constants import ( - EXCLUDED_ROOT_DIRECTORIES, - NATIVE_REUSE_DIRECTORY_NAME, - NATIVE_REUSE_SUFFIX, - RETIRED_REUSE_DIRECTORY_NAME, - REUSE_DISABLE_ENV_VAR, - REUSE_LOGGER_NAME, -) from sqlbuild.compiler.analysis_session.constants import NATIVE_ANALYSIS_STORE_FILE_NAME from sqlbuild.compiler.compile.constants import ( RETIRED_FACT_CACHE_DIRECTORY_NAME, @@ -60,6 +51,9 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts.models import ContractValidationResult +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from sqlbuild.compiler.frontier.constants import ( COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, @@ -83,6 +77,10 @@ contract_statuses, ) +NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" +NATIVE_REUSE_SUFFIX: str = ".store" +RETIRED_REUSE_DIRECTORY_NAME: str = "project-reuse-v1" +REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" _DBT_SHAPED_SQL_SIZE_PROFILE: tuple[tuple[float, int], ...] = ( (0.50, 1_800), (0.75, 4_500), @@ -3395,7 +3393,7 @@ def record_digested_files(*, monkeypatch: pytest.MonkeyPatch) -> list[int]: def settle_racy_window(*, monkeypatch: pytest.MonkeyPatch) -> None: """Treat files written before a compile starts as settled instead of waiting two seconds.""" - monkeypatch.setattr(reuse_constants, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(native_module, "COMPILE_REUSE_RACY_WINDOW_NS", 0) def write_large_file(*, project_dir: Path, relative_path: str, size_bytes: int) -> Path: @@ -4219,7 +4217,9 @@ def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: """Make compile reuse stop watching the models folder, so a model edit goes unseen.""" monkeypatch.setattr( - reuse_constants, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} + native_module, + "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES", + native_module.COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES | {"models"}, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py index 57a431ed34..22e2a5d2bb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py @@ -6,7 +6,9 @@ from pathlib import Path from typing import Any, NamedTuple, cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index eb59249a10..d3e3787872 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -14,7 +14,9 @@ from pathlib import Path from typing import IO, NamedTuple, cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_input_fingerprint, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index e24bb68e25..c3a4911d69 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -13,19 +13,18 @@ import sqlbuild._native as native_module from sqlbuild._native import PreparedCompileArtifacts from sqlbuild._native import run_cli as main -from sqlbuild.cli.compile_reuse import constants as reuse_constants_module from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity -from sqlbuild.cli.compile_reuse.constants import ( - NATIVE_REUSE_DIRECTORY_NAME, - NATIVE_REUSE_SUFFIX, - REUSE_DISABLE_ENV_VAR, +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, ) from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory STAGING_PREFIX: str = ".sqlbuild-staging-" +NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" +NATIVE_REUSE_SUFFIX: str = ".store" class _StagingEveryProject(PreparedCompileArtifacts): @@ -110,7 +109,7 @@ def prepare_output_compiles(*, monkeypatch: pytest.MonkeyPatch, reuse_disabled: monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, reuse_disabled) monkeypatch.setattr(native_module, "PreparedCompileArtifacts", _StagingEveryProject) - monkeypatch.setattr(reuse_constants_module, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(native_module, "COMPILE_REUSE_RACY_WINDOW_NS", 0) def main_compile_json(*, project_dir: Path) -> int: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index c170be2c02..5a2ae9f03a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -11,7 +11,9 @@ import pytest -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ReusedCompileImportFootprintTestCase, StartupImportFootprintTestCase, diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index dc5bba1d5b..9e25be6485 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -7,7 +7,9 @@ from pathlib import Path import sqlbuild -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) type LauncherRun = tuple[int, bytes, bytes] diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 468c2c0081..5f6df47ca4 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -114,6 +114,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli.commands.classes.prepared_compile_artifacts", "sqlbuild.cli.observability", "sqlbuild.compile", + "sqlbuild.compile.reuse", "sqlbuild.cost", "sqlbuild.executor.runtime", "sqlbuild.fingerprints", diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py index b322238284..356455b32f 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py @@ -9,7 +9,9 @@ from pathlib import Path from typing import cast -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, +) def compile_in_fresh_process( From 43eb1a5047302eeae1d74f059dd0afe5ad4e31f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:15:07 +0100 Subject: [PATCH 13/33] refactor: own build, dbt and warehouse-group command constants natively --- .../src/command_connection/constants.rs | 32 ++++++++ .../src/build_command/constants.rs | 16 ++++ .../_helpers/build_command/defer_clone.rs | 6 +- .../_helpers/build_command/planning.rs | 19 +++-- .../command_runtime/warehouse_scope.rs | 20 +++-- .../_helpers/compile_driver/phases.rs | 5 +- .../_helpers/dbt_interop/sqlbuild_work.rs | 17 ++-- src/sqlbuild/cli/commands/constants.py | 78 ------------------- .../_helpers/build_planning/helpers.py | 22 ++++-- .../build_planning/test_execution_limits.py | 4 +- .../commands/_helpers/runtime/_test_types.py | 7 -- .../_helpers/runtime/test_warehouses.py | 20 +---- .../cli/commands/main/entry/conftest.py | 2 +- 13 files changed, 109 insertions(+), 139 deletions(-) delete mode 100644 src/sqlbuild/cli/commands/constants.py diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 3fd864dcbd..969eeb4ed6 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -22,3 +22,35 @@ pub(crate) const UNSUPPORTED_WAREHOUSE_CODE: &str = "C260"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; +/// The warehouse group each CLI command selects a target's default warehouse from. +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 29] = [ + ("audit", Some("build")), + ("build", Some("build")), + ("check", Some("build")), + ("clone", Some("build")), + ("dbt", Some("build")), + ("load", Some("build")), + ("plan", Some("build")), + ("seed", Some("build")), + ("contract", Some("query")), + ("cost", Some("query")), + ("debug", Some("query")), + ("diff", Some("query")), + ("freshness", Some("query")), + ("query", Some("query")), + ("janitor", Some("query")), + ("scenario", Some("query")), + ("test", Some("query")), + ("clean", None), + ("compile", None), + ("dag", None), + ("format", None), + ("init", None), + ("lineage", None), + ("mv", None), + ("playground", None), + ("rename", None), + ("rules", None), + ("scope", None), + ("skills", None), +]; diff --git a/crates/sqlbuild-core/src/build_command/constants.rs b/crates/sqlbuild-core/src/build_command/constants.rs index e4672a7a50..a7f88bc604 100644 --- a/crates/sqlbuild-core/src/build_command/constants.rs +++ b/crates/sqlbuild-core/src/build_command/constants.rs @@ -53,3 +53,19 @@ pub(crate) const PYTHON_NODE_FAILED: &str = "failed"; pub(crate) const PYTHON_STATUS_OK: &str = "OK"; pub(crate) const PYTHON_STATUS_SKIP: &str = "SKIP"; pub(crate) const PYTHON_STATUS_FAIL: &str = "FAIL"; + +pub const MISSING_ORIGIN_BUILD_HELP: &str = + "Pass --allow-missing-migration-origin to confirm in non-interactive runs."; +pub const MISSING_ORIGIN_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected models with sqb build --allow-missing-migration-origin and leave them out of this dbt selection, or set the target's missing_migration_origin to 'allow'."; +pub const SNAPSHOT_FULL_REFRESH_BUILD_HELP: &str = + "Pass --allow-snapshot-full-refresh to confirm in non-interactive runs."; +pub const SNAPSHOT_FULL_REFRESH_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected snapshots with sqb build --allow-snapshot-full-refresh and leave them out of this dbt selection, or relax the snapshot_full_refresh policy."; +pub const TABLE_TYPE_DOWNGRADE_BUILD_HELP: &str = + "Pass --allow-table-type-downgrade to confirm in non-interactive runs."; +pub const TABLE_TYPE_DOWNGRADE_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected models with sqb build --allow-table-type-downgrade and leave them out of this dbt selection, or set the target's table_type_downgrade to 'allow'."; +pub const RETENTION_DECREASE_BUILD_HELP: &str = + "Pass --allow-retention-decrease to confirm in non-interactive runs."; +pub const RETENTION_DECREASE_DBT_HELP: &str = "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected models with sqb build --allow-retention-decrease and leave them out of this dbt selection, or set the target's time_travel_retention_decrease to 'allow'."; +pub const EXECUTION_LIMIT_BUILD_NOTE: &str = "No warehouse changes were made."; +pub const EXECUTION_LIMIT_DBT_NOTE: &str = + "dbt work has already run; no SQLBuild models were built."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 6b41b598c7..1165ddcea3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -6,7 +6,8 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::constants::EXECUTION_LIMIT_BUILD_NOTE; use sqlbuild_core::build_command::errors::BuildCommandError; @@ -15,7 +16,6 @@ use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; @@ -633,7 +633,7 @@ pub(crate) fn run_defer_clone_boundary_prephase( ("limits", invocation.getattr("execution_limits")?), ( "refusal_note", - imported(py, CONSTANTS_MODULE, "EXECUTION_LIMIT_BUILD_NOTE")?, + EXECUTION_LIMIT_BUILD_NOTE.into_bound_py_any(py)?, ), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 64be5880e4..ddc33b18dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -1,6 +1,10 @@ //! Compile the build plan, show it, gate it and write the compile target. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_core::build_command::constants::{ + EXECUTION_LIMIT_BUILD_NOTE, MISSING_ORIGIN_BUILD_HELP, RETENTION_DECREASE_BUILD_HELP, + SNAPSHOT_FULL_REFRESH_BUILD_HELP, TABLE_TYPE_DOWNGRADE_BUILD_HELP, +}; use sqlbuild_core::build_command::models::PlanSafetyGates; use crate::bindings::_helpers::build_command::defer_clone::run_defer_clone_boundary_prephase; @@ -13,7 +17,6 @@ use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; @@ -148,10 +151,6 @@ pub(crate) fn write_plan_text( emit(&stream, &format!("\n{plan_text}\n\n")) } -fn constant(py: Python<'_>, name: &str) -> PyResult { - imported(py, CONSTANTS_MODULE, name)?.extract() -} - /// Apply every plan safety gate with the build's allow flags. pub(crate) fn enforce_build_plan_policies( context: &BuildContext<'_>, @@ -163,11 +162,11 @@ pub(crate) fn enforce_build_plan_policies( allow_snapshot_full_refresh: context.request_flag("allow_snapshot_full_refresh")?, allow_table_type_downgrade: context.request_flag("allow_table_type_downgrade")?, allow_retention_decrease: context.request_flag("allow_retention_decrease")?, - missing_origin_help: constant(py, "MISSING_ORIGIN_BUILD_HELP")?, - snapshot_full_refresh_help: constant(py, "SNAPSHOT_FULL_REFRESH_BUILD_HELP")?, - table_type_downgrade_help: constant(py, "TABLE_TYPE_DOWNGRADE_BUILD_HELP")?, - retention_decrease_help: constant(py, "RETENTION_DECREASE_BUILD_HELP")?, - execution_limit_note: constant(py, "EXECUTION_LIMIT_BUILD_NOTE")?, + missing_origin_help: MISSING_ORIGIN_BUILD_HELP.to_owned(), + snapshot_full_refresh_help: SNAPSHOT_FULL_REFRESH_BUILD_HELP.to_owned(), + table_type_downgrade_help: TABLE_TYPE_DOWNGRADE_BUILD_HELP.to_owned(), + retention_decrease_help: RETENTION_DECREASE_BUILD_HELP.to_owned(), + execution_limit_note: EXECUTION_LIMIT_BUILD_NOTE.to_owned(), }; let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); enforce_plan_policies( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index ae3c52c241..53167885f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -7,11 +7,13 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_config::command_connection::constants::COMMAND_WAREHOUSE_GROUPS; + use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; -const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; +const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild.spec.contracts.types", "WarehouseGroup"); const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; @@ -65,10 +67,18 @@ pub(crate) fn command_warehouse_scope<'py>( let group: Bound<'py, PyAny> = if command.is_none() { py.None().into_bound(py) } else { - imported(py, COMMAND_CONSTANTS, "COMMAND_WAREHOUSE_GROUPS")?.call_method1( - "get", - (imported(py, COMMAND_TYPES, "CliCommand")?.call1((command,))?,), - )? + let name: String = imported(py, COMMAND_TYPES, "CliCommand")? + .call1((command,))? + .str()? + .to_string(); + match COMMAND_WAREHOUSE_GROUPS + .iter() + .find(|(candidate, _)| *candidate == name) + .and_then(|(_, group)| *group) + { + Some(group) => imported(py, WAREHOUSE_GROUP.0, WAREHOUSE_GROUP.1)?.call1((group,))?, + None => py.None().into_bound(py), + } }; let vars: Bound<'py, PyAny> = match cli_vars.filter(|cli_vars| !cli_vars.is_none()) { Some(cli_vars) => py.get_type::().call1((cli_vars,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 5f8f1e40c2..abd42d48c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -20,7 +20,7 @@ const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const CLI_CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; +const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; const DEFAULT_DAG_FILE: &str = "sqlbuild_dag.json"; @@ -90,8 +90,7 @@ pub(crate) fn resolved_dag_path<'py>( dag_path: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let empty: Bound<'py, PyAny> = imported(py, CLI_CONSTANTS_MODULE, "EMPTY_DAG_PATH")?; - if dag_path.eq(&empty)? { + if dag_path.eq(EMPTY_DAG_PATH)? { return target_dir(project_dir)?.call_method1("__truediv__", (DEFAULT_DAG_FILE,)); } let path: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?.call1((dag_path,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 2c82dfb7d8..2746bcd6c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -2,7 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::constants::{ + EXECUTION_LIMIT_DBT_NOTE, MISSING_ORIGIN_DBT_HELP, RETENTION_DECREASE_DBT_HELP, + SNAPSHOT_FULL_REFRESH_DBT_HELP, TABLE_TYPE_DOWNGRADE_DBT_HELP, +}; use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, keyword_relay, none, text, tuple, @@ -24,7 +28,6 @@ const WORK_OPTIONS: (&[&str], &[&str]) = ( ); const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; /// Interop has no allow flags; destructive work uses the ordinary interactive safety gates. #[pyfunction] @@ -38,23 +41,23 @@ fn dbt_interop_plan_safety_gates(py: Python<'_>) -> PyResult> { ("allow_retention_decrease", boolean(py, false)), ( "missing_origin_help", - imported(py, COMMAND_CONSTANTS, "MISSING_ORIGIN_DBT_HELP")?, + MISSING_ORIGIN_DBT_HELP.into_bound_py_any(py)?, ), ( "snapshot_full_refresh_help", - imported(py, COMMAND_CONSTANTS, "SNAPSHOT_FULL_REFRESH_DBT_HELP")?, + SNAPSHOT_FULL_REFRESH_DBT_HELP.into_bound_py_any(py)?, ), ( "table_type_downgrade_help", - imported(py, COMMAND_CONSTANTS, "TABLE_TYPE_DOWNGRADE_DBT_HELP")?, + TABLE_TYPE_DOWNGRADE_DBT_HELP.into_bound_py_any(py)?, ), ( "retention_decrease_help", - imported(py, COMMAND_CONSTANTS, "RETENTION_DECREASE_DBT_HELP")?, + RETENTION_DECREASE_DBT_HELP.into_bound_py_any(py)?, ), ( "execution_limit_note", - imported(py, COMMAND_CONSTANTS, "EXECUTION_LIMIT_DBT_NOTE")?, + EXECUTION_LIMIT_DBT_NOTE.into_bound_py_any(py)?, ), ], ) diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py deleted file mode 100644 index 2be1351002..0000000000 --- a/src/sqlbuild/cli/commands/constants.py +++ /dev/null @@ -1,78 +0,0 @@ -"""CLI command decision constants.""" - -from types import MappingProxyType - -from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.spec.contracts.types import WarehouseGroup - -EMPTY_DAG_PATH: str = "" -TARGET_DIRECTORY_NAME: str = "target" -SQLBUILD_CONCURRENCY_ENV_VAR: str = "SQLBUILD_CONCURRENCY" -SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" -MISSING_ORIGIN_BUILD_HELP: str = ( - "Pass --allow-missing-migration-origin to confirm in non-interactive runs." -) -MISSING_ORIGIN_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "models with sqb build --allow-missing-migration-origin and leave them out of this dbt " - "selection, or set the target's missing_migration_origin to 'allow'." -) -SNAPSHOT_FULL_REFRESH_BUILD_HELP: str = ( - "Pass --allow-snapshot-full-refresh to confirm in non-interactive runs." -) -SNAPSHOT_FULL_REFRESH_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "snapshots with sqb build --allow-snapshot-full-refresh and leave them out of this dbt " - "selection, or relax the snapshot_full_refresh policy." -) -TABLE_TYPE_DOWNGRADE_BUILD_HELP: str = ( - "Pass --allow-table-type-downgrade to confirm in non-interactive runs." -) -TABLE_TYPE_DOWNGRADE_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "models with sqb build --allow-table-type-downgrade and leave them out of this dbt " - "selection, or set the target's table_type_downgrade to 'allow'." -) -RETENTION_DECREASE_BUILD_HELP: str = ( - "Pass --allow-retention-decrease to confirm in non-interactive runs." -) -RETENTION_DECREASE_DBT_HELP: str = ( - "sqb dbt has no confirmation flag: confirm on an interactive terminal, build the affected " - "models with sqb build --allow-retention-decrease and leave them out of this dbt " - "selection, or set the target's time_travel_retention_decrease to 'allow'." -) -EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." -EXECUTION_LIMIT_DBT_NOTE: str = "dbt work has already run; no SQLBuild models were built." -COMMAND_WAREHOUSE_GROUPS: MappingProxyType[CliCommand, WarehouseGroup | None] = MappingProxyType( - { - CliCommand.AUDIT: WarehouseGroup.BUILD, - CliCommand.BUILD: WarehouseGroup.BUILD, - CliCommand.CHECK: WarehouseGroup.BUILD, - CliCommand.CLONE: WarehouseGroup.BUILD, - CliCommand.DBT: WarehouseGroup.BUILD, - CliCommand.LOAD: WarehouseGroup.BUILD, - CliCommand.PLAN: WarehouseGroup.BUILD, - CliCommand.SEED: WarehouseGroup.BUILD, - CliCommand.CONTRACT: WarehouseGroup.QUERY, - CliCommand.COST: WarehouseGroup.QUERY, - CliCommand.DEBUG: WarehouseGroup.QUERY, - CliCommand.DIFF: WarehouseGroup.QUERY, - CliCommand.FRESHNESS: WarehouseGroup.QUERY, - CliCommand.QUERY: WarehouseGroup.QUERY, - CliCommand.JANITOR: WarehouseGroup.QUERY, - CliCommand.SCENARIO: WarehouseGroup.QUERY, - CliCommand.TEST: WarehouseGroup.QUERY, - CliCommand.CLEAN: None, - CliCommand.COMPILE: None, - CliCommand.DAG: None, - CliCommand.FORMAT: None, - CliCommand.INIT: None, - CliCommand.LINEAGE: None, - CliCommand.MV: None, - CliCommand.PLAYGROUND: None, - CliCommand.RENAME: None, - CliCommand.RULES: None, - CliCommand.SCOPE: None, - CliCommand.SKILLS: None, - } -) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index aec230981d..9249d6654f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -4,13 +4,6 @@ from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.cli.commands.constants import ( - EXECUTION_LIMIT_BUILD_NOTE, - MISSING_ORIGIN_BUILD_HELP, - RETENTION_DECREASE_BUILD_HELP, - SNAPSHOT_FULL_REFRESH_BUILD_HELP, - TABLE_TYPE_DOWNGRADE_BUILD_HELP, -) from sqlbuild.cli.commands.models import PlanSafetyGates from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType @@ -123,3 +116,18 @@ def build_policy_gates( retention_decrease_help=RETENTION_DECREASE_BUILD_HELP, execution_limit_note=EXECUTION_LIMIT_BUILD_NOTE, ) + + +EXECUTION_LIMIT_BUILD_NOTE: str = "No warehouse changes were made." +MISSING_ORIGIN_BUILD_HELP: str = ( + "Pass --allow-missing-migration-origin to confirm in non-interactive runs." +) +RETENTION_DECREASE_BUILD_HELP: str = ( + "Pass --allow-retention-decrease to confirm in non-interactive runs." +) +SNAPSHOT_FULL_REFRESH_BUILD_HELP: str = ( + "Pass --allow-snapshot-full-refresh to confirm in non-interactive runs." +) +TABLE_TYPE_DOWNGRADE_BUILD_HELP: str = ( + "Pass --allow-table-type-downgrade to confirm in non-interactive runs." +) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py index 9084ca978e..b0cc3de61b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py @@ -1,12 +1,14 @@ import pytest from sqlbuild._native import enforce_model_execution_limit -from sqlbuild.cli.commands.constants import EXECUTION_LIMIT_BUILD_NOTE from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.spec.contracts.models import ExecutionLimitsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( ModelExecutionLimitTestCase, ) +from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning.helpers import ( + EXECUTION_LIMIT_BUILD_NOTE, +) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py index 31d2463627..b7cfa9bd13 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py @@ -89,13 +89,6 @@ class WarehouseFlagRejectedTestCase: expected_error_fragment: str = "unrecognized arguments: --warehouse" -@dataclass(frozen=True) -class CommandWarehouseClassificationTestCase: - description: str - command: CliCommand - expected_classified: bool = True - - @dataclass(frozen=True) class CommandWarehouseGroupTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index ef9873a880..d703c13970 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -8,11 +8,11 @@ from sqlbuild._native import ( command_warehouse_scope, + current_command_warehouse_scope, describe_connection_warehouse, parse_cli_invocation, resolve_project_connection_config, ) -from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand from sqlbuild.cli.entry.models import ParsedCliInvocation, ResolvedWarehouse @@ -20,7 +20,6 @@ from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig from sqlbuild.spec.contracts.types import WarehouseGroup from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( - CommandWarehouseClassificationTestCase, CommandWarehouseErrorTestCase, CommandWarehouseGroupTestCase, CommandWarehouseResolutionTestCase, @@ -263,20 +262,6 @@ def test_given_adapter_without_warehouses_when_cli_warehouse_is_given_then_it_fa assert test_case.expected_message_fragment in error.value.message -@pytest.mark.parametrize( - "test_case", - [ - CommandWarehouseClassificationTestCase(description=command.value, command=command) - for command in CliCommand - ], - ids=lambda case: case.description, -) -def test_given_cli_command_when_checking_warehouse_groups_then_it_is_classified( - test_case: CommandWarehouseClassificationTestCase, -) -> None: - assert (test_case.command in COMMAND_WAREHOUSE_GROUPS) == test_case.expected_classified - - @pytest.mark.parametrize( "test_case", [ @@ -380,7 +365,8 @@ def test_given_command_without_group_when_parsing_warehouse_flag_then_it_is_reje def test_given_cli_command_when_resolving_group_then_matches_its_workload( test_case: CommandWarehouseGroupTestCase, ) -> None: - assert COMMAND_WAREHOUSE_GROUPS[test_case.command] is test_case.expected_group + with command_warehouse_scope(command=test_case.command, cli_warehouse=None): + assert current_command_warehouse_scope().group is test_case.expected_group if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py index d48b040d55..6b1be16d45 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/conftest.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cli.commands.constants import SQLBUILD_CONCURRENCY_ENV_VAR +SQLBUILD_CONCURRENCY_ENV_VAR: str = "SQLBUILD_CONCURRENCY" @pytest.fixture(autouse=True) From 9c25c46d1c2a5c58e5a095501adb3ad3d1433ef8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:20:52 +0100 Subject: [PATCH 14/33] refactor: track cost ledger failures and the default rate source natively --- .../_helpers/cost_report/degradation.rs | 4 +- .../_helpers/cost_telemetry/health.rs | 55 +++++++++++++++++++ .../_helpers/cost_telemetry/ledger.rs | 13 +---- .../bindings/_helpers/cost_telemetry/mod.rs | 1 + .../_helpers/cost_telemetry/registration.rs | 3 +- .../_helpers/cost_telemetry/snowflake.rs | 10 +--- .../_helpers/cost_telemetry/values.rs | 1 - src/sqlbuild/_native.pyi | 6 ++ .../cost/classes/cost_telemetry_health.py | 17 ------ src/sqlbuild/cost/constants.py | 5 -- .../test_snowflake_connection.py | 3 +- .../commands/_helpers/cost/test_refresh.py | 3 +- .../unit/src/sqlbuild/cost/test_snowflake.py | 2 +- 13 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs delete mode 100644 src/sqlbuild/cost/classes/cost_telemetry_health.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 15fce9daa5..274a060f52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -19,7 +19,7 @@ const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; const COST_TYPES: &str = "sqlbuild.cost.types"; const COST_MODELS: &str = "sqlbuild.cost.models"; -const COST_CONSTANTS: &str = "sqlbuild.cost.constants"; +const DEFAULT_RATE_SOURCE: &str = "default"; const CONFIGURED_RATE_SOURCE: &str = "configured"; const STATEMENT_LEDGER_PATH: [&str; 4] = ["target", "executions", "", "statements.jsonl"]; const BUILD_BREAKDOWN_LIMIT: usize = 5; @@ -273,7 +273,7 @@ pub(crate) fn finalize_build_cost<'py>( } let config: Bound<'py, PyAny> = finalization.getattr("config")?; let rate_source: String = if config.getattr("usd_per_credit_is_default")?.is_truthy()? { - imported(py, COST_CONSTANTS, "DEFAULT_RATE_SOURCE")?.extract()? + DEFAULT_RATE_SOURCE.to_owned() } else { CONFIGURED_RATE_SOURCE.to_owned() }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs new file mode 100644 index 0000000000..9c2db29c23 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs @@ -0,0 +1,55 @@ +//! Process-local statement-ledger failure tracking for the active build. + +use std::collections::HashMap; +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyTypeMethods; +use pyo3::{pyclass, pymethods}; + +const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; + +static LEDGER_FAILURES: Mutex>> = Mutex::new(None); + +fn failures() -> MutexGuard<'static, Option>> { + LEDGER_FAILURES + .lock() + .unwrap_or_else(PoisonError::into_inner) +} + +/// Remember the first ledger failure of `run_id` by its exception type name. +pub(crate) fn mark_ledger_failure(run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { + let name: String = error.get_type().name()?.extract()?; + failures() + .get_or_insert_with(HashMap::new) + .entry(run_id) + .or_insert(name); + Ok(()) +} + +/// The ledger failure recorded for `run_id`, forgetting it. +pub(crate) fn consume_ledger_failure(run_id: &str) -> Option { + failures().as_mut()?.remove(run_id) +} + +/// The process-wide ledger failure tracker, exposed as `COST_TELEMETRY_HEALTH`. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CostTelemetryHealth; + +#[pymethods] +impl CostTelemetryHealth { + #[pyo3(signature = (*, run_id, error))] + fn mark_ledger_failure(&self, run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { + mark_ledger_failure(run_id, error) + } + + #[pyo3(signature = (*, run_id))] + fn consume_ledger_failure(&self, run_id: &str) -> Option { + consume_ledger_failure(run_id) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add(HEALTH_ATTRIBUTE, CostTelemetryHealth) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs index db37db2b86..25119a9b80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -18,8 +18,9 @@ use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJso use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; use crate::bindings::_helpers::cost_telemetry::values::{ - COST_CONSTANTS, COST_EXCEPTIONS, COST_MODELS, process_object, + COST_EXCEPTIONS, COST_MODELS, process_object, }; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; @@ -32,16 +33,8 @@ fn ledger_lock(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { }) } -fn health(py: Python<'_>) -> PyResult> { - imported(py, COST_CONSTANTS, "COST_TELEMETRY_HEALTH") -} - fn mark_failure(run_id: &Bound<'_, PyAny>, error: &Bound<'_, PyAny>) -> PyResult<()> { - call_with( - &health(run_id.py())?.getattr("mark_ledger_failure")?, - &[("run_id", run_id.clone()), ("error", error.clone())], - )?; - Ok(()) + mark_ledger_failure(run_id.extract()?, error) } fn optional_text(value: Option) -> JsonValue { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs index 4bc1ed6824..89f46b212d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod allocation; pub(crate) mod context; +pub(crate) mod health; pub(crate) mod ledger; pub(crate) mod registration; pub(crate) mod snowflake; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs index 20c54783fe..4ce5de8966 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_telemetry::{allocation, context, ledger, snowflake}; +use crate::bindings::_helpers::cost_telemetry::{allocation, context, health, ledger, snowflake}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { allocation::register(module)?; context::register(module)?; + health::register(module)?; ledger::register(module)?; snowflake::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 742f1c5f47..420e44cdbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -19,8 +19,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::cost_telemetry::allocation::{ SummaryState, allocate_observations, priced, summary_object, }; +use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; -use crate::bindings::_helpers::cost_telemetry::values::{COST_CONSTANTS, COST_MODELS}; +use crate::bindings::_helpers::cost_telemetry::values::COST_MODELS; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; const DEADLINE_SECONDS: f64 = 10.0; @@ -473,12 +474,7 @@ fn collect<'py>( history.deadline = monotonic(py)? + DEADLINE_SECONDS; let run_id: String = collection.run_id.extract()?; let ledger: Vec> = read_ledger(ledger_path, &collection.run_id)?; - let failure: Option = call_with( - &imported(py, COST_CONSTANTS, "COST_TELEMETRY_HEALTH")? - .getattr("consume_ledger_failure")?, - &[("run_id", collection.run_id.clone())], - )? - .extract()?; + let failure: Option = consume_ledger_failure(&run_id); let by_query_id: Bound<'py, PyDict> = PyDict::new(py); for entry in &ledger { let id: Bound<'py, PyAny> = entry.getattr("query_id")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs index b3a3a431be..b172fe1fb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs @@ -13,7 +13,6 @@ use crate::bindings::_helpers::state_rows::timestamps::offset_aware; pub(crate) const COST_MODELS: &str = "sqlbuild.cost.models"; pub(crate) const COST_TYPES: &str = "sqlbuild.cost.types"; -pub(crate) const COST_CONSTANTS: &str = "sqlbuild.cost.constants"; pub(crate) const COST_EXCEPTIONS: &str = "sqlbuild.cost.exceptions"; const DECIMAL_MODULE: &str = "decimal"; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index acb9a5c99c..ea145d474d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6112,6 +6112,12 @@ def list_source_candidates( entries: tuple[SourceEntry, ...], ) -> tuple[RelationInfo, ...]: ... +class CostTelemetryHealth: + def mark_ledger_failure(self, *, run_id: str, error: BaseException) -> None: ... + def consume_ledger_failure(self, *, run_id: str) -> str | None: ... + +COST_TELEMETRY_HEALTH: CostTelemetryHealth + class CostScope: def __enter__(self) -> None: ... def __exit__(self, *exception: object) -> None: ... diff --git a/src/sqlbuild/cost/classes/cost_telemetry_health.py b/src/sqlbuild/cost/classes/cost_telemetry_health.py deleted file mode 100644 index 358450ed1f..0000000000 --- a/src/sqlbuild/cost/classes/cost_telemetry_health.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Process-local telemetry failure tracking for the active build.""" - -import threading - - -class CostTelemetryHealth: - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._ledger_failures: dict[str, str] = {} - - def mark_ledger_failure(self, *, run_id: str, error: Exception) -> None: - with self._lock: - self._ledger_failures.setdefault(run_id, type(error).__name__) - - def consume_ledger_failure(self, *, run_id: str) -> str | None: - with self._lock: - return self._ledger_failures.pop(run_id, None) diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py index d0277f87c6..43664c4c4a 100644 --- a/src/sqlbuild/cost/constants.py +++ b/src/sqlbuild/cost/constants.py @@ -2,10 +2,5 @@ from decimal import Decimal -from sqlbuild.cost.classes.cost_telemetry_health import CostTelemetryHealth - DEFAULT_USD_PER_CREDIT: Decimal = Decimal("3.00") -DEFAULT_RATE_SOURCE: str = "default" SQLBUILD_QUERY_TAG_APP: str = "sqlbuild" -LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX: str = "Statement-ledger persistence failed (" -COST_TELEMETRY_HEALTH: CostTelemetryHealth = CostTelemetryHealth() diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py b/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py index dae2f53662..6e73d0ec60 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/classes/snowflake_connection/test_snowflake_connection.py @@ -8,10 +8,9 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import COST_TELEMETRY_HEALTH, CostContext from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.adapters.snowflake.classes.snowflake_cursor import _SnowflakeCursor -from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.observability import ( EventDispatcher, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index b7c2fc1f1d..25acc186fd 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -11,7 +11,6 @@ from sqlbuild._native import read_cost_run, refresh_pending_cost_run, write_cost_run from sqlbuild.cli.commands.models import CostCommandRequest -from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( @@ -249,7 +248,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur ) -> None: now: datetime = datetime(2026, 8, 23, tzinfo=UTC) record: CostRunRecord = build_cost_run_record(run_id="run-ledger-failure", completed_at=now) - failure_limitation: str = f"{LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX}OSError)." + failure_limitation: str = "Statement-ledger persistence failed (OSError)." persisted_failure: RunCostSummary = RunCostSummary( status=CostStatus.PARTIAL, usd_per_credit=record.cost.usd_per_credit, diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index 7b6355c511..0a558b5c77 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -10,13 +10,13 @@ import pytest from sqlbuild._native import ( + COST_TELEMETRY_HEALTH, collect_snowflake_cost, parse_sqlbuild_query_tag, render_snowflake_query_history_sql, snowflake_query_observations, snowflake_statement_is_metadata_only, ) -from sqlbuild.cost.constants import COST_TELEMETRY_HEALTH from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import ( From efd492970794de750da45bfd70bfb05ab6501412 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:29:58 +0100 Subject: [PATCH 15/33] refactor: raise source freshness and cost artifact errors from native classes --- .../_helpers/cost_report/exception_types.rs | 19 +++++ .../src/bindings/_helpers/cost_report/mod.rs | 1 + .../bindings/_helpers/cost_report/records.rs | 2 +- .../_helpers/cost_report/registration.rs | 3 +- .../_helpers/cost_telemetry/health.rs | 71 ++++++++++++------- .../_helpers/cost_telemetry/snowflake.rs | 2 +- .../_helpers/cost_telemetry/values.rs | 2 +- .../_helpers/freshness/exception_types.rs | 30 ++++++++ .../src/bindings/_helpers/freshness/mod.rs | 1 + .../_helpers/freshness/registration.rs | 3 +- .../bindings/_helpers/freshness/support.rs | 2 +- .../bindings/_helpers/state_rows/freshness.rs | 2 +- fensu.toml | 4 +- src/sqlbuild/_native.pyi | 4 ++ .../compiler/source_freshness/exceptions.py | 11 --- src/sqlbuild/cost/exceptions.py | 5 -- .../test_observation.py | 2 +- .../source_freshness/main/test_read.py | 7 +- tests/unit/src/sqlbuild/cost/test_store.py | 9 ++- 19 files changed, 124 insertions(+), 56 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs delete mode 100644 src/sqlbuild/compiler/source_freshness/exceptions.py delete mode 100644 src/sqlbuild/cost/exceptions.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs new file mode 100644 index 0000000000..eaeadd2a00 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/exception_types.rs @@ -0,0 +1,19 @@ +//! Cost observability exceptions. + +use pyo3::create_exception; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +create_exception!( + sqlbuild._native, + CostArtifactError, + PyValueError, + "Raised when a persisted cost artifact has an invalid shape." +); + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "CostArtifactError", + module.py().get_type::(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index 242aaff1b3..f734d355cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod degradation; +pub(crate) mod exception_types; pub(crate) mod functions; pub(crate) mod records; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs index 4d9322f5fa..93da8e9177 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const COST_MODELS: &str = "sqlbuild.cost.models"; const COST_TYPES: &str = "sqlbuild.cost.types"; -const COST_EXCEPTIONS: &str = "sqlbuild.cost.exceptions"; +const COST_EXCEPTIONS: &str = "sqlbuild._native"; /// The `CostArtifactError` Python raised for the same artifact. pub(crate) fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs index 7dbf137270..3b2f71866c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cost_report::{degradation, functions}; +use crate::bindings::_helpers::cost_report::{degradation, exception_types, functions}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { degradation::register(module)?; + exception_types::register(module)?; functions::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs index 9c2db29c23..55ccc7814e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs @@ -3,53 +3,72 @@ use std::collections::HashMap; use std::sync::{Mutex, MutexGuard, PoisonError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::PyTypeMethods; use pyo3::{pyclass, pymethods}; -const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; +use crate::bindings::_helpers::cost_telemetry::values::process_object; -static LEDGER_FAILURES: Mutex>> = Mutex::new(None); +const HEALTH_ATTRIBUTE: &str = "COST_TELEMETRY_HEALTH"; -fn failures() -> MutexGuard<'static, Option>> { - LEDGER_FAILURES - .lock() - .unwrap_or_else(PoisonError::into_inner) -} +static HEALTH: PyOnceLock> = PyOnceLock::new(); -/// Remember the first ledger failure of `run_id` by its exception type name. -pub(crate) fn mark_ledger_failure(run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { - let name: String = error.get_type().name()?.extract()?; - failures() - .get_or_insert_with(HashMap::new) - .entry(run_id) - .or_insert(name); - Ok(()) +/// The process-wide ledger failure tracker, exposed as `COST_TELEMETRY_HEALTH`. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CostTelemetryHealth { + ledger_failures: Mutex>, } -/// The ledger failure recorded for `run_id`, forgetting it. -pub(crate) fn consume_ledger_failure(run_id: &str) -> Option { - failures().as_mut()?.remove(run_id) +impl CostTelemetryHealth { + fn failures(&self) -> MutexGuard<'_, HashMap> { + self.ledger_failures + .lock() + .unwrap_or_else(PoisonError::into_inner) + } } -/// The process-wide ledger failure tracker, exposed as `COST_TELEMETRY_HEALTH`. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct CostTelemetryHealth; - #[pymethods] impl CostTelemetryHealth { + /// Remember the first ledger failure of `run_id` by its exception type name. #[pyo3(signature = (*, run_id, error))] fn mark_ledger_failure(&self, run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { - mark_ledger_failure(run_id, error) + let name: String = error.get_type().name()?.extract()?; + self.failures().entry(run_id).or_insert(name); + Ok(()) } + /// The ledger failure recorded for `run_id`, forgetting it. #[pyo3(signature = (*, run_id))] fn consume_ledger_failure(&self, run_id: &str) -> Option { - consume_ledger_failure(run_id) + self.failures().remove(run_id) } } +fn health(py: Python<'_>) -> PyResult> { + let tracker: &Bound<'_, PyAny> = process_object(&HEALTH, py, || { + Ok(Bound::new( + py, + CostTelemetryHealth { + ledger_failures: Mutex::new(HashMap::new()), + }, + )? + .into_any()) + })?; + Ok(tracker.cast::()?.clone()) +} + +/// Remember the first ledger failure of `run_id` by its exception type name. +pub(crate) fn mark_ledger_failure(run_id: String, error: &Bound<'_, PyAny>) -> PyResult<()> { + health(error.py())?.get().mark_ledger_failure(run_id, error) +} + +/// The ledger failure recorded for `run_id`, forgetting it. +pub(crate) fn consume_ledger_failure(py: Python<'_>, run_id: &str) -> PyResult> { + Ok(health(py)?.get().consume_ledger_failure(run_id)) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add(HEALTH_ATTRIBUTE, CostTelemetryHealth) + module.add(HEALTH_ATTRIBUTE, health(module.py())?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 420e44cdbb..a9dd18e9be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -474,7 +474,7 @@ fn collect<'py>( history.deadline = monotonic(py)? + DEADLINE_SECONDS; let run_id: String = collection.run_id.extract()?; let ledger: Vec> = read_ledger(ledger_path, &collection.run_id)?; - let failure: Option = consume_ledger_failure(&run_id); + let failure: Option = consume_ledger_failure(py, &run_id)?; let by_query_id: Bound<'py, PyDict> = PyDict::new(py); for entry in &ledger { let id: Bound<'py, PyAny> = entry.getattr("query_id")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs index b172fe1fb3..30d073aa86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::state_rows::timestamps::offset_aware; pub(crate) const COST_MODELS: &str = "sqlbuild.cost.models"; pub(crate) const COST_TYPES: &str = "sqlbuild.cost.types"; -pub(crate) const COST_EXCEPTIONS: &str = "sqlbuild.cost.exceptions"; +pub(crate) const COST_EXCEPTIONS: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; /// A finite non-negative `Decimal` as its exact digits and exponent. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs new file mode 100644 index 0000000000..8d6c959c03 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/exception_types.rs @@ -0,0 +1,30 @@ +//! Direct source freshness state and observation exceptions. + +use pyo3::create_exception; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +create_exception!( + sqlbuild._native, + SourceFreshnessInputError, + PyValueError, + "Raised when persisted source freshness state cannot be read safely." +); +create_exception!( + sqlbuild._native, + SourceFreshnessObservationError, + PyValueError, + "Raised when source freshness cannot be observed safely." +); + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add( + "SourceFreshnessInputError", + py.get_type::(), + )?; + module.add( + "SourceFreshnessObservationError", + py.get_type::(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index ed6018a87d..8d6c6cb4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod exception_types; pub(crate) mod freshness_command; pub(crate) mod freshness_observe; pub(crate) mod observation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index 050369cb2d..0d0d44f770 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -3,10 +3,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::freshness::{ - freshness_command, freshness_observe, observation, planning, state_io, + exception_types, freshness_command, freshness_observe, observation, planning, state_io, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + exception_types::register(module)?; freshness_command::register(module)?; freshness_observe::register(module)?; observation::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 877d03130d..610d3e5aeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; -pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.source_freshness.exceptions"; +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; pub(crate) const FRESHNESS_TYPES: &str = "sqlbuild.compiler.source_freshness.types"; pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index f1db12a90e..9b30e3252c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::state_rows::timestamps::{ date_time_typed, in_utc, offset_aware, presumed_utc, }; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.source_freshness.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TYPES_MODULE: &str = "sqlbuild.compiler.source_freshness.types"; const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; diff --git a/fensu.toml b/fensu.toml index e54c77f4fd..4526751f7c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -66,12 +66,12 @@ reason = "Native bindings own planning, plan entries, plan output and command pl [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/source_freshness/__init__.py" -reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types, constants and errors they build and raise." +reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types and constants they build." [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cost/constants.py" -reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types, constants, errors and the telemetry health tracker they use." +reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." [[targets.python.rule_exceptions]] rule = "FFR309" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ea145d474d..b8f9cc93d4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6112,6 +6112,10 @@ def list_source_candidates( entries: tuple[SourceEntry, ...], ) -> tuple[RelationInfo, ...]: ... +class CostArtifactError(ValueError): ... +class SourceFreshnessInputError(ValueError): ... +class SourceFreshnessObservationError(ValueError): ... + class CostTelemetryHealth: def mark_ledger_failure(self, *, run_id: str, error: BaseException) -> None: ... def consume_ledger_failure(self, *, run_id: str) -> str | None: ... diff --git a/src/sqlbuild/compiler/source_freshness/exceptions.py b/src/sqlbuild/compiler/source_freshness/exceptions.py deleted file mode 100644 index bb93a51fd9..0000000000 --- a/src/sqlbuild/compiler/source_freshness/exceptions.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Direct source freshness state exceptions.""" - -from __future__ import annotations - - -class SourceFreshnessInputError(ValueError): - """Raised when persisted source freshness state cannot be read safely.""" - - -class SourceFreshnessObservationError(ValueError): - """Raised when source freshness cannot be observed safely.""" diff --git a/src/sqlbuild/cost/exceptions.py b/src/sqlbuild/cost/exceptions.py deleted file mode 100644 index ba6a557195..0000000000 --- a/src/sqlbuild/cost/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Cost observability errors.""" - - -class CostArtifactError(ValueError): - """Raised when a persisted cost artifact has an invalid shape.""" diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index ac86b3eca9..11edc5e466 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -6,12 +6,12 @@ import pytest from sqlbuild._native import ( + SourceFreshnessObservationError, normalize_source_freshness_data_version, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py index 4f0fd2b953..f889c13e35 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py @@ -4,9 +4,12 @@ import pytest -from sqlbuild._native import read_latest_source_freshness, write_source_freshness_records +from sqlbuild._native import ( + SourceFreshnessInputError, + read_latest_source_freshness, + write_source_freshness_records, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessInputError from sqlbuild.compiler.source_freshness.models import ( SourceFreshnessIdentity, SourceFreshnessRecord, diff --git a/tests/unit/src/sqlbuild/cost/test_store.py b/tests/unit/src/sqlbuild/cost/test_store.py index b7ee3061e6..00fdc8ae77 100644 --- a/tests/unit/src/sqlbuild/cost/test_store.py +++ b/tests/unit/src/sqlbuild/cost/test_store.py @@ -6,8 +6,13 @@ import pytest -from sqlbuild._native import list_cost_runs, read_cost_run, resolve_cost_run, write_cost_run -from sqlbuild.cost.exceptions import CostArtifactError +from sqlbuild._native import ( + CostArtifactError, + list_cost_runs, + read_cost_run, + resolve_cost_run, + write_cost_run, +) from sqlbuild.cost.models import CostRunRecord from tests.unit.src.sqlbuild.cost._test_types import ( CostStoreTestCase, From 9a9745067b511a0030622995779436809a1439db Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:49:17 +0100 Subject: [PATCH 16/33] refactor: carry the compile reuse attempt as a native class --- .../bindings/_helpers/compile_command/mod.rs | 1 + .../_helpers/compile_command/models.rs | 65 +++++++++++++++++++ .../_helpers/compile_command/registration.rs | 3 +- .../_helpers/compile_command/reuse_attempt.rs | 18 ++--- .../_helpers/compile_driver/command.rs | 11 ++-- src/sqlbuild/_native.pyi | 28 +++++++- src/sqlbuild/cli/compile_reuse/models.py | 23 +------ src/sqlbuild/cli/compile_reuse/types.py | 13 ---- 8 files changed, 110 insertions(+), 52 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs delete mode 100644 src/sqlbuild/cli/compile_reuse/types.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index 2e471ccf9b..73544bca28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod artifact_batch; pub(crate) mod constants; +pub(crate) mod models; pub(crate) mod prepared_artifacts; pub(crate) mod registration; pub(crate) mod report; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs new file mode 100644 index 0000000000..487a131940 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs @@ -0,0 +1,65 @@ +//! The per-invocation compile reuse state carried from the reuse check into a full compile. + +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; + +/// Result of checking for a reusable compile, carried into a full compile on a miss. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +pub(crate) struct CompileReuseAttempt { + /// `hit`, `miss` or `bypass`. + outcome: Py, + project_dir: Py, + started: Py, + check_ms: Py, + exit_code: Py, + entry_path: Py, + /// The natively checked attempt, recorded natively after a miss. + native: Py, + selected_target: Py, + json_output: Py, +} + +const FIELDS: (&[&str], &[&str]) = ( + &["outcome", "project_dir", "started"], + &[ + "check_ms", + "exit_code", + "entry_path", + "native", + "selected_target", + "json_output", + ], +); + +#[pymethods] +impl CompileReuseAttempt { + #[new] + #[pyo3(signature = (**options))] + fn new(py: Python<'_>, options: Option>) -> PyResult { + let fields: KeywordArguments<'_> = + KeywordArguments::read(py, "CompileReuseAttempt", options, FIELDS)?; + let field = |name: &str| -> PyResult> { Ok(fields.value(name)?.unbind()) }; + Ok(Self { + outcome: field("outcome")?, + project_dir: field("project_dir")?, + started: field("started")?, + check_ms: if fields.passed("check_ms")? { + field("check_ms")? + } else { + 0_i64.into_py_any(py)? + }, + exit_code: field("exit_code")?, + entry_path: field("entry_path")?, + native: field("native")?, + selected_target: field("selected_target")?, + json_output: fields.value_or_false("json_output")?.unbind(), + }) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index dfc638391e..987392d015 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -3,10 +3,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, + models, prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + models::register(module)?; report::register(module)?; reuse_attempt::register(module)?; reuse_recording::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 01c9ce461e..89f5eaa684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -24,25 +24,25 @@ use crate::bindings::_helpers::compile_command::constants::{ REUSE_MAX_ENTRY_BYTES, REUSE_MAX_STORED_ENTRIES, REUSE_MISS_TIMING, TARGET_DIRECTORY_NAME, TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, }; +use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REUSE_MODELS: &str = "sqlbuild.cli.compile_reuse.models"; -const REUSE_TYPES: &str = "sqlbuild.cli.compile_reuse.types"; +const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"); const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; -const HIT: &str = "HIT"; +const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; -const MISS: &str = "MISS"; -const BYPASS: &str = "BYPASS"; +const MISS: &str = "miss"; +const BYPASS: &str = "bypass"; fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) @@ -245,7 +245,7 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { } fn outcome<'py>(py: Python<'py>, name: &str) -> PyResult> { - imported(py, REUSE_TYPES, "CompileReuseOutcome")?.getattr(name) + name.into_bound_py_any(py) } /// A `CompileReuseAttempt` with `fields`. @@ -253,7 +253,7 @@ fn reuse_attempt<'py>( py: Python<'py>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - call_with(&imported(py, REUSE_MODELS, "CompileReuseAttempt")?, fields) + call_with(py.get_type::().as_any(), fields) } /// The native reuse check request for this invocation. @@ -481,7 +481,7 @@ fn reuse_request<'py>(request: &Bound<'py, PyAny>) -> PyResult fields.push((name, request.getattr(name)?)); } fields.push(("profiling", profiling.into_bound_py_any(py)?)); - call_with(&imported(py, REUSE_MODELS, "CompileReuseRequest")?, &fields) + call_with(&imported(py, REUSE_REQUEST.0, REUSE_REQUEST.1)?, &fields) } /// Run one compile command request, replaying the previous compile when nothing changed. @@ -490,7 +490,7 @@ fn run_compile<'py>(request: &Bound<'py, PyAny>) -> PyResult> let py: Python<'py> = request.py(); let attempt: Bound<'py, PyAny> = attempt_compile_reuse(&reuse_request(request)?)?; let exit_code: Bound<'py, PyAny> = attempt.getattr("exit_code")?; - if attempt.getattr("outcome")?.is(&outcome(py, HIT)?) && !exit_code.is_none() { + if attempt.getattr("outcome")?.eq(outcome(py, HIT)?)? && !exit_code.is_none() { return Ok(exit_code); } imported(py, NATIVE_MODULE, "run_compile_command")?.call1((request, attempt)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 30f86b80b8..2cbe5f40e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -5,6 +5,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ format_compile_json, format_compile_text, json_semantic_notice, }; @@ -22,8 +23,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; -const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; -const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; +const BYPASS_OUTCOME: &str = "bypass"; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -104,12 +104,9 @@ fn bypass_attempt<'py>( path_class.call_method0("cwd")? }; call_with( - &imported(py, REUSE_MODELS_MODULE, "CompileReuseAttempt")?, + py.get_type::().as_any(), &[ - ( - "outcome", - imported(py, REUSE_TYPES_MODULE, "CompileReuseOutcome")?.getattr("BYPASS")?, - ), + ("outcome", BYPASS_OUTCOME.into_pyobject(py)?.into_any()), ("project_dir", path_class.call1((project_dir,))?), ("started", started.into_pyobject(py)?.into_any()), ], diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b8f9cc93d4..8fdea511e5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -54,7 +54,7 @@ from sqlbuild.cli.commands.models import ( ) from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest +from sqlbuild.cli.compile_reuse.models import CompileReuseRequest from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, CommandWarehouseScope, @@ -1655,6 +1655,32 @@ class NativeReuseAttempt: def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... def run_compile(request: CompileCommandRequest) -> int: ... +class CompileReuseAttempt: + """Result of checking for a reusable compile, carried into a full compile on a miss.""" + + outcome: str + project_dir: Path + started: float + check_ms: int + exit_code: int | None + entry_path: Path | None + native: NativeReuseAttempt | None + selected_target: str | None + json_output: bool + def __init__( + self, + *, + outcome: str, + project_dir: Path, + started: float, + check_ms: int = 0, + exit_code: int | None = None, + entry_path: Path | None = None, + native: NativeReuseAttempt | None = None, + selected_target: str | None = None, + json_output: bool = False, + ) -> None: ... + COMPILE_REUSE_RACY_WINDOW_NS: int COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES: frozenset[str] diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index 5ea1860000..6c1af6b816 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -1,13 +1,10 @@ -"""Compile reuse requests, provider settings inputs, and per-invocation reuse state.""" +"""Compile reuse requests and provider settings inputs.""" from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path -from sqlbuild._native import NativeReuseAttempt -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome - @dataclass(frozen=True) class CompileReuseRequest: @@ -47,19 +44,3 @@ class SettingsInputsResult: inputs: tuple[SettingsEnvironmentInputs, ...] unsupported_reason: str | None = None - - -@dataclass(frozen=True) -class CompileReuseAttempt: - """Result of checking for a reusable compile, carried into a full compile on a miss.""" - - outcome: CompileReuseOutcome - project_dir: Path - started: float - check_ms: int = 0 - exit_code: int | None = None - entry_path: Path | None = None - native: NativeReuseAttempt | None = field(default=None, compare=False, repr=False) - """The natively checked attempt, recorded natively after a miss.""" - selected_target: str | None = None - json_output: bool = False diff --git a/src/sqlbuild/cli/compile_reuse/types.py b/src/sqlbuild/cli/compile_reuse/types.py deleted file mode 100644 index c592c4e642..0000000000 --- a/src/sqlbuild/cli/compile_reuse/types.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Compile reuse outcome types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class CompileReuseOutcome(StrEnum): - """How one compile invocation used the stored previous compile.""" - - HIT = "hit" - MISS = "miss" - BYPASS = "bypass" From 96e0de05d3dff06460186adffa655c611869be33 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:51:33 +0100 Subject: [PATCH 17/33] refactor: keep the compile reuse attempt fields in the native constants module --- .../_helpers/compile_command/constants.rs | 12 ++++++++++ .../_helpers/compile_command/models.rs | 22 ++++--------------- .../_helpers/compile_command/registration.rs | 8 ++++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs index a72ed97f9f..b278291eed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -67,3 +67,15 @@ pub(crate) const REUSE_STORE_SKIPPED_PREFIX: &str = "Did not record compile for reuse: files changed while it ran ("; pub(crate) const COMPUTE_LOG_DIRECTORY_NAME: &str = "logs"; pub(crate) const COMPILE_COMMAND_NAME: &str = "compile"; +/// The required and optional keyword fields of `CompileReuseAttempt`. +pub(crate) const REUSE_ATTEMPT_FIELDS: (&[&str], &[&str]) = ( + &["outcome", "project_dir", "started"], + &[ + "check_ms", + "exit_code", + "entry_path", + "native", + "selected_target", + "json_output", + ], +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs index 487a131940..79d636aea0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/models.rs @@ -1,13 +1,15 @@ //! The per-invocation compile reuse state carried from the reuse check into a full compile. -use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; +use crate::bindings::_helpers::compile_command::constants::REUSE_ATTEMPT_FIELDS; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; /// Result of checking for a reusable compile, carried into a full compile on a miss. #[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] pub(crate) struct CompileReuseAttempt { /// `hit`, `miss` or `bypass`. outcome: Py, @@ -22,25 +24,13 @@ pub(crate) struct CompileReuseAttempt { json_output: Py, } -const FIELDS: (&[&str], &[&str]) = ( - &["outcome", "project_dir", "started"], - &[ - "check_ms", - "exit_code", - "entry_path", - "native", - "selected_target", - "json_output", - ], -); - #[pymethods] impl CompileReuseAttempt { #[new] #[pyo3(signature = (**options))] fn new(py: Python<'_>, options: Option>) -> PyResult { let fields: KeywordArguments<'_> = - KeywordArguments::read(py, "CompileReuseAttempt", options, FIELDS)?; + KeywordArguments::read(py, "CompileReuseAttempt", options, REUSE_ATTEMPT_FIELDS)?; let field = |name: &str| -> PyResult> { Ok(fields.value(name)?.unbind()) }; Ok(Self { outcome: field("outcome")?, @@ -59,7 +49,3 @@ impl CompileReuseAttempt { }) } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index 987392d015..8e6f6d42e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -1,13 +1,15 @@ //! Register the `sqb compile` report, artifact writing and reuse recording. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::{ - models, prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, + prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - models::register(module)?; + module.add_class::()?; report::register(module)?; reuse_attempt::register(module)?; reuse_recording::register(module)?; From 399f2cc48ada005e744ca20cb4d5b2d8662c4898 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:31:00 +0100 Subject: [PATCH 18/33] chore: reconcile integrated freshness and data-model boundary --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 7de02ac944..30190d8337 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 15231 +remaining_lines = 14911 [[stays]] path = "executor" From 135af6c0894ec16d4df573071170acaab2df32cb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:27:31 +0100 Subject: [PATCH 19/33] refactor: host compiler path, location, class-table and config-error glue natively Build the contract and YAML entry class tables, native config error exceptions, SQL-test CTE call locations, typed macro argument references, the compiler cache directory, YAML entry lines and project-relative paths in the native bindings. Delete the Python wrappers they replace, the dead path-default selection, SQL-test CTE extractor and syntax check wrappers, and the test-only analysis normalization entry. --- .../tests/test_select_path_default.rs | 20 +++ .../_helpers/auditing/schema_audits.rs | 9 +- .../compile_attachment/model_inputs.rs | 9 +- .../_helpers/compile_attachment/target.rs | 5 +- .../_helpers/compile_command/reuse_attempt.rs | 5 +- .../_helpers/compiled_project/loop_classes.rs | 5 - .../_helpers/compiled_project/model_loop.rs | 27 +--- .../compiled_project/model_loop_config.rs | 6 +- .../compiled_project/model_loop_header.rs | 18 +-- .../compiled_project/model_loop_hooks.rs | 2 +- .../_helpers/compiled_project/source_loop.rs | 15 +- .../bindings/_helpers/discovery/locations.rs | 51 +++--- .../_helpers/discovery/yaml_entries.rs | 5 +- .../_helpers/discovery/yaml_records.rs | 15 +- .../_helpers/frontier/cache_directory.rs | 35 ++++ .../src/bindings/_helpers/frontier/mod.rs | 1 + .../_helpers/frontier/registration.rs | 3 +- .../_helpers/model_config/config_errors.rs | 65 +++++++- .../_helpers/model_config/contract_classes.rs | 153 ++++++++++++++++++ .../_helpers/model_config/header_columns.rs | 17 +- .../src/bindings/_helpers/model_config/mod.rs | 1 + .../model_config/model_config_builder.rs | 16 +- .../_helpers/model_config/template_data.rs | 10 +- .../_helpers/planner_warehouse/constants.rs | 5 +- .../_helpers/project_assembly/resources.rs | 16 -- .../_helpers/render/macro_expansion_result.rs | 55 ++++--- .../bindings/_helpers/scopes/scope_loading.rs | 15 +- .../_helpers/sql_test_glue/plan_errors.rs | 54 ++++--- .../_helpers/sqltext/sql_references.rs | 5 +- fensu.toml | 5 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 19 +-- .../explicit_references/macro_arguments.py | 46 ------ .../compile/classes/sql_test_cte_extractor.py | 19 --- .../compile/classes/sql_test_cte_locator.py | 32 ---- .../discovery/_helpers/filesystem/core.py | 5 +- .../_helpers/filesystem/scoped_paths.py | 18 --- .../discovery/_helpers/yml/yaml_classes.py | 45 ------ .../discovery/main/_yaml_entry_line.py | 19 --- .../frontier/main/compiler_cache_directory.py | 21 --- .../compiler/model_config/__init__.py | 1 - .../compiler/model_config/constants.py | 9 -- .../compiler/model_config/main/__init__.py | 1 - .../model_config/main/_native_config_error.py | 35 ---- .../main/_native_contract_classes.py | 29 ---- .../compiler/path_defaults/__init__.py | 0 .../path_defaults/_helpers/__init__.py | 0 .../path_defaults/_helpers/matching.py | 51 ------ .../compiler/path_defaults/constants.py | 8 - .../compiler/path_defaults/main/__init__.py | 0 .../compiler/path_defaults/main/_select.py | 62 ------- src/sqlbuild/compiler/path_defaults/models.py | 13 -- .../project_assembly/main/__init__.py | 1 - .../main/_check_native_sql_syntax.py | 18 --- .../sql_analysis/main/_normalize_analysis.py | 23 --- .../presentation/_helpers/structure.py | 24 --- .../test_runtime_artifact_preservation.py | 2 +- .../cli/commands/main/compile/helpers.py | 2 +- .../cli/commands/main/scope/helpers.py | 2 +- .../commands/main/compile_outputs/helpers.py | 3 +- .../compile/native_model_config/helpers.py | 6 +- .../compiler/pipeline/test_review_bindings.py | 2 +- .../compiler/compile/_helpers/_test_types.py | 16 -- .../compile/_helpers/config/helpers.py | 5 +- .../compile/_helpers/test_path_defaults.py | 133 --------------- .../sqlbuild/compiler/discovery/helpers.py | 3 - .../discovery/main/test_yaml_entry_line.py | 2 +- .../compiler/scopes/test_offline_cache.py | 7 +- .../compiler/sql_analysis/main/helpers.py | 19 +++ .../main/test_normalize_analysis.py | 2 +- .../main/test_normalize_analysis_batch.py | 2 +- 71 files changed, 478 insertions(+), 877 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py delete mode 100644 src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py delete mode 100644 src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py delete mode 100644 src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py delete mode 100644 src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py delete mode 100644 src/sqlbuild/compiler/model_config/__init__.py delete mode 100644 src/sqlbuild/compiler/model_config/constants.py delete mode 100644 src/sqlbuild/compiler/model_config/main/__init__.py delete mode 100644 src/sqlbuild/compiler/model_config/main/_native_config_error.py delete mode 100644 src/sqlbuild/compiler/model_config/main/_native_contract_classes.py delete mode 100644 src/sqlbuild/compiler/path_defaults/__init__.py delete mode 100644 src/sqlbuild/compiler/path_defaults/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/path_defaults/_helpers/matching.py delete mode 100644 src/sqlbuild/compiler/path_defaults/constants.py delete mode 100644 src/sqlbuild/compiler/path_defaults/main/__init__.py delete mode 100644 src/sqlbuild/compiler/path_defaults/main/_select.py delete mode 100644 src/sqlbuild/compiler/path_defaults/models.py delete mode 100644 src/sqlbuild/compiler/project_assembly/main/__init__.py delete mode 100644 src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py delete mode 100644 src/sqlbuild/presentation/_helpers/structure.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py diff --git a/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs b/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs index 7431baeb5e..08d632294d 100644 --- a/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs +++ b/crates/sqlbuild-model-config/src/path_defaults/tests/test_select_path_default.rs @@ -45,6 +45,26 @@ fn given_path_default_keys_when_selecting_then_the_python_choice_is_made() { .with_help(CONFLICT_HELP), ), }, + SelectTestCase { + description: "a single glob consumes exactly one segment", + model_path: "models/commerce/eu/partner_feed/staging/orders.sql", + keys: &["commerce/*/staging"], + expected_choice: PathDefaultChoice::Selected(None), + }, + SelectTestCase { + description: "a single glob outranks a recursive glob", + model_path: "models/commerce/partner_feed/staging/orders.sql", + keys: &["commerce/**/staging", "commerce/*/staging"], + expected_choice: PathDefaultChoice::Selected(Some("commerce/*/staging".to_owned())), + }, + SelectTestCase { + description: "a literal prefix outranks a recursive glob on a Windows path", + model_path: "models\\commerce\\partner_feed\\staging\\orders.sql", + keys: &["commerce/**/staging", "commerce/partner_feed/staging"], + expected_choice: PathDefaultChoice::Selected(Some( + "commerce/partner_feed/staging".to_owned(), + )), + }, SelectTestCase { description: "no key matches", model_path: "staging/orders.sql", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs index 6b169c8998..3d904ab50e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs @@ -11,10 +11,9 @@ use sqlbuild_model_config::header_metadata::models::ParsedAudit; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; -const CONTRACT_CLASSES_MODULE: &str = - "sqlbuild.compiler.model_config.main._native_contract_classes"; const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; /// A resource identity error as `ResourceIdentityError`, anything else as `error_class(message)`. @@ -54,11 +53,7 @@ fn parse_audit_instances<'py>( Ok(parsed) => parsed, Err(error) => return Err(raised(py, error, &error_class)?), }; - let classes: ContractClasses<'py> = py - .import(CONTRACT_CLASSES_MODULE)? - .getattr("native_contract_classes")? - .call0()? - .extract()?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; audit_tuple(py, &classes, &parsed) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 24d16e9b97..c8c558a2c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; @@ -26,8 +27,6 @@ const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.c const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; -const CONTRACT_CLASSES_MODULE: &str = - "sqlbuild.compiler.model_config.main._native_contract_classes"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; @@ -276,11 +275,7 @@ pub(crate) fn build_model_inputs<'py>( let syntax: Bound<'py, PyAny> = context.getattr("sql_lexical_syntax")?; let native_loop: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeModelLoop")?.call1(( - ( - validator, - imported(py, CONTRACT_CLASSES_MODULE, "native_contract_classes")?.call0()?, - classes, - ), + (validator, contract_classes(py)?.into_any(), classes), syntax.getattr("native_mapping")?, reserved_cursor_markers(), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 23fc340a8d..7b6c02757f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -8,13 +8,13 @@ use crate::bindings::_helpers::compile_attachment::effective_config::project_var use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; -const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -92,8 +92,7 @@ fn compile_cache_root<'py>( { return Ok(py.None().into_bound(py)); } - let root: Bound<'py, PyAny> = - imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")?.call1((project_dir,))?; + let root: Bound<'py, PyAny> = cache_directory(&project_dir)?; remove_retired_caches(&root)?; Ok(root) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 89f5eaa684..0305e5f21d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -28,6 +28,7 @@ use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -35,7 +36,6 @@ const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "Compi const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; -const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; const HIT: &str = "hit"; @@ -268,8 +268,7 @@ fn reuse_check<'py>( check.set_item("project_dir", project_dir)?; check.set_item( "store_directory", - imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? - .call1((imported(py, "pathlib", "Path")?.call1((project_dir,))?,))? + cache_directory(&imported(py, "pathlib", "Path")?.call1((project_dir,))?)? .call_method1("__truediv__", (NATIVE_REUSE_DIRECTORY_NAME,))?, )?; check.set_item("selected_target", request.getattr("selected_target")?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index ba5e0ba839..d60a1188ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -12,7 +12,6 @@ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; @@ -443,10 +442,6 @@ pub(crate) fn model_loop_classes<'py>( "sql_hook_entry", class(py, DISCOVERY_MODELS_MODULE, "SqlHookEntry")?, ), - ( - "config_error", - imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, - ), ( "audit_factories", by_attribute(audit_factories, "name")?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index a208ee6dd8..a9a455c55b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -55,6 +55,7 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ AttachedSqlTests, SqlTestSettings, TestFileRow, TestScope, }; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::render::declaration_expansion::{ @@ -100,8 +101,6 @@ const VALIDATE_METHOD: &str = "validate"; const FILE_PATH_ARGUMENT: &str = "file_path"; const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const HELP_ARGUMENT: &str = "help"; -const ERROR_ARGUMENT: &str = "error"; -const VALUES_ARGUMENT: &str = "values"; const CODE_ARGUMENT: &str = "code"; const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; const ANALYSIS_CONFLICT_CODE: &str = "P003"; @@ -122,8 +121,6 @@ pub(crate) struct LoopClasses<'py> { pub(crate) usage_record: Bound<'py, PyAny>, pub(crate) compile_error: Bound<'py, PyAny>, pub(crate) sql_hook_entry: Bound<'py, PyAny>, - /// `native_config_error`: the exception a native config error raises as. - pub(crate) config_error: Bound<'py, PyAny>, /// Discovered audit factories by name. pub(crate) audit_factories: Bound<'py, PyDict>, /// `UsageKind.GENERATED`, for usages an enum-typed column generates. @@ -816,13 +813,12 @@ impl NativeModelLoop { (rows, declared_columns, unmanaged, declared), ), )?; - validation_outcome(context, &outcome, external, values) + validation_outcome(&outcome, external, values) } } /// `_raise_validation_outcome`: nothing, the validator's error, or a rejected reference's error. pub(crate) fn validation_outcome( - context: &LoopContext<'_, '_>, outcome: &Bound<'_, PyAny>, external: Vec>, values: &Bound<'_, PyDict>, @@ -835,7 +831,7 @@ pub(crate) fn validation_outcome( { return Err(error); } - Err(config_error(context.classes, outcome, Some(values))) + Err(config_error(outcome, Some(values))) } /// `applied_interpolation`: record the interpolation's template reads, then its SQL or error. @@ -900,21 +896,8 @@ pub(crate) fn helped_compile_error( } /// The Python exception a native config error raises as, given the values it names. -pub(crate) fn config_error( - classes: &LoopClasses<'_>, - error: &Bound<'_, PyAny>, - values: Option<&Bound<'_, PyDict>>, -) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(error.py()); - keywords.set_item(ERROR_ARGUMENT, error)?; - keywords.set_item(VALUES_ARGUMENT, values)?; - classes.config_error.call((), Some(&keywords)) - })(); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } +pub(crate) fn config_error(error: &Bound<'_, PyAny>, values: Option<&Bound<'_, PyDict>>) -> PyErr { + raised_config_exception(error, values.map(Bound::as_any)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs index 5e754d3fdd..0a400c0a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -76,7 +76,7 @@ impl<'py> ModelConfigs<'py> { let builder: &NativeModelConfigBuilder = classes.builder.get(); let matched: Option = builder .selected_path_default(py, relative_path)? - .map_err(|error| config_error(context.classes, error.bind(py).as_any(), None))?; + .map_err(|error| config_error(error.bind(py).as_any(), None))?; let metadata: Option> = reusable_metadata(header)?; let reusable: bool = match metadata { Some(_) => self.reusable(context, &matched)?, @@ -94,7 +94,7 @@ impl<'py> ModelConfigs<'py> { let built: BuiltConfig<'py> = match built { ConfigBuild::Built(built) => built, ConfigBuild::Rejected(error) => { - return Err(config_error(context.classes, error.bind(py).as_any(), None)); + return Err(config_error(error.bind(py).as_any(), None)); } }; let fields: Bound<'py, PyDict> = config_fields(classes, &built, matched.as_deref())?; @@ -113,7 +113,7 @@ impl<'py> ModelConfigs<'py> { let builder: &NativeModelConfigBuilder = context.classes.config.builder.get(); let values: Bound<'py, PyDict> = builder .inherited_values(py, matched.as_deref())? - .map_err(|error| config_error(context.classes, error.bind(py).as_any(), None))?; + .map_err(|error| config_error(error.bind(py).as_any(), None))?; let checked: Bound<'py, PyTuple> = PyTuple::new( py, [values.as_any(), builder.target_namespace(py)?.as_any()], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index dc7a179449..f73e9a239a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -99,7 +99,7 @@ pub(crate) fn header_schema<'py>( let metadata = parse_header_metadata(&node(columns), &node(audits), source.relative_path); let parsed_columns = metadata .columns - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let local_columns: Bound<'py, PyTuple> = column_tuple( py, context.contracts, @@ -110,13 +110,13 @@ pub(crate) fn header_schema<'py>( Some(schema) => merged_columns(context, schema, &local_columns, model)?, None => local_columns, }; - let families = dynamic_families(py, context, &node(dynamic), model)?; + let families = dynamic_families(py, &node(dynamic), model)?; let parsed_audits = metadata .audits - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let available: Vec = context.classes.audit_factories.keys().extract()?; let references: Vec = audit_factory_references(&node(factories), model, &available) - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let direct_audits: Bound<'py, PyTuple> = audit_tuple(py, context.contracts, &parsed_audits)?; let (generated, generated_identities) = factory_audits(py, context, &references)?; let direct_identities: Vec = direct_audits @@ -124,14 +124,14 @@ pub(crate) fn header_schema<'py>( .map(|audit| audit_identity(&audit)) .collect::>()?; merged_audit_identities(&direct_identities, &generated_identities, model) - .map_err(|error| raised_config_error(py, context, error))?; + .map_err(|error| raised_config_error(py, error))?; let migrations: Vec = columns .iter() .map(|column| column_migration(&column)) .collect::>()?; if let Some(error) = column_migration_error(source.name, source.values, &migrations)? { let error = native_config_error(py, error)?.into_bound(py).into_any(); - return Err(config_error(context.classes, &error, Some(source.values))); + return Err(config_error(&error, Some(source.values))); } let mut type_enforcement: bool = !families.is_empty(); for column in columns.iter() { @@ -180,11 +180,10 @@ pub(crate) fn header_schema<'py>( /// The header's dynamic column families, each `name_pattern` compiled natively. fn dynamic_families( py: Python<'_>, - context: &LoopContext<'_, '_>, raw: &PyNode<'_>, model: (&str, &str), ) -> PyResult> { - parse_dynamic_columns(raw, model).map_err(|error| raised_config_error(py, context, error)) + parse_dynamic_columns(raw, model).map_err(|error| raised_config_error(py, error)) } /// The header description: absent, or a non-blank string by Python's `str.strip`. @@ -209,11 +208,10 @@ fn description( fn raised_config_error( py: Python<'_>, - context: &LoopContext<'_, '_>, error: sqlbuild_model_config::errors::ConfigError, ) -> pyo3::PyErr { match native_config_error(py, error) { - Ok(native) => config_error(context.classes, native.bind(py).as_any(), None), + Ok(native) => config_error(native.bind(py).as_any(), None), Err(failure) => failure, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index 8dc7b26e73..a0a5504022 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -152,7 +152,7 @@ pub(crate) fn expanded_hooks<'py>( .validator .bind(py) .call_method1(REFERENCES_METHOD, ((model.name, model.relative_path), rows))?; - validation_outcome(context, &outcome, external, &PyDict::new(py))?; + validation_outcome(&outcome, external, &PyDict::new(py))?; reject_hook_intrinsics(model_loop, context, model.name, &expanded)?; Ok(ExpandedHooks { values: expanded, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs index 286209cd78..01ab413136 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_macros::{ }; use crate::bindings::_helpers::compiled_project::source_views::{AttachedSource, replaced}; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; use crate::bindings::_helpers::model_config::config_templates::{ PythonHost, Stop, TemplateSources, expanded, template_error, }; @@ -350,23 +351,11 @@ pub(crate) fn expanded_template<'py>( let error: Bound<'py, PyAny> = native_config_error(py, template_error(&error, label))? .into_bound(py) .into_any(); - Err(raised_config_error(context, &error)) + Err(raised_config_exception(&error, None)) } } } -fn raised_config_error(context: &LoopContext<'_, '_>, error: &Bound<'_, PyAny>) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(error.py()); - keywords.set_item("error", error)?; - context.classes.config_error.call((), Some(&keywords)) - })(); - match raised { - Ok(instance) => PyErr::from_value(instance), - Err(failure) => failure, - } -} - /// `expand_source_column_templates`. fn expanded_column( context: &LoopContext<'_, '_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs index d01171a01d..db24afda82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -1,37 +1,44 @@ -//! Authored locations Python consumers read: YAML entry lines and SQL-test helper spans. +//! Authored locations Python consumers read: YAML entry lines and project-relative paths. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; -use sqlbuild_analysis::sql_test_attachment::models::CteLocation; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; -/// Each outermost `- name:` entry name with its 1-based line. +/// The 1-based line of the outermost `- name: ` entry, if present. #[pyfunction] -fn yaml_entry_line_index(contents: &str) -> Vec<(String, usize)> { +#[pyo3(signature = (*, contents, name))] +fn yaml_entry_line(contents: &str, name: &str) -> Option { yaml_entry_lines(contents) + .into_iter() + .find_map(|(entry, line)| (entry == name).then_some(line)) } -/// `(line, column, end line, end column)` of a helper problem in one SQL test block. +/// `path.relative_to(project_dir)` without pathlib's per-parent comparison scan. #[pyfunction] -#[pyo3(signature = (contents, sql_body, cte_name, call))] -fn sql_test_cte_span( - contents: &str, - sql_body: &str, - cte_name: &str, - call: Option<&str>, -) -> (usize, usize, usize, usize) { - let location: CteLocation = sql_test_cte_location(contents, sql_body, cte_name, call); - ( - location.line, - location.column, - location.end_line, - location.end_column, +#[pyo3(signature = (*, path, project_dir))] +fn project_relative_path<'py>( + path: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let root_parts: Bound<'py, PyTuple> = project_dir.getattr("parts")?.cast_into()?; + let path_parts: Bound<'py, PyTuple> = path.getattr("parts")?.cast_into()?; + let same_anchor: bool = path.call_method0("is_absolute")?.is_truthy()? + == project_dir.call_method0("is_absolute")?.is_truthy()?; + if !same_anchor + || path_parts.len() < root_parts.len() + || !path_parts.get_slice(0, root_parts.len()).eq(&root_parts)? + { + return path.call_method1("relative_to", (project_dir,)); + } + path.call_method1( + "with_segments", + path_parts.get_slice(root_parts.len(), path_parts.len()), ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(yaml_entry_line_index, module)?)?; - module.add_function(wrap_pyfunction!(sql_test_cte_span, module)?)?; + module.add_function(wrap_pyfunction!(yaml_entry_line, module)?)?; + module.add_function(wrap_pyfunction!(project_relative_path, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs index 3d3638e69d..5173582032 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs @@ -14,6 +14,7 @@ use sqlbuild_discovery::yaml_entries::models::{ use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::contract_classes::yaml_entry_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, audit_tuple}; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; @@ -161,8 +162,8 @@ pub(crate) fn parse_schema_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, - classes: YamlClasses<'py>, ) -> PyResult> { + let classes: YamlClasses<'py> = yaml_entry_classes(py)?.extract()?; let in_seeds_directory: bool = std::path::Path::new(path) .components() .any(|component| component.as_os_str() == SEEDS_DIRECTORY); @@ -343,8 +344,8 @@ pub(crate) fn parse_sources_document<'py>( py: Python<'py>, loaded: Bound<'py, PyAny>, path: &str, - classes: YamlClasses<'py>, ) -> PyResult> { + let classes: YamlClasses<'py> = yaml_entry_classes(py)?.extract()?; let sources: Vec>> = parse_sources(&PyNode(loaded), path) .map_err(|error| { parse_error(py, error, SOURCE_PARSE_ERROR).unwrap_or_else(|error| error) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 1d33e1f78a..15a8adf641 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -27,14 +27,13 @@ use crate::bindings::_helpers::discovery::snapshot::{ escaped, in_snapshot, project_root, project_tree, }; use crate::bindings::_helpers::discovery::yaml_entries::{ - YamlClasses, parse_schema_document, parse_sources_document, + parse_schema_document, parse_sources_document, }; use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; -const YAML_CLASSES_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.yaml_classes"; const PATHLIB_MODULE: &str = "pathlib"; const PROJECT_PATH_KIND: &str = "project_path"; const SCHEMA_KIND: &str = "schema"; @@ -204,12 +203,6 @@ fn unnamed_error(project_dir: &Bound<'_, PyAny>, relative: &Bound<'_, PyAny>) -> )) } -fn yaml_classes(py: Python<'_>) -> PyResult> { - imported(py, YAML_CLASSES_MODULE, "yaml_entry_classes")? - .call0()? - .extract() -} - /// Discover source YAML files natively, reporting failing files to `on_fault`. #[pyfunction] #[pyo3(signature = (*, project_dir, on_fault=None))] @@ -245,8 +238,7 @@ pub(crate) fn discover_source_files<'py>( ("contents", contents.into_bound_py_any(py)?), ( "source_entries", - parse_sources_document(py, loaded, &shown, yaml_classes(py)?)? - .into_any(), + parse_sources_document(py, loaded, &shown)?.into_any(), ), ], ) @@ -307,8 +299,7 @@ pub(crate) fn discover_schema_files<'py>( ("model_entries", PyTuple::empty(py).into_any()), ( "seed_entries", - parse_schema_document(py, loaded, &shown, yaml_classes(py)?)? - .into_any(), + parse_schema_document(py, loaded, &shown)?.into_any(), ), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs new file mode 100644 index 0000000000..a24bff111a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs @@ -0,0 +1,35 @@ +//! The project-local compiler store root. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; + +/// `project_dir / target / cache / compiler`, keeping the path type of `project_dir`. +pub(crate) fn cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let constant = |name: &str| -> PyResult { + imported(py, FRONTIER_CONSTANTS_MODULE, name)?.extract() + }; + let store: String = format!( + "{}{}", + constant("COMPILER_CACHE_DIRECTORY_NAME")?, + constant("NATIVE_CACHE_NAMESPACE_SUFFIX")? + ); + project_dir + .div(constant("TARGET_DIRECTORY_NAME")?)? + .div(constant("CACHE_DIRECTORY_NAME")?)? + .div(store) +} + +/// Return the project's compiler store root. +#[pyfunction] +fn compiler_cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + cache_directory(project_dir) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(compiler_cache_directory, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs index fa17bbe460..d752ee27cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod cache_directory; pub(crate) mod compile_frontier; pub(crate) mod encoder; pub(crate) mod python_types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs index 82b9d86c7e..de98b71172 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::frontier::{compile_frontier, stage_capture}; +use crate::bindings::_helpers::frontier::{cache_directory, compile_frontier, stage_capture}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + cache_directory::register(module)?; compile_frontier::register(module)?; stage_capture::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs index ee7d58978a..4727631155 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs @@ -1,11 +1,19 @@ //! Native config errors handed to Python, which raises them as the Python stages would. -use pyo3::Py; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{Py, PyErr, pymethods}; use sqlbuild_model_config::errors::{ConfigError, ErrorClass}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::model_config::errors::NativeConfigError; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const CONTRACT_ERRORS_MODULE: &str = "sqlbuild.spec.contracts.exceptions"; +const EXPECTED_STRING: &str = "a string"; + const COMPILE_INPUT_CLASS: &str = "compile_input"; const CONFIG_VALUE_TYPE_CLASS: &str = "config_value_type"; const RESOURCE_IDENTITY_CLASS: &str = "resource_identity"; @@ -34,6 +42,59 @@ pub(crate) fn native_config_error( ) } +/// The Python exception a native config error raises as, given the values it names. +pub(crate) fn config_exception<'py>( + error: &Bound<'py, PyAny>, + values: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let error: Bound<'py, NativeConfigError> = error.cast::()?.clone(); + error.get().exception(error.py(), values) +} + +/// `config_exception` as the raised error, or the failure building it. +pub(crate) fn raised_config_exception( + error: &Bound<'_, PyAny>, + values: Option<&Bound<'_, PyAny>>, +) -> PyErr { + match config_exception(error, values) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +#[pymethods] +impl NativeConfigError { + /// The exception this error raises as: its class, message, code and help. + #[pyo3(signature = (values = None))] + fn exception<'py>( + &self, + py: Python<'py>, + values: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + if self.class_name == CONFIG_VALUE_TYPE_CLASS + && let (Some(key), Some(values)) = (&self.key, values) + { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("key", key)?; + keywords.set_item("expected", EXPECTED_STRING)?; + keywords.set_item("actual_type", values.get_item(key)?.get_type())?; + return imported(py, CONTRACT_ERRORS_MODULE, "ConfigValueTypeError")? + .call((), Some(&keywords)); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("help", &self.help)?; + let (module, class) = match self.class_name { + RESOURCE_IDENTITY_CLASS => (IDENTITY_ERRORS_MODULE, "ResourceIdentityError"), + DISCOVERY_CONFLICT_CLASS => (DISCOVERY_ERRORS_MODULE, "DiscoveryConflictError"), + _ => { + keywords.set_item("code", self.code)?; + (COMPILE_ERRORS_MODULE, "CompileInputError") + } + }; + imported(py, module, class)?.call((&self.message,), Some(&keywords)) + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs new file mode 100644 index 0000000000..1cca55b389 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs @@ -0,0 +1,153 @@ +//! The Python classes native header and YAML parsing build contract objects as. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const DLT_MODELS_MODULE: &str = "sqlbuild.integrations.dlt.models"; +const INGESTR_MODELS_MODULE: &str = "sqlbuild.integrations.ingestr.models"; +const CONTRACT_CLASSES: [(&str, &str, &str); 6] = [ + ("schema_column", CONTRACT_MODELS_MODULE, "SchemaColumn"), + ( + "schema_audit_instance", + CONTRACT_MODELS_MODULE, + "SchemaAuditInstance", + ), + ( + "schema_dynamic_column_family", + CONTRACT_MODELS_MODULE, + "SchemaDynamicColumnFamily", + ), + ("source_location", CONTRACT_MODELS_MODULE, "SourceLocation"), + ( + "measurement_thresholds", + AUDITING_MODELS_MODULE, + "MeasurementThresholds", + ), + ( + "measurement_threshold_bound", + AUDITING_MODELS_MODULE, + "MeasurementThresholdBound", + ), +]; +const YAML_ENTRY_CLASSES: [(&str, &str, &str); 13] = [ + ( + "schema_seed_entry", + CONTRACT_MODELS_MODULE, + "SchemaSeedEntry", + ), + ( + "seed_csv_settings", + CONTRACT_MODELS_MODULE, + "SeedCsvSettings", + ), + ("source_entry", CONTRACT_MODELS_MODULE, "SourceEntry"), + ( + "source_column_entry", + CONTRACT_MODELS_MODULE, + "SourceColumnEntry", + ), + ( + "source_freshness_config", + CONTRACT_MODELS_MODULE, + "SourceFreshnessConfig", + ), + ( + "source_freshness_age_policy", + CONTRACT_MODELS_MODULE, + "SourceFreshnessAgePolicy", + ), + ( + "integration_loader_config", + CONTRACT_MODELS_MODULE, + "IntegrationLoaderConfig", + ), + ( + "ingestr_source_config", + INGESTR_MODELS_MODULE, + "IngestrSourceConfig", + ), + ("dlt_source_config", DLT_MODELS_MODULE, "DltSourceConfig"), + ( + "dlt_resource_config", + DLT_MODELS_MODULE, + "DltResourceConfig", + ), + ( + "source_write_strategy", + CONTRACT_TYPES_MODULE, + "SourceWriteStrategy", + ), + ( + "source_freshness_strategy", + CONTRACT_TYPES_MODULE, + "SourceFreshnessStrategy", + ), + ( + "source_freshness_value_kind", + CONTRACT_TYPES_MODULE, + "SourceFreshnessValueKind", + ), +]; + +static CONTRACT_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); +static YAML_ENTRY_CLASS_TABLE: PyOnceLock> = PyOnceLock::new(); + +/// The contract classes native header parsing builds columns and audits as. +pub(crate) fn contract_classes(py: Python<'_>) -> PyResult> { + CONTRACT_CLASS_TABLE + .get_or_try_init(py, || built_contract_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) +} + +/// The classes parsed seed and source declarations become. +pub(crate) fn yaml_entry_classes(py: Python<'_>) -> PyResult> { + YAML_ENTRY_CLASS_TABLE + .get_or_try_init(py, || built_yaml_entry_classes(py).map(Bound::unbind)) + .map(|classes| classes.bind(py).clone()) +} + +fn built_contract_classes(py: Python<'_>) -> PyResult> { + let classes: Bound<'_, PyDict> = PyDict::new(py); + for (key, module, name) in CONTRACT_CLASSES { + classes.set_item(key, imported(py, module, name)?)?; + } + classes.set_item( + "severities", + members_by_value(&imported(py, AUDITING_TYPES_MODULE, "AuditSeverity")?)?, + )?; + classes.set_item( + "threshold_operators", + members_by_value(&imported(py, AUDITING_TYPES_MODULE, "ThresholdOperator")?)?, + )?; + classes.set_item( + "allocate", + imported(py, "builtins", "object")?.getattr("__new__")?, + )?; + Ok(classes) +} + +fn built_yaml_entry_classes(py: Python<'_>) -> PyResult> { + let classes: Bound<'_, PyDict> = PyDict::new(py); + classes.set_item("contracts", contract_classes(py)?)?; + for (key, module, name) in YAML_ENTRY_CLASSES { + classes.set_item(key, imported(py, module, name)?)?; + } + Ok(classes) +} + +/// `{member.value: member for member in enum}`. +fn members_by_value<'py>(enumeration: &Bound<'py, PyAny>) -> PyResult> { + let members: Bound<'py, PyDict> = PyDict::new(enumeration.py()); + for member in enumeration.try_iter()? { + let member: Bound<'py, PyAny> = member?; + members.set_item(member.getattr("value")?, member)?; + } + Ok(members) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs index c7f002ec28..3c8b08c2a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs @@ -10,14 +10,13 @@ use sqlbuild_model_config::header_metadata::main::parse_schema_columns::parse_sc use sqlbuild_model_config::header_metadata::models::ColumnRequest; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::header_metadata::{ContractClasses, column_tuple}; -const CONTRACT_CLASSES_MODULE: &str = - "sqlbuild.compiler.model_config.main._native_contract_classes"; -const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; const DECLARATION_ERROR_CLASS: &str = "DeclarationParseError"; @@ -44,10 +43,7 @@ pub(crate) fn schema_columns<'py>( }; let columns = parse_columns(&PyNode(raw_columns.clone()), request) .map_err(|error| raised(py, error, declaration_errors))?; - let classes: ContractClasses<'py> = - imported(py, CONTRACT_CLASSES_MODULE, "native_contract_classes")? - .call0()? - .extract()?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; column_tuple(py, &classes, &columns, column_locations) } @@ -70,10 +66,7 @@ fn raised(py: Python<'_>, error: ConfigError, declaration_errors: bool) -> PyErr .call1((error.message,)); } let native = native_config_error(py, error)?.into_bound(py).into_any(); - call_with( - &imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, - &[("error", native)], - ) + config_exception(&native, None) })(); match built { Ok(instance) => PyErr::from_value(instance), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index d76eb0322f..69bf36cd84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod column_migrations; pub(crate) mod config_errors; pub(crate) mod config_templates; pub(crate) mod config_values; +pub(crate) mod contract_classes; pub(crate) mod described_models; pub(crate) mod errors; pub(crate) mod header_columns; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 4e219ebeab..5ee6943fc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -22,6 +22,7 @@ use sqlbuild_model_config::types::AuthoredNode; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, built_config}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::config_templates::{ PythonHost, Stop, TemplateSources, expanded, template_error, @@ -36,8 +37,6 @@ const MODEL_CONFIG_LABEL: &str = "model config"; const ENVIRONMENT_DATABASE_LABEL: &str = "environment database"; const ENVIRONMENT_SCHEMA_LABEL: &str = "environment schema"; const TAGS_KEY: &str = "tags"; -const NATIVE_CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; -const NATIVE_CONFIG_ERROR_FUNCTION: &str = "native_config_error"; const ROW_DIFF_EXCLUDE_COLUMNS_KEY: &str = "row_diff_exclude_columns"; const ROW_DIFF_TOLERANCES_KEY: &str = "row_diff_tolerances"; const AUDIT_OVERRIDE_SECTIONS: [&str; 2] = ["by_type", "by_column"]; @@ -586,17 +585,14 @@ pub(crate) fn layered_values<'py>( }) } -/// The Python exception `native_config_error` makes of a native config error. +/// The Python exception a native config error raises as. pub(crate) fn raised_config_error(py: Python<'_>, error: ConfigError) -> PyErr { let raised = || -> PyResult { let error: Py = native_config_error(py, error)?; - let arguments: Bound<'_, PyDict> = PyDict::new(py); - arguments.set_item("error", error)?; - let exception: Bound<'_, PyAny> = py - .import(NATIVE_CONFIG_ERROR_MODULE)? - .getattr(NATIVE_CONFIG_ERROR_FUNCTION)? - .call((), Some(&arguments))?; - Ok(PyErr::from_value(exception)) + Ok(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, + )?)) }; raised().unwrap_or_else(|error| error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs index 28dedd70e0..696826b8e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -11,7 +11,8 @@ use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_a use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_templates::{ TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, resolve_effective_vars, @@ -20,7 +21,6 @@ use crate::bindings::_helpers::model_config::config_templates::{ const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; -const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; const UUID_MODULE: &str = "uuid"; const ENVIRONMENT_READ: &str = "env"; const RUN_ID_KEY: &str = "run.id"; @@ -52,9 +52,9 @@ fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult Ok(value), - Err(error) => Err(PyErr::from_value(call_with( - &imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, - &[("error", error.into_bound(py).into_any())], + Err(error) => Err(PyErr::from_value(config_exception( + error.bind(py).as_any(), + None, )?)), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 15f735c851..09151463ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -39,10 +39,7 @@ pub(crate) const CURSOR_POLICIES: (&str, &str) = pub(crate) const MAXIMUM_START: (&str, &str) = ("sqlbuild._native", "maximum_allowed_start"); pub(crate) const RENDER_SOURCE_READ: (&str, &str) = ("sqlbuild._native", "render_source_read_relation"); -pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ( - "sqlbuild.compiler.discovery.main._yaml_entry_line", - "yaml_entry_line", -); +pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ("sqlbuild._native", "yaml_entry_line"); pub(crate) const RUN_SCOPE: (&str, &str) = ("sqlbuild.runtime.observability.main.run_scope", "run_scope"); pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index 77eb7ebbfa..d626d26f8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -6,7 +6,6 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{PyErr, pyfunction, wrap_pyfunction}; -use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; @@ -64,20 +63,6 @@ fn syntax_value_error(failure: SyntaxFailure) -> PyErr { } } -/// Whether every `(sql, placeholders)` parses, raising the `ValueError` Python's validation raises. -#[pyfunction] -fn check_native_sql_syntax( - py: Python<'_>, - request: (String, Vec<(String, Pairs)>), -) -> PyResult { - let (dialect, rows) = request; - let checks: Vec = rows - .into_iter() - .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) - .collect(); - syntax_valid(py, &dialect, &checks) -} - /// Whether every check parses, raising what Python's validation raises. pub(crate) fn syntax_valid( py: Python<'_>, @@ -90,7 +75,6 @@ pub(crate) fn syntax_valid( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index d6fe88b2b8..cd25c9cee6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion::{ ExpansionOutput, NativeMacroExpansion, expanded, }; +use crate::bindings::_helpers::sqltext::sql_references::extract_references; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; @@ -23,8 +24,11 @@ const SCOPES_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPES_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; -const MACRO_ARGUMENTS_MODULE: &str = - "sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments"; +const TYPED_REFERENCE_FUNCTIONS: [(&str, &str); 3] = [ + ("model", "__ref"), + ("source", "__source"), + ("seed", "__seed"), +]; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; @@ -97,12 +101,12 @@ fn generated_classes(py: Python<'_>) -> PyResult> { fn built_classes(py: Python<'_>) -> PyResult> { let native: Bound<'_, PyAny> = py.import(NATIVE_MODULE)?.into_any(); let declaration_kind: Bound<'_, PyAny> = imported(py, SCOPES_TYPES_MODULE, "DeclarationKind")?; - let ref_kinds: Bound<'_, PyAny> = - imported(py, MACRO_ARGUMENTS_MODULE, "typed_reference_kinds")?.call0()?; + let ref_kind: Bound<'_, PyAny> = imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?; + let ref_kinds: Bound<'_, PyDict> = PyDict::new(py); let ref_functions: Bound<'_, PyDict> = PyDict::new(py); - for item in ref_kinds.call_method0("items")?.try_iter()? { - let (function, kind): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; - ref_functions.set_item(kind.getattr("value")?, function)?; + for (kind, function) in TYPED_REFERENCE_FUNCTIONS { + ref_kinds.set_item(function, ref_kind.call1((kind,))?)?; + ref_functions.set_item(kind, function)?; } let store_tokens: Bound<'_, PyDict> = keywords( py, @@ -152,7 +156,7 @@ fn built_classes(py: Python<'_>) -> PyResult> { "resource_ref", imported(py, REF_MODELS_MODULE, "SqlResourceRef")?, ), - ("ref_kinds", ref_kinds), + ("ref_kinds", ref_kinds.into_any()), ("ref_functions", ref_functions.into_any()), ("store_tokens", store_tokens.into_any()), ( @@ -245,21 +249,18 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = imported(py, REF_MODELS_MODULE, "SqlResourceRef")?; - let ref_kind: Bound<'py, PyAny> = imported(py, REF_TYPES_MODULE, "SqlResourceRefKind")?; - let mut references: Vec> = Vec::new(); - for (kind, name) in output.argument_references { - references.push(call_with( - &resource_ref, - &[ - ("kind", ref_kind.call1((kind,))?), - ("name", PyString::new(py, &name).into_any()), - ], - )?); - } - let argument_references: Bound<'py, PyAny> = - imported(py, MACRO_ARGUMENTS_MODULE, "call_site_sql_references")? - .call1((PyTuple::new(py, references)?,))?; + let argument_references: Bound<'py, PyAny> = if output.argument_references.is_empty() { + PyTuple::empty(py).into_any() + } else { + let calls: Vec = output + .argument_references + .iter() + .map(|(kind, name)| format!("{}(\"{name}\")", typed_reference_function(kind))) + .collect(); + let syntax: Bound<'py, PyAny> = + imported(py, SQL_ANALYSIS_MODELS_MODULE, "SqlLexicalSyntax")?.call0()?; + extract_references(&calls.join(" "), &syntax)?.into_any() + }; result_model( py, &[ @@ -272,6 +273,14 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult &'static str { + TYPED_REFERENCE_FUNCTIONS + .iter() + .find(|(typed, _)| *typed == kind) + .map_or("", |(_, function)| function) +} + fn optional<'py>(value: Bound<'py, PyAny>) -> Option> { (!value.is_none()).then_some(value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 37bee60bec..372315ecd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -11,6 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapshot; +use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::scopes::scope_cache::{ project_fingerprint, read_scope_index_cache, write_scope_index_cache, }; @@ -19,7 +20,6 @@ use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; const NATIVE_MODULE: &str = "sqlbuild._native"; -const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; const SCOPE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.scopes.constants"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -33,14 +33,11 @@ const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile._helpers.render. /// Where the scope index cache file of a project lives: `(directory, file name)`. fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { let py: Python<'_> = project_dir.py(); - let directory: Bound<'_, PyAny> = - imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? - .call1((project_dir,))? - .div(imported( - py, - SCOPE_CONSTANTS_MODULE, - "SCOPE_CACHE_DIRECTORY_NAME", - )?)?; + let directory: Bound<'_, PyAny> = cache_directory(project_dir)?.div(imported( + py, + SCOPE_CONSTANTS_MODULE, + "SCOPE_CACHE_DIRECTORY_NAME", + )?)?; Ok(( directory.extract()?, imported(py, SCOPE_CONSTANTS_MODULE, "SCOPE_CACHE_FILENAME")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs index a90e0ead0e..dabf3065ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -1,8 +1,39 @@ //! Project native planning failures into existing located diagnostic exceptions. -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::sql_test_glue::plan_values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; +use sqlbuild_analysis::sql_test_attachment::models::CteLocation; + +const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; + +/// The location of `call` inside one CTE of a SQL test block, else the CTE header, else the block. +fn cte_call_location<'py>( + py: Python<'py>, + test: &Bound<'py, PyAny>, + (cte, call): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let test_file: Bound<'py, PyAny> = test.getattr("test_file")?; + let call: Option = call.extract()?; + let located: CteLocation = sql_test_cte_location( + &test_file.getattr("contents")?.extract::()?, + &test + .getattr("test_block")? + .getattr("sql_body")? + .extract::()?, + &cte.extract::()?, + call.as_deref(), + ); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("path", test_file.getattr("relative_path")?)?; + arguments.set_item("line", located.line)?; + arguments.set_item("column", located.column)?; + arguments.set_item("end_line", located.end_line)?; + arguments.set_item("end_column", located.end_column)?; + imported(py, SOURCE_LOCATION_MODULE, "SourceLocation")?.call((), Some(&arguments)) +} pub(super) fn planning_error( py: Python<'_>, @@ -38,27 +69,10 @@ pub(super) fn planning_error( "Mock it in the test, for example {} AS (SELECT ...), or call __ref(\"\") on a model the test runs.", payload.get_item("mockCte")?.extract::()? ); - let kwargs = pyo3::types::PyDict::new(py); - use pyo3::types::PyDictMethods; + let kwargs = PyDict::new(py); kwargs.set_item("help", help)?; let class = if let Some(test) = found { - kwargs.set_item( - "location", - call_with( - &imported( - py, - "sqlbuild.compiler.compile.classes.sql_test_cte_locator", - "SqlTestCteLocator", - )? - .getattr("locate")?, - &[ - ("test_file", test.getattr("test_file")?), - ("test_block", test.getattr("test_block")?), - ("cte_name", cte), - ("call", call), - ], - )?, - )?; + kwargs.set_item("location", cte_call_location(py, &test, (&cte, &call))?)?; "SqlTestReferenceError" } else { kwargs.set_item( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 99fad543d4..304c4c4b9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -176,7 +176,10 @@ fn scan_sql_reference_calls<'py>( /// Logical SQL refs outside comments and quotes, reporting rejected calls as P012. #[pyfunction(name = "extract_sql_references")] #[pyo3(signature = (*, sql, syntax))] -fn extract_references<'py>(sql: &str, syntax: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn extract_references<'py>( + sql: &str, + syntax: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = syntax.py(); let (references, invalid_calls) = scanned(py, sql, syntax)?; let report: Bound<'py, PyAny> = imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?; diff --git a/fensu.toml b/fensu.toml index 4526751f7c..7d94b0e267 100644 --- a/fensu.toml +++ b/fensu.toml @@ -73,6 +73,11 @@ rule = "FFR309" path = "src/sqlbuild/cost/constants.py" reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/project_assembly/__init__.py" +reason = "Native bindings own project assembly and SQL syntax checks; this package only declares the resource fact models the native assembly builds." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 30190d8337..3107742933 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 14911 +remaining_lines = 14436 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8fdea511e5..e16906354e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2989,12 +2989,8 @@ class DiscoveredSqlModelFile: @property def extract_implicit_alias_columns(self) -> bool: ... -def parse_schema_document( - loaded: object, path: str, classes: dict[str, object] -) -> tuple[SchemaSeedEntry, ...]: ... -def parse_sources_document( - loaded: object, path: str, classes: dict[str, object] -) -> tuple[SourceEntry, ...]: ... +def parse_schema_document(loaded: object, path: str, /) -> tuple[SchemaSeedEntry, ...]: ... +def parse_sources_document(loaded: object, path: str, /) -> tuple[SourceEntry, ...]: ... def analyze_compile_project( *, project_dir: Path, @@ -3500,7 +3496,9 @@ def effective_config_connection( def effective_config_settings( settings: tuple[object, object], overrides: frozenset[str], names: list[str], / ) -> dict[str, object]: ... -def yaml_entry_line_index(contents: str, /) -> list[tuple[str, int]]: ... +def yaml_entry_line(*, contents: str, name: str) -> int | None: ... +def compiler_cache_directory(project_dir: Path, /) -> Path: ... +def project_relative_path(*, path: Path, project_dir: Path) -> Path: ... def resolve_target_name( *, project_config: ProjectConfig, local_config: LocalConfig, selected_target: str | None ) -> str | None: ... @@ -3671,9 +3669,6 @@ def load_project_config_file(project_dir: Path, /) -> ProjectConfig: ... def load_local_config_file(project_dir: Path, /) -> LocalConfig: ... def resolve_config_file_path(project_dir: Path, local: bool, /) -> Path: ... def validate_local_sql_analysis(project_dir: Path, flags: tuple[bool, bool, bool], /) -> None: ... -def sql_test_cte_span( - contents: str, sql_body: str, cte_name: str, call: str | None, / -) -> tuple[int, int, int, int]: ... def normalize_scope_path(*, path: str | PurePath) -> str: ... def scope_index_fingerprint(*, project_dir: Path) -> str | None: ... def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> ScopeIndex: ... @@ -3689,6 +3684,7 @@ class NativeConfigError: code: str | None help: str | None key: str | None + def exception(self, values: Mapping[str, object] | None = None) -> Exception: ... def parse_schema_columns( raw_columns: object, @@ -4576,9 +4572,6 @@ def sql_syntax_error( dialect: str | None, parse_one: bool = False, ) -> str | None: ... -def check_native_sql_syntax( - request: tuple[str, list[tuple[str, list[tuple[str, str]]]]], / -) -> bool: ... def assemble_native_project_resources( *, inputs: CompileInputsView, diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py deleted file mode 100644 index d2f2fac5da..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Typed reference arguments and explicit-reference enforcement for macro expansion.""" - -from __future__ import annotations - -from sqlbuild._native import extract_sql_references -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - -_TYPED_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = { - "__ref": SqlResourceRefKind.MODEL, - "__source": SqlResourceRefKind.SOURCE, - "__seed": SqlResourceRefKind.SEED, -} -_REFERENCE_FUNCTION_BY_KIND: dict[SqlResourceRefKind, str] = { - kind: function for function, kind in _TYPED_REFERENCE_KINDS.items() -} - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - - -def reference_call_text(ref: SqlResourceRef) -> str: - """Return the SQL reference call that names ``ref``, such as ``__ref("orders")``.""" - - return f'{_REFERENCE_FUNCTION_BY_KIND[ref.kind]}("{ref.name}")' - - -def call_site_sql_references( - refs: tuple[SqlResourceRef, ...], -) -> tuple[CompileSqlReference, ...]: - """Return the SQL references written as typed macro arguments, as SQL extraction sees them.""" - - if not refs: - return () - return extract_sql_references( - sql=" ".join(reference_call_text(ref) for ref in refs), syntax=_GENERIC_SQL_SYNTAX - ) - - -def typed_reference_kinds() -> dict[str, SqlResourceRefKind]: - """Each typed reference function, such as `__ref`, and the resource kind it names.""" - - return dict(_TYPED_REFERENCE_KINDS) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py deleted file mode 100644 index a01db8e479..0000000000 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py +++ /dev/null @@ -1,19 +0,0 @@ -"""SQL-test CTE extraction class.""" - -from __future__ import annotations - -from sqlbuild._native import extract_sql_test_ctes -from sqlbuild.compiler.compile.models import CompileSqlTestCte -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -class SqlTestCteExtractor: - """Expose authored SQL-test CTE parsing to compiler consumers.""" - - @staticmethod - def extract( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax - ) -> tuple[CompileSqlTestCte, ...]: - """Extract top-level SQL-test CTEs without classifying their roles.""" - - return extract_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py deleted file mode 100644 index 139df54c06..0000000000 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py +++ /dev/null @@ -1,32 +0,0 @@ -"""SQL-test CTE location class.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.spec.contracts.models import SourceLocation - - -class SqlTestCteLocator: - """Expose SQL-test CTE and reference-call locations to compiler consumers.""" - - @staticmethod - def locate( - *, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - cte_name: str, - call: str | None, - ) -> SourceLocation: - """Locate a call inside one CTE of a SQL test block, else the CTE header, else the block.""" - - line, column, end_line, end_column = _native.sql_test_cte_span( - test_file.contents, test_block.sql_body, cte_name, call - ) - return SourceLocation( - path=test_file.relative_path, - line=line, - column=column, - end_line=end_line, - end_column=end_column, - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index c981bd9dc3..5831d09637 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -14,11 +14,8 @@ from typing import TYPE_CHECKING, Any, get_type_hints import sqlbuild._native as _native -from sqlbuild._native import validate_resource_identity +from sqlbuild._native import project_relative_path, validate_resource_identity from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.discovery._helpers.filesystem.scoped_paths import ( - project_relative_path, -) from sqlbuild.compiler.discovery.constants import ( PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py deleted file mode 100644 index 0757686bc3..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/scoped_paths.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Project-relative path facts shared by filesystem discovery.""" - -from __future__ import annotations - -from pathlib import Path - - -def project_relative_path(*, path: Path, project_dir: Path) -> Path: - """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" - - root_parts: tuple[str, ...] = project_dir.parts - path_parts: tuple[str, ...] = path.parts - if ( - path.is_absolute() is not project_dir.is_absolute() - or path_parts[: len(root_parts)] != root_parts - ): - return path.relative_to(project_dir) - return path.with_segments(*path_parts[len(root_parts) :]) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py b/src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py deleted file mode 100644 index 2b96c6cf9f..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py +++ /dev/null @@ -1,45 +0,0 @@ -"""The Python classes native YAML entry parsing builds source and seed declarations as.""" - -from __future__ import annotations - -from functools import cache - -from sqlbuild.compiler.model_config.main._native_contract_classes import native_contract_classes -from sqlbuild.integrations.dlt.models import DltResourceConfig, DltSourceConfig -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import ( - IntegrationLoaderConfig, - SchemaSeedEntry, - SeedCsvSettings, - SourceColumnEntry, - SourceEntry, - SourceFreshnessAgePolicy, - SourceFreshnessConfig, -) -from sqlbuild.spec.contracts.types import ( - SourceFreshnessStrategy, - SourceFreshnessValueKind, - SourceWriteStrategy, -) - - -@cache -def yaml_entry_classes() -> dict[str, object]: - """The classes parsed seed and source declarations become.""" - - return { - "contracts": native_contract_classes(), - "schema_seed_entry": SchemaSeedEntry, - "seed_csv_settings": SeedCsvSettings, - "source_entry": SourceEntry, - "source_column_entry": SourceColumnEntry, - "source_freshness_config": SourceFreshnessConfig, - "source_freshness_age_policy": SourceFreshnessAgePolicy, - "integration_loader_config": IntegrationLoaderConfig, - "ingestr_source_config": IngestrSourceConfig, - "dlt_source_config": DltSourceConfig, - "dlt_resource_config": DltResourceConfig, - "source_write_strategy": SourceWriteStrategy, - "source_freshness_strategy": SourceFreshnessStrategy, - "source_freshness_value_kind": SourceFreshnessValueKind, - } diff --git a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py deleted file mode 100644 index 30fa09e4a7..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Locate one named entry in an authored YAML declaration file.""" - -from __future__ import annotations - -from collections.abc import Mapping -from functools import lru_cache - -import sqlbuild._native as _native - - -def yaml_entry_line(*, contents: str, name: str) -> int | None: - """Return the 1-based line of the outermost `- name: ` entry, if present.""" - - return _entry_lines(contents).get(name) - - -@lru_cache(maxsize=16) -def _entry_lines(contents: str) -> Mapping[str, int]: - return dict(_native.yaml_entry_line_index(contents)) diff --git a/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py b/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py deleted file mode 100644 index cf99e3f7e0..0000000000 --- a/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Project-local compiler store root.""" - -from pathlib import Path - -from sqlbuild.compiler.frontier.constants import ( - CACHE_DIRECTORY_NAME, - COMPILER_CACHE_DIRECTORY_NAME, - NATIVE_CACHE_NAMESPACE_SUFFIX, - TARGET_DIRECTORY_NAME, -) - - -def compiler_cache_directory(project_dir: Path) -> Path: - """Return the project's compiler store root.""" - - return ( - project_dir - / TARGET_DIRECTORY_NAME - / CACHE_DIRECTORY_NAME - / f"{COMPILER_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}" - ) diff --git a/src/sqlbuild/compiler/model_config/__init__.py b/src/sqlbuild/compiler/model_config/__init__.py deleted file mode 100644 index cdb238defa..0000000000 --- a/src/sqlbuild/compiler/model_config/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model configuration.""" diff --git a/src/sqlbuild/compiler/model_config/constants.py b/src/sqlbuild/compiler/model_config/constants.py deleted file mode 100644 index ff4deb0c27..0000000000 --- a/src/sqlbuild/compiler/model_config/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Names and error classes the native model config bindings share.""" - -from __future__ import annotations - -ENVIRONMENT_READ: str = "env" -CONFIG_VALUE_TYPE_ERROR: str = "config_value_type" -RESOURCE_IDENTITY_ERROR: str = "resource_identity" -DISCOVERY_CONFLICT_ERROR: str = "discovery_conflict" -EXPECTED_STRING: str = "a string" diff --git a/src/sqlbuild/compiler/model_config/main/__init__.py b/src/sqlbuild/compiler/model_config/main/__init__.py deleted file mode 100644 index 8665a5bb71..0000000000 --- a/src/sqlbuild/compiler/model_config/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model configuration entry points.""" diff --git a/src/sqlbuild/compiler/model_config/main/_native_config_error.py b/src/sqlbuild/compiler/model_config/main/_native_config_error.py deleted file mode 100644 index b83dab89d6..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_native_config_error.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Raise a native config stage's exact error as the Python stage raises it.""" - -from __future__ import annotations - -from collections.abc import Mapping - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.model_config.constants import ( - CONFIG_VALUE_TYPE_ERROR, - DISCOVERY_CONFLICT_ERROR, - EXPECTED_STRING, - RESOURCE_IDENTITY_ERROR, -) -from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.spec.contracts.exceptions import ConfigValueTypeError - - -def native_config_error( - *, - error: _native.NativeConfigError, - values: Mapping[str, object] | None = None, -) -> Exception: - """Return the exception class `error` names, with its message, code and help.""" - - if error.class_name == CONFIG_VALUE_TYPE_ERROR and error.key is not None and values is not None: - return ConfigValueTypeError( - key=error.key, expected=EXPECTED_STRING, actual_type=type(values[error.key]) - ) - if error.class_name == RESOURCE_IDENTITY_ERROR: - return ResourceIdentityError(error.message, help=error.help) - if error.class_name == DISCOVERY_CONFLICT_ERROR: - return DiscoveryConflictError(error.message, help=error.help) - return CompileInputError(error.message, code=error.code, help=error.help) diff --git a/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py b/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py deleted file mode 100644 index ee045084c6..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Contract classes native header parsing builds Python columns and audits from.""" - -from functools import cache - -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator -from sqlbuild.spec.contracts.models import ( - SchemaAuditInstance, - SchemaColumn, - SchemaDynamicColumnFamily, - SourceLocation, -) - - -@cache -def native_contract_classes() -> dict[str, object]: - """The contract classes native header parsing builds columns and audits as.""" - - return { - "schema_column": SchemaColumn, - "schema_audit_instance": SchemaAuditInstance, - "schema_dynamic_column_family": SchemaDynamicColumnFamily, - "source_location": SourceLocation, - "measurement_thresholds": MeasurementThresholds, - "measurement_threshold_bound": MeasurementThresholdBound, - "severities": {severity.value: severity for severity in AuditSeverity}, - "threshold_operators": {operator.value: operator for operator in ThresholdOperator}, - "allocate": object.__new__, - } diff --git a/src/sqlbuild/compiler/path_defaults/__init__.py b/src/sqlbuild/compiler/path_defaults/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/path_defaults/_helpers/__init__.py b/src/sqlbuild/compiler/path_defaults/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/path_defaults/_helpers/matching.py b/src/sqlbuild/compiler/path_defaults/_helpers/matching.py deleted file mode 100644 index f7caf920e1..0000000000 --- a/src/sqlbuild/compiler/path_defaults/_helpers/matching.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Path-default glob matching mechanics.""" - -from __future__ import annotations - -from functools import cache - -from sqlbuild.compiler.path_defaults.constants import ( - GLOB_SEGMENTS, - RECURSIVE_SEGMENT_GLOB, - SINGLE_SEGMENT_GLOB, -) - - -def matches_prefix(*, path_parts: tuple[str, ...], path_key: str) -> bool: - """Return whether a path-default pattern matches a prefix of a model path.""" - - pattern_parts: tuple[str, ...] = tuple(path_key.split("/")) - - @cache - def matches(*, pattern_index: int, path_index: int) -> bool: - if pattern_index == len(pattern_parts): - return True - pattern_part: str = pattern_parts[pattern_index] - if pattern_part == RECURSIVE_SEGMENT_GLOB: - return matches(pattern_index=pattern_index + 1, path_index=path_index) or ( - path_index < len(path_parts) - and matches(pattern_index=pattern_index, path_index=path_index + 1) - ) - if path_index == len(path_parts): - return False - if pattern_part == SINGLE_SEGMENT_GLOB or pattern_part == path_parts[path_index]: - return matches(pattern_index=pattern_index + 1, path_index=path_index + 1) - return False - - return matches(pattern_index=0, path_index=0) - - -def is_wildcard(*, path_key: str) -> bool: - """Return whether a path-default key contains a supported wildcard segment.""" - - return any(part in GLOB_SEGMENTS for part in path_key.split("/")) - - -def specificity(*, path_key: str) -> tuple[int, int, int]: - """Return deterministic wildcard specificity based on literal path structure.""" - - parts: tuple[str, ...] = tuple(path_key.split("/")) - literal_count: int = sum(part not in GLOB_SEGMENTS for part in parts) - single_segment_count: int = parts.count(SINGLE_SEGMENT_GLOB) - recursive_count: int = parts.count(RECURSIVE_SEGMENT_GLOB) - return literal_count, single_segment_count, -recursive_count diff --git a/src/sqlbuild/compiler/path_defaults/constants.py b/src/sqlbuild/compiler/path_defaults/constants.py deleted file mode 100644 index 094f5cba2c..0000000000 --- a/src/sqlbuild/compiler/path_defaults/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Constants for path-default glob matching.""" - -from __future__ import annotations - -SINGLE_SEGMENT_GLOB: str = "*" -RECURSIVE_SEGMENT_GLOB: str = "**" -GLOB_SEGMENTS: frozenset[str] = frozenset({SINGLE_SEGMENT_GLOB, RECURSIVE_SEGMENT_GLOB}) -UNSUPPORTED_GLOB_MARKERS: tuple[str, ...] = (SINGLE_SEGMENT_GLOB, "?", "[") diff --git a/src/sqlbuild/compiler/path_defaults/main/__init__.py b/src/sqlbuild/compiler/path_defaults/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/path_defaults/main/_select.py b/src/sqlbuild/compiler/path_defaults/main/_select.py deleted file mode 100644 index 7970f191e3..0000000000 --- a/src/sqlbuild/compiler/path_defaults/main/_select.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Shared path-default matching and selection entrypoint.""" - -from __future__ import annotations - -from functools import lru_cache - -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.path_defaults._helpers.matching import ( - is_wildcard, - matches_prefix, - specificity, -) -from sqlbuild.compiler.path_defaults.models import PathDefaultSelection - - -def select_path_default(*, model_path: str, path_keys: tuple[str, ...]) -> PathDefaultSelection: - """Match and deterministically select a path default for a model path.""" - - normalized_path: str = model_path.replace("\\", "/").removeprefix("models/") - return _select_normalized_path_default(normalized_path=normalized_path, path_keys=path_keys) - - -@lru_cache(maxsize=32_768) -def _select_normalized_path_default( - *, normalized_path: str, path_keys: tuple[str, ...] -) -> PathDefaultSelection: - path_parts: tuple[str, ...] = tuple(part for part in normalized_path.split("/") if part) - matched_keys: tuple[str, ...] = tuple( - sorted( - path_key - for path_key in path_keys - if matches_prefix(path_parts=path_parts, path_key=path_key) - ) - ) - literal_matches: tuple[str, ...] = tuple( - path_key for path_key in matched_keys if not is_wildcard(path_key=path_key) - ) - if literal_matches: - return PathDefaultSelection( - matched_keys=matched_keys, - selected_key=max(literal_matches, key=lambda path_key: len(path_key.split("/"))), - ) - if not matched_keys: - return PathDefaultSelection(matched_keys=(), selected_key=None) - - best_score: tuple[int, int, int] = max( - specificity(path_key=path_key) for path_key in matched_keys - ) - best_matches: tuple[str, ...] = tuple( - path_key for path_key in matched_keys if specificity(path_key=path_key) == best_score - ) - if len(best_matches) > 1: - conflicting_keys: str = ", ".join(f"'{path_key}'" for path_key in best_matches) - raise DiscoveryConflictError( - f"Model path '{normalized_path}' matches equally specific path_defaults keys: " - f"{conflicting_keys}.", - help=( - "Make one pattern narrower by adding literal or '*' path segments, or remove the " - "overlap. Path-default selection never depends on declaration order." - ), - ) - return PathDefaultSelection(matched_keys=matched_keys, selected_key=best_matches[0]) diff --git a/src/sqlbuild/compiler/path_defaults/models.py b/src/sqlbuild/compiler/path_defaults/models.py deleted file mode 100644 index a97bce0052..0000000000 --- a/src/sqlbuild/compiler/path_defaults/models.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Structured path-default selection models.""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PathDefaultSelection: - """Deterministic path-default matching result for one model path.""" - - matched_keys: tuple[str, ...] - selected_key: str | None diff --git a/src/sqlbuild/compiler/project_assembly/main/__init__.py b/src/sqlbuild/compiler/project_assembly/main/__init__.py deleted file mode 100644 index 646fd16ff4..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native compiled project assembly entry points.""" diff --git a/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py b/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py deleted file mode 100644 index 0a00801549..0000000000 --- a/src/sqlbuild/compiler/project_assembly/main/_check_native_sql_syntax.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Check SQL syntax natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def check_native_sql_syntax( - *, checks: tuple[tuple[str, dict[str, str] | None], ...], dialect: str | None -) -> bool: - """Whether every `(sql, placeholders)` parses; raises Python's pre-parse `ValueError`.""" - - return _native.check_native_sql_syntax( - ( - dialect or "generic", - [(sql, list((placeholders or {}).items())) for sql, placeholders in checks], - ) - ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py b/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py deleted file mode 100644 index d21014a46c..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Normalize analysis SQL using native offset-preserving transformations.""" - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.sql_analysis.types import NativePositionsModule - - -def normalize_analysis_sql( - *, - sql: str, - dialect: str | None, - stubs: dict[str, str] | None = None, - placeholders: dict[str, str] | None = None, -) -> str: - return cast(NativePositionsModule, _native).normalize_analysis_sql( - { - "sql": sql, - "dialect": dialect or "generic", - "stubs": stubs or {}, - "placeholders": placeholders or {}, - } - ) diff --git a/src/sqlbuild/presentation/_helpers/structure.py b/src/sqlbuild/presentation/_helpers/structure.py deleted file mode 100644 index dfe6e82998..0000000000 --- a/src/sqlbuild/presentation/_helpers/structure.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Structural CLI vocabulary implementations (trees, phase lines, headers).""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.constants import ( - TREE_BRANCH_GLYPH, - TREE_LAST_GLYPH, -) - - -def tree_connector(*, style: CliStyle, last: bool) -> str: - """Render a dim tree connector for a group entry.""" - - return style.muted(TREE_LAST_GLYPH if last else TREE_BRANCH_GLYPH) - - -def format_surface_header(*, style: CliStyle, title: str, context: str | None = None) -> str: - """Render a command surface header: accent title plus dim context.""" - - rendered: str = style.title(title) - if context: - rendered = f"{rendered} {style.muted(context)}" - return rendered diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 49fe59ce12..23b64d7332 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild._native import compiler_cache_directory from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( RuntimeArtifactPreservationBuildE2ETestCase, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 9d8d4409f8..3bde194cc4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -41,6 +41,7 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from sqlbuild._native import compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect @@ -58,7 +59,6 @@ COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, ) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index 55fa9e2633..3bcbe1fb76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -8,7 +8,7 @@ from pathlib import Path from time import perf_counter -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild._native import compiler_cache_directory from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index c3a4911d69..6dab1de89a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -11,7 +11,7 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import PreparedCompileArtifacts +from sqlbuild._native import PreparedCompileArtifacts, compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module @@ -20,7 +20,6 @@ COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, ) from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory STAGING_PREFIX: str = ".sqlbuild-staging-" NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index 144f892d98..8504effbe6 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -15,7 +15,6 @@ NativeModelConfigSession, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ProjectConfig @@ -83,10 +82,7 @@ def validation_outcome(*, values: dict[str, object]) -> object: ) return [ _ACCEPTED, - *( - raised_error_shape(native_config_error(error=error, values=values)) - for error in filter(None, (outcome,)) - ), + *(raised_error_shape(error.exception(values=values)) for error in filter(None, (outcome,))), ][-1] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py index cafffb3ff6..ab108f5ff5 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import run_cli as main -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( IdentifierBindingCase, NativeCatalogCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import normalize_analysis_sql @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 230ed782a7..b81b5a40bf 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -82,22 +82,6 @@ class RetentionValidationValidTestCase: expected_unmanaged: bool -@dataclass(frozen=True) -class FindMatchingPathDefaultTestCase: - description: str - model_relative_path: str - path_defaults: dict[str, dict[str, object]] - expected_path_default: str | None - - -@dataclass(frozen=True) -class FindMatchingPathDefaultErrorTestCase: - description: str - model_relative_path: str - path_defaults: dict[str, dict[str, object]] - expected_error_fragment: str - - @dataclass(frozen=True) class LoadProjectMacrosTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 15e844b23e..2c34d0dfde 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -16,7 +16,6 @@ NativeModelConfigSession, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, @@ -87,7 +86,7 @@ def validate_natively( # noqa: PLR0913 ) assert not isinstance(outcome, int) for error in filter(None, (outcome,)): - raise native_config_error(error=error, values=config.values) + raise error.exception(values=config.values) def build_config_natively( # noqa: PLR0913 @@ -127,7 +126,7 @@ def build_config_natively( # noqa: PLR0913 native_config_classes(session=session), ) for error in filter(_is_config_error, (built,)): - raise native_config_error(error=cast(_native.NativeConfigError, error)) + raise cast(_native.NativeConfigError, error).exception() return cast(CompileModelConfig, built) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py deleted file mode 100644 index 91845ccf44..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_path_defaults.py +++ /dev/null @@ -1,133 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.path_defaults.main._select import select_path_default -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - FindMatchingPathDefaultErrorTestCase, - FindMatchingPathDefaultTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - FindMatchingPathDefaultTestCase( - description="matches Windows-style model path against nested path default", - model_relative_path="models\\staging\\nested\\orders.sql", - path_defaults={"staging": {}, "staging/nested": {}}, - expected_path_default="staging/nested", - ), - FindMatchingPathDefaultTestCase( - description="recursive glob matches a staging directory prefix", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={"commerce/**/staging": {}}, - expected_path_default="commerce/**/staging", - ), - FindMatchingPathDefaultTestCase( - description="recursive glob matches zero path segments", - model_relative_path="models/commerce/staging/orders.sql", - path_defaults={"commerce/**/staging": {}}, - expected_path_default="commerce/**/staging", - ), - FindMatchingPathDefaultTestCase( - description="single glob consumes exactly one path segment", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={"commerce/*/staging": {}}, - expected_path_default="commerce/*/staging", - ), - FindMatchingPathDefaultTestCase( - description="single glob does not consume multiple path segments", - model_relative_path="models/commerce/eu/partner_feed/staging/orders.sql", - path_defaults={"commerce/*/staging": {}}, - expected_path_default=None, - ), - FindMatchingPathDefaultTestCase( - description="literal prefix outranks a matching recursive glob", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={ - "commerce/**/staging": {}, - "commerce/partner_feed/staging": {}, - }, - expected_path_default="commerce/partner_feed/staging", - ), - FindMatchingPathDefaultTestCase( - description="single segment glob outranks recursive glob", - model_relative_path="models/commerce/partner_feed/staging/orders.sql", - path_defaults={ - "commerce/**/staging": {}, - "commerce/*/staging": {}, - }, - expected_path_default="commerce/*/staging", - ), - FindMatchingPathDefaultTestCase( - description="recursive glob matches Windows-style model path", - model_relative_path="models\\commerce\\partner_feed\\staging\\orders.sql", - path_defaults={"commerce/**/staging": {}}, - expected_path_default="commerce/**/staging", - ), - ), - ids=lambda case: case.description, -) -def test_given_windows_style_model_path_when_finding_path_default_then_returns_nearest_match( - test_case: FindMatchingPathDefaultTestCase, -) -> None: - model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( - file_path=Path("/repo") / test_case.model_relative_path, - relative_path=Path(test_case.model_relative_path), - contents="MODEL (description 'Test model.');\n\nselect 1\n", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="select 1", - ) - - path_default: str | None = select_path_default( - model_path=str(model_file.relative_path), path_keys=tuple(test_case.path_defaults) - ).selected_key - - assert path_default == test_case.expected_path_default - - -@pytest.mark.parametrize( - "test_case", - ( - FindMatchingPathDefaultErrorTestCase( - description="equally specific matching globs are ambiguous regardless of values", - model_relative_path="models/commerce/eu/staging/orders.sql", - path_defaults={ - "commerce/*/staging": {"schema": "shared"}, - "commerce/eu/*": {"schema": "shared"}, - }, - expected_error_fragment=( - r"matches equally specific path_defaults keys: " - r"'commerce/\*/staging', 'commerce/eu/\*'" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_ambiguous_globs_when_finding_path_default_then_raises_structured_error( - test_case: FindMatchingPathDefaultErrorTestCase, -) -> None: - model_file: DiscoveredSqlModelFile = DiscoveredSqlModelFile( - file_path=Path("/repo") / test_case.model_relative_path, - relative_path=Path(test_case.model_relative_path), - contents="MODEL (description 'Test model.');\n\nselect 1\n", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="select 1", - ) - - with pytest.raises(DiscoveryConflictError, match=test_case.expected_error_fragment) as error: - select_path_default( - model_path=str(model_file.relative_path), path_keys=tuple(test_case.path_defaults) - ) - - assert error.value.code == "D007" - assert error.value.help is not None diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index f6b19d46e7..19f4aba554 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,7 +1,6 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.discovery._helpers.yml.yaml_classes import yaml_entry_classes from sqlbuild.compiler.discovery.constants import ( REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, @@ -95,7 +94,6 @@ def parse_sources_yml(*, contents: str, file_path: Path) -> tuple[SourceEntry, . return _native.parse_sources_document( _native.load_yaml_document(str(file_path), contents, "source"), str(file_path), - yaml_entry_classes(), ) @@ -107,5 +105,4 @@ def parse_schema_yml( return (), _native.parse_schema_document( _native.load_yaml_document(str(file_path), contents, "schema"), str(file_path), - yaml_entry_classes(), ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py b/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py index 278ad2b932..c1c98d2e03 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line +from sqlbuild._native import yaml_entry_line from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import YamlEntryLineTestCase _SOURCES: str = ( diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index 54e38e925e..64ec4f7b5f 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -9,8 +9,11 @@ import pytest -from sqlbuild._native import load_or_build_scope_index, scope_index_fingerprint -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild._native import ( + compiler_cache_directory, + load_or_build_scope_index, + scope_index_fingerprint, +) from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from sqlbuild.compiler.scopes.models import DeclarationIdentity, ScopeIndex from sqlbuild.compiler.scopes.types import ( diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py index 1e58ac30a7..040d0a2090 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py @@ -75,3 +75,22 @@ def normalize_sql_for_polyglot(*, sql: str, dialect: str | None) -> str: return cast(NativePositionsModule, _native).normalize_dialect_sql( sql=sql, dialect=dialect or "generic" ) + + +def normalize_analysis_sql( + *, + sql: str, + dialect: str | None, + stubs: Mapping[str, str] | None = None, + placeholders: Mapping[str, str] | None = None, +) -> str: + """Normalize one analysis SQL natively.""" + + return cast(NativePositionsModule, _native).normalize_analysis_sql( + { + "sql": sql, + "dialect": dialect or "generic", + "stubs": dict(stubs or {}), + "placeholders": dict(placeholders or {}), + } + ) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py index 038b660a85..0975cc0988 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py @@ -2,10 +2,10 @@ import pytest -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( PolyglotSqlNormalizationTestCase, ) +from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import normalize_analysis_sql @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py index 74f0ebef88..91468e9505 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis_batch.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( NormalizationBatchFailureTestCase, NormalizationBatchTestCase, NormalizationChunkTestCase, ) from tests.unit.src.sqlbuild.compiler.sql_analysis.main.helpers import ( + normalize_analysis_sql, normalize_analysis_sql_results, pool_catalog, without_catalog, From 54b39727e005a115767ae2bd41df6ce3ad0cbef5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:53:37 +0100 Subject: [PATCH 20/33] refactor: collect compile diagnostics and input reads natively Move the compile diagnostic collector, the process-wide template input read registry, the SQL-test scan cache and the UTF-8 checked environment view into the native bindings, and delete the Python classes they replace. --- .../_helpers/cache/sql_test_scan_store.rs | 81 +++++++++- .../compile_attachment/declaration_scope.rs | 2 +- .../compile_attachment/model_config.rs | 7 +- .../compile_attachment/model_inputs.rs | 9 +- .../_helpers/compile_driver/assembly.rs | 2 +- .../compile_driver/assembly_sql_tests.rs | 2 +- .../_helpers/compile_driver/command.rs | 4 +- .../compile_driver/diagnostic_collector.rs | 142 ++++++++++++++++++ .../_helpers/compile_driver/input_reads.rs | 117 ++++++++++++++- .../_helpers/compile_driver/inputs.rs | 2 +- .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/registration.rs | 6 +- .../src/bindings/_helpers/model_config/mod.rs | 1 + .../_helpers/model_config/registration.rs | 3 +- .../_helpers/model_config/template_data.rs | 16 +- .../model_config/unicode_environment.rs | 79 ++++++++++ .../bindings/_helpers/render/authored_sql.rs | 4 +- .../_helpers/render/macro_expansion_result.rs | 2 +- .../_helpers/scopes/compile_usages.rs | 2 +- .../bindings/_helpers/scopes/scope_errors.rs | 2 +- .../_helpers/sqltext/sql_references.rs | 2 +- src/sqlbuild/_native.pyi | 31 +++- .../compile/_helpers/diagnostics/collector.py | 50 ------ .../compiler/compile/classes/__init__.py | 0 .../classes/collected_compile_diagnostics.py | 23 --- .../classes/compile_input_read_registry.py | 51 ------- .../compile/classes/sql_test_scan_cache.py | 44 ------ .../compile/classes/unicode_environment.py | 37 ----- src/sqlbuild/compiler/compile/constants.py | 4 - src/sqlbuild/compiler/compile/models.py | 2 +- .../discovery/_helpers/filesystem/core.py | 11 +- .../compiler/project_assembly/helpers.py | 4 +- .../compiler/compile/_helpers/helpers.py | 4 +- .../compile/_helpers/test_sql_references.py | 3 +- .../compile/_helpers/test_templating.py | 2 +- 35 files changed, 489 insertions(+), 263 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py delete mode 100644 src/sqlbuild/compiler/compile/classes/__init__.py delete mode 100644 src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py delete mode 100644 src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py delete mode 100644 src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py delete mode 100644 src/sqlbuild/compiler/compile/classes/unicode_environment.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs index 6c69245ecd..a315e0f06a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs @@ -1,7 +1,8 @@ //! Whole per-file SQL-test scan results kept as one kind of the shared native store. use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyclass, pymethods}; use sqlbuild_cache::digest::main::content_digest::content_digest; use sqlbuild_cache::digest::main::hex_digest::hex_digest; @@ -12,6 +13,11 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; +const METRIC_MODULE: &str = "sqlbuild.compiler.profiling.main._metric"; +const SCAN_CACHE_LOGGER: &str = "sqlbuild.compile"; /// The native store kind holding SQL-test scan results. const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; @@ -111,8 +117,81 @@ impl SqlTestScanStore { } } +/// The compile cache folder's scan store, opened on first use and saved once. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SqlTestScanCache { + state: Mutex<(Option, Option>)>, +} + +#[pymethods] +impl SqlTestScanCache { + #[new] + #[pyo3(signature = (*, cache_dir))] + fn new(cache_dir: Option) -> Self { + Self { + state: Mutex::new((cache_dir, None)), + } + } + + /// The native store this compile reads and writes, or None when it has none. + fn opened(&self, py: Python<'_>) -> PyResult>> { + let mut state = self.state.lock().map_err(poisoned)?; + let (cache_dir, store) = &mut *state; + let Some(directory) = cache_dir.as_ref().filter(|_| store.is_none()) else { + return Ok(store.as_ref().map(|store| store.clone_ref(py))); + }; + let identity: String = imported(py, CODE_IDENTITY_MODULE, "compiled_code_identity")? + .call0()? + .extract()?; + let (opened, error) = SqlTestScanStore::open_in(py, directory.clone(), &identity); + *store = opened.map(|opened| Py::new(py, opened)).transpose()?; + if let Some(error) = error { + debug(py, "SQL test scan store not loaded: %s", &error)?; + *cache_dir = None; + } + Ok(store.as_ref().map(|store| store.clone_ref(py))) + } + + /// Persist results stored by this compile and report hit and miss counts. + fn save(&self, py: Python<'_>) -> PyResult<()> { + let store: Option> = self + .state + .lock() + .map_err(poisoned)? + .1 + .as_ref() + .map(|store| store.clone_ref(py)); + let (hits, misses) = store.as_ref().map_or((0, 0), |store| store.get().counts()); + metric(py, "sql_test_scan_cache_hits", hits)?; + metric(py, "sql_test_scan_cache_misses", misses)?; + let Some(store) = store else { + return Ok(()); + }; + if let (_, Some(error)) = store.get().save(py)? { + debug(py, "SQL test scan store not saved: %s", &error)?; + } + Ok(()) + } +} + +fn metric(py: Python<'_>, name: &str, value: usize) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("metric", name)?; + keywords.set_item("value", value)?; + imported(py, METRIC_MODULE, "record_compile_metric")?.call((), Some(&keywords))?; + Ok(()) +} + +fn debug(py: Python<'_>, message: &str, error: &str) -> PyResult<()> { + imported(py, "logging", "getLogger")? + .call1((SCAN_CACHE_LOGGER,))? + .call_method1("debug", (message, error))?; + Ok(()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_class::()?; module.add( "SQL_TEST_SCAN_STORE_FILE_NAME", SQL_TEST_SCAN_STORE_FILE_NAME, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index f3769dc70d..82f8bd7b0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; const NATIVE_MODULE: &str = "sqlbuild._native"; -const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; +const SCAN_CACHE_MODULE: &str = "sqlbuild._native"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 14b2053410..2bca63eb99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -7,11 +7,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_attachment::storage_policies::inherited_storage; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; /// Capture the project inputs every model's native config build and validation read. @@ -49,10 +49,7 @@ pub(crate) fn native_model_config_session<'py>( project_config.getattr("path_defaults")?, hook_entries, ), - ( - inputs.getattr("effective_vars")?, - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, - ), + (inputs.getattr("effective_vars")?, unicode_environment(py)?), ( ( inputs.getattr("effective_target_name")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index c8c558a2c2..ca5b0aebec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; @@ -23,8 +24,7 @@ const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; @@ -127,10 +127,7 @@ fn sql_hook_inputs<'py>( ), ("hook_files", discovered.getattr("sql_hook_files")?), ("effective_vars", context.getattr("effective_vars")?), - ( - "environment", - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, - ), + ("environment", unicode_environment(py)?), ( "render_project_var", imported(py, PROJECT_VARS_MODULE, "render_project_var_text")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 635fbd2cd9..69309e0a44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const RECORD_ANALYSES_MODULE: &str = "sqlbuild._native"; const EXPLICIT_REFERENCES_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 40601cab98..48343b56db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; const DATACLASSES_MODULE: &str = "dataclasses"; const SQL_TEST_INPUT_FAILURE: &str = "input"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 2cbe5f40e7..c2cdee3133 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -25,7 +25,6 @@ use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; const BYPASS_OUTCOME: &str = "bypass"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; @@ -151,8 +150,7 @@ fn recorded_compile<'py>( &imported(py, PROFILING_MODULE, "collect_compile_timings")?.call0()?, |detailed_timings| { within( - &imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")? - .call_method0("recording")?, + &imported(py, NATIVE_MODULE, "record_compile_input_reads")?.call0()?, |input_reads| { let artifact_writes: Py = new_artifact_recorder(py)?; let manager: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs new file mode 100644 index 0000000000..7d0777d104 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -0,0 +1,142 @@ +//! Collect compile diagnostics so one compile reports every violation, not only the first. + +use std::sync::Mutex; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_collected_compile_diagnostics"; + +static ACTIVE_COLLECTOR: PyOnceLock> = PyOnceLock::new(); + +/// The context variable holding the collection diagnostics are reported into, if any. +fn active_collector(py: Python<'_>) -> PyResult> { + ACTIVE_COLLECTOR + .get_or_try_init(py, || { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")? + .call((CONTEXT_VARIABLE_NAME,), Some(&keywords)) + .map(Bound::unbind) + }) + .map(|variable| variable.bind(py).clone()) +} + +/// Deduplicated compile diagnostics gathered while the collection is entered. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CollectedCompileDiagnostics { + by_key: Py, + token: Mutex>>, +} + +#[pymethods] +impl CollectedCompileDiagnostics { + fn __enter__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let token: Bound<'_, PyAny> = + active_collector(slf.py())?.call_method1("set", (slf.clone(),))?; + *slf.get() + .token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(token.unbind()); + Ok(slf.clone()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self + .token + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(token) = token { + active_collector(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } + + /// Record one diagnostic; a repeated key keeps the first diagnostic. + #[pyo3(signature = (*, key, diagnostic))] + fn add( + &self, + py: Python<'_>, + key: &Bound<'_, PyAny>, + diagnostic: &Bound<'_, PyAny>, + ) -> PyResult<()> { + self.by_key + .bind(py) + .call_method1("setdefault", (key, diagnostic))?; + Ok(()) + } + + /// The recorded diagnostics in deterministic key order. + #[getter] + fn diagnostics<'py>(&self, py: Python<'py>) -> PyResult> { + let by_key: &Bound<'py, PyDict> = self.by_key.bind(py); + let keys: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((by_key,))?; + let diagnostics: Vec> = keys + .try_iter()? + .map(|key| by_key.as_any().get_item(key?)) + .collect::>()?; + PyTuple::new(py, diagnostics) + } +} + +/// A new collection; entering it collects diagnostics instead of raising the first one. +#[pyfunction] +pub(crate) fn collect_compile_diagnostics( + py: Python<'_>, +) -> PyResult> { + Bound::new( + py, + CollectedCompileDiagnostics { + by_key: PyDict::new(py).unbind(), + token: Mutex::new(None), + }, + ) +} + +/// Record a diagnostic in the active collection, or raise it when none is active. +#[pyfunction] +#[pyo3(signature = (*, key, diagnostic))] +pub(crate) fn report_compile_diagnostic( + key: &Bound<'_, PyAny>, + diagnostic: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = key.py(); + let collector: Bound<'_, PyAny> = active_collector(py)?.call_method0("get")?; + if let Ok(collector) = collector.cast::() { + return collector.get().add(py, key, diagnostic); + } + let location: Bound<'_, PyAny> = diagnostic.getattr("location")?; + let path: Bound<'_, PyAny> = diagnostic.getattr("path")?; + let pointer: String = if !location.is_none() { + format!( + "\n --> {}:{}", + location.getattr("path")?.call_method0("as_posix")?.str()?, + location.getattr("line")?.str()? + ) + } else if !path.is_none() { + format!("\n --> {}", path.call_method0("as_posix")?.str()?) + } else { + String::new() + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", diagnostic.getattr("code")?)?; + keywords.set_item("help", diagnostic.getattr("help")?)?; + let message: String = format!("{}{pointer}", diagnostic.getattr("message")?.str()?); + Err(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call((message,), Some(&keywords))?, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(collect_compile_diagnostics, module)?)?; + module.add_function(wrap_pyfunction!(report_compile_diagnostic, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs index 0bc33c1533..bccd5ac5c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs @@ -4,8 +4,118 @@ use std::collections::BTreeSet; use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; -use pyo3::{pyclass, pymethods}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +const RUN_ID_KEY: &str = "run.id"; + +/// The recorders of every compile currently recording, delivered to from every thread. +#[pyclass(frozen, module = "sqlbuild._native")] +struct CompileInputReadRegistry { + recorders: Mutex>>, +} + +impl CompileInputReadRegistry { + fn locked(&self) -> MutexGuard<'_, Vec>> { + self.recorders + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + fn recorders(&self, py: Python<'_>) -> Vec> { + self.locked() + .iter() + .map(|recorder| recorder.clone_ref(py)) + .collect() + } +} + +static REGISTRY: PyOnceLock> = PyOnceLock::new(); + +fn registry(py: Python<'_>) -> PyResult<&Py> { + REGISTRY.get_or_try_init(py, || { + Py::new( + py, + CompileInputReadRegistry { + recorders: Mutex::new(Vec::new()), + }, + ) + }) +} + +fn recorders(py: Python<'_>) -> PyResult>> { + Ok(registry(py)?.get().recorders(py)) +} + +/// Note that a template looked up one environment variable, present or not. +pub(crate) fn environment_read(py: Python<'_>, name: &str) -> PyResult<()> { + for recorder in recorders(py)? { + recorder.get().add_environment_name(name.to_owned()); + } + Ok(()) +} + +/// Note a CTX lookup; only the per-invocation run identity is volatile. +pub(crate) fn context_read(py: Python<'_>, name: &str) -> PyResult<()> { + if name != RUN_ID_KEY { + return Ok(()); + } + for recorder in recorders(py)? { + recorder.get().mark_run_id_read(); + } + Ok(()) +} + +/// Note a provider settings class instantiated from the environment and files. +#[pyfunction] +fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { + for recorder in recorders(settings_class.py())? { + recorder.get().add_settings_class(settings_class); + } + Ok(()) +} + +/// Records environment variables and run-identity reads while entered. +#[pyclass(frozen, module = "sqlbuild._native")] +struct CompileInputRecording { + recorder: Mutex>>, +} + +#[pymethods] +impl CompileInputRecording { + fn __enter__<'py>(&self, py: Python<'py>) -> PyResult> { + let recorder: Bound<'py, CompileInputReads> = Bound::new(py, CompileInputReads::new())?; + registry(py)?.get().locked().push(recorder.clone().unbind()); + *self.recorder.lock().unwrap_or_else(PoisonError::into_inner) = + Some(recorder.clone().unbind()); + Ok(recorder) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + if let Some(recorder) = self + .recorder + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + { + registry(py)? + .get() + .locked() + .retain(|item| !item.is(&recorder)); + } + Ok(false) + } +} + +/// Record template input reads from every thread until the returned block exits. +#[pyfunction] +fn record_compile_input_reads() -> CompileInputRecording { + CompileInputRecording { + recorder: Mutex::new(None), + } +} #[derive(Default)] struct Reads { @@ -83,5 +193,8 @@ impl CompileInputReads { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(record_settings_class_read, module)?)?; + module.add_function(wrap_pyfunction!(record_compile_input_reads, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index d4ad6d5525..944e325e08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::model_config::template_data::run_id; const CALL_STORE_MODULE: &str = "sqlbuild.compiler.compile._helpers.macro_bridge.call_store"; const ATTACHMENT_MODULE: &str = "sqlbuild._native"; const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 102f4a0e04..d7352be8ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod assembly_sql_tests; pub(crate) mod command; pub(crate) mod config_validation; pub(crate) mod deferral; +pub(crate) mod diagnostic_collector; pub(crate) mod diff; pub(crate) mod discovery; pub(crate) mod input_reads; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 050f838115..9a3810ab61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -3,8 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ - analysis, assembly, command, deferral, diff, discovery, input_reads, inputs, pipeline, - pipeline_entries, project, python_plan_entries, relation_targets, selection_phases, targets, + analysis, assembly, command, deferral, diagnostic_collector, diff, discovery, input_reads, + inputs, pipeline, pipeline_entries, project, python_plan_entries, relation_targets, + selection_phases, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -18,6 +19,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { pipeline::register(module)?; pipeline_entries::register(module)?; deferral::register(module)?; + diagnostic_collector::register(module)?; diff::register(module)?; python_plan_entries::register(module)?; relation_targets::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 69bf36cd84..0d0f9cbb49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -13,3 +13,4 @@ pub(crate) mod model_validator; pub(crate) mod project_defaults; pub(crate) mod registration; pub(crate) mod template_data; +pub(crate) mod unicode_environment; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 8f9293b80f..fc8b3615f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ column_migrations, config_errors, config_values, described_models, header_columns, - model_config_builder, model_validator, template_data, + model_config_builder, model_validator, template_data, unicode_environment, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -16,5 +16,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { model_config_builder::register(module)?; model_validator::register(module)?; template_data::register(module)?; + unicode_environment::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs index 696826b8e1..dcd165dfa1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -11,16 +11,16 @@ use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_a use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::input_reads::{context_read, environment_read}; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_templates::{ TemplateOutcome, TemplateSources, expand_config_templates, lone_surrogate_site, resolve_effective_vars, }; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const UUID_MODULE: &str = "uuid"; const ENVIRONMENT_READ: &str = "env"; const RUN_ID_KEY: &str = "run.id"; @@ -34,14 +34,12 @@ pub(crate) fn record_reads<'a>( py: Python<'_>, reads: impl IntoIterator, ) -> PyResult<()> { - let registry: Bound<'_, PyAny> = imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")?; for (kind, name) in reads { - let method: &str = if kind == ENVIRONMENT_READ { - "environment_read" + if kind == ENVIRONMENT_READ { + environment_read(py, name)?; } else { - "context_read" - }; - registry.call_method1(method, (name,))?; + context_read(py, name)?; + } } Ok(()) } @@ -61,7 +59,7 @@ fn applied<'py>(py: Python<'py>, outcome: TemplateOutcome<'py>) -> PyResult) -> PyResult> { - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0() + unicode_environment(py) } /// `value` with every `${...}` template expanded, recording the reads it made. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs new file mode 100644 index 0000000000..5a2970e1dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs @@ -0,0 +1,79 @@ +//! The process environment as compile text reads it, rejecting values that are not UTF-8. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyString}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; + +/// `os.environ`, raising a clear error for a value holding undecodable bytes. +#[pyclass(module = "sqlbuild._native", frozen, mapping)] +pub(crate) struct UnicodeEnvironment; + +fn environ(py: Python<'_>) -> PyResult> { + imported(py, "os", "environ") +} + +#[pymethods] +impl UnicodeEnvironment { + #[new] + fn new() -> Self { + Self + } + + fn __getitem__<'py>(&self, name: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = name.py(); + let value: Bound<'py, PyAny> = environ(py)?.get_item(name)?; + let encoded: Bound<'py, PyBytes> = value + .cast::()? + .call_method1("encode", ("utf-8", "surrogatepass"))? + .cast_into::()?; + let byte_offset: usize = match std::str::from_utf8(encoded.as_bytes()) { + Ok(_) => return Ok(value), + Err(error) => error.valid_up_to(), + }; + let name: String = name.str()?.to_string(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item( + "help", + format!( + "Set '{name}' to UTF-8 text, since SQLBuild cannot send undecodable bytes to a \ + warehouse; the value is not shown because it may be a secret" + ), + )?; + Err(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call( + (format!( + "Environment variable '{name}' is not valid UTF-8 text: byte {byte_offset} \ + starts an invalid UTF-8 sequence" + ),), + Some(&keywords), + )?, + )) + } + + fn __contains__(&self, name: &Bound<'_, PyAny>) -> PyResult { + environ(name.py())?.contains(name) + } + + fn __iter__<'py>(&self, py: Python<'py>) -> PyResult> { + environ(py)?.try_iter().map(Bound::into_any) + } + + fn __len__(&self, py: Python<'_>) -> PyResult { + environ(py)?.len() + } +} + +/// A new view of the process environment as compile text reads it. +pub(crate) fn unicode_environment(py: Python<'_>) -> PyResult> { + Ok(Bound::new(py, UnicodeEnvironment)?.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 05dfc3df4a..38926e328f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -9,6 +9,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::render::declaration_expansion::{ DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, expand_declarations, span_rows, @@ -22,7 +23,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", @@ -40,7 +40,7 @@ fn substituted<'py>( let py: Python<'py> = sql.py(); let sources = InterpolationSources( effective_vars.downcast()?.clone(), - imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + unicode_environment(py)?, if context_values.is_none() { None } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index cd25c9cee6..1ee2b2e061 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -32,7 +32,7 @@ const TYPED_REFERENCE_FUNCTIONS: [(&str, &str); 3] = [ const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; -const DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const BUILTINS_MODULE: &str = "builtins"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index ec680f1ab9..ec8592a9ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_me const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs index 40b2b87e1c..76dcf00193 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const ERROR_SEVERITY: &str = "error"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 304c4c4b9f..8fd76caa2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -19,7 +19,7 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; -const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COLLECTOR_MODULE: &str = "sqlbuild._native"; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; /// `(kind, name, package, call_argument_count)` for one reference. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e16906354e..380ec072f4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,6 +1,6 @@ """Private SQLBuild native engine bindings.""" -from collections.abc import Callable, Iterable, Mapping, Sequence, Set +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set from concurrent.futures import Future from contextlib import AbstractContextManager from contextvars import Token @@ -76,7 +76,6 @@ from sqlbuild.compiler.auditing.types import ( AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAnalysisSelection, @@ -3497,6 +3496,23 @@ def effective_config_settings( settings: tuple[object, object], overrides: frozenset[str], names: list[str], / ) -> dict[str, object]: ... def yaml_entry_line(*, contents: str, name: str) -> int | None: ... + +class CollectedCompileDiagnostics: + def __enter__(self) -> CollectedCompileDiagnostics: ... + def __exit__(self, *exception: object) -> bool: ... + def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + +def collect_compile_diagnostics() -> CollectedCompileDiagnostics: ... + +class CompileInputRecording: + def __enter__(self) -> CompileInputReads: ... + def __exit__(self, *exception: object) -> bool: ... + +def record_compile_input_reads() -> CompileInputRecording: ... +def record_settings_class_read(settings_class: type) -> None: ... +def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: ... def compiler_cache_directory(project_dir: Path, /) -> Path: ... def project_relative_path(*, path: Path, project_dir: Path) -> Path: ... def resolve_target_name( @@ -3959,6 +3975,17 @@ class SqlTestScanStore: def counts(self) -> tuple[int, int]: ... def save(self) -> tuple[int | None, str | None]: ... +class UnicodeEnvironment(Mapping[str, str]): + def __init__(self) -> None: ... + def __getitem__(self, name: str) -> str: ... + def __iter__(self) -> Iterator[str]: ... + def __len__(self) -> int: ... + +class SqlTestScanCache: + def __init__(self, *, cache_dir: Path | None) -> None: ... + def opened(self) -> SqlTestScanStore | None: ... + def save(self) -> None: ... + SQL_TEST_SCAN_STORE_FILE_NAME: str def content_digest(parts: list[str]) -> str: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py deleted file mode 100644 index 9ec47b3b56..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Collect compile diagnostics so one compile reports every violation, not only the first.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token - -from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( - CollectedCompileDiagnostics, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, -) - -_ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( - "sqlbuild_collected_compile_diagnostics", default=None -) - - -@contextmanager -def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: - """Collect diagnostics reported inside the block instead of raising the first one.""" - - collector: CollectedCompileDiagnostics = CollectedCompileDiagnostics() - token: Token[CollectedCompileDiagnostics | None] = _ACTIVE_COLLECTOR.set(collector) - try: - yield collector - finally: - _ACTIVE_COLLECTOR.reset(token) - - -def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: - """Record a diagnostic in the active collector, or raise it when none is active.""" - - collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() - if collector is not None: - collector.add(key=key, diagnostic=diagnostic) - return - location: str = ( - f"\n --> {diagnostic.location.path.as_posix()}:{diagnostic.location.line}" - if diagnostic.location is not None - else f"\n --> {diagnostic.path.as_posix()}" - if diagnostic.path is not None - else "" - ) - raise CompileInputError( - f"{diagnostic.message}{location}", code=diagnostic.code, help=diagnostic.help - ) diff --git a/src/sqlbuild/compiler/compile/classes/__init__.py b/src/sqlbuild/compiler/compile/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py deleted file mode 100644 index 9a1a32e725..0000000000 --- a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Compile diagnostics collected during one compile phase instead of raised.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompilerDiagnostic - - -class CollectedCompileDiagnostics: - """Deduplicated compile diagnostics gathered during one compile phase.""" - - def __init__(self) -> None: - self._by_key: dict[tuple[str, ...], CompilerDiagnostic] = {} - - def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: - """Record one diagnostic; a repeated key keeps the first diagnostic.""" - - self._by_key.setdefault(key, diagnostic) - - @property - def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: - """Return the recorded diagnostics in deterministic key order.""" - - return tuple(self._by_key[key] for key in sorted(self._by_key)) diff --git a/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py b/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py deleted file mode 100644 index 8cb4d1a463..0000000000 --- a/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Process-wide delivery of template input reads to the active compile recorders.""" - -from __future__ import annotations - -import threading -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild._native import CompileInputReads -from sqlbuild.compiler.compile.types import CompileContextKey - - -class CompileInputReadRegistry: - """Deliver template reads from every thread, including ones without copied context.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._recorders: tuple[CompileInputReads, ...] = () - - @contextmanager - def recording(self) -> Iterator[CompileInputReads]: - """Record environment variables and run-identity reads until the block exits.""" - - recorder: CompileInputReads = CompileInputReads() - with self._lock: - self._recorders = (*self._recorders, recorder) - try: - yield recorder - finally: - with self._lock: - self._recorders = tuple(item for item in self._recorders if item is not recorder) - - def environment_read(self, name: str) -> None: - """Note that a template looked up one environment variable, present or not.""" - - for recorder in self._recorders: - recorder.add_environment_name(name) - - def settings_class_read(self, settings_class: type) -> None: - """Note a provider settings class instantiated from the environment and files.""" - - for recorder in self._recorders: - recorder.add_settings_class(settings_class) - - def context_read(self, name: str) -> None: - """Note a CTX lookup; only the per-invocation run identity is volatile.""" - - if name != CompileContextKey.RUN_ID: - return - for recorder in self._recorders: - recorder.mark_run_id_read() diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py deleted file mode 100644 index f13c663dde..0000000000 --- a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py +++ /dev/null @@ -1,44 +0,0 @@ -"""The compile's native SQL-test scan store, opened on first use and saved once.""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.profiling.main._metric import record_compile_metric - - -class SqlTestScanCache: - """The compile cache folder's scan store, keyed by the installed and native code.""" - - def __init__(self, *, cache_dir: Path | None) -> None: - self._cache_dir: Path | None = cache_dir - self._store: _native.SqlTestScanStore | None = None - - def opened(self) -> _native.SqlTestScanStore | None: - """The native store this compile reads and writes, or None when it has none.""" - - if self._cache_dir is None or self._store is not None: - return self._store - error: str | None - self._store, error = _native.SqlTestScanStore.open_in( - str(self._cache_dir), compiled_code_identity() - ) - if error is not None: - logging.getLogger(__name__).debug("SQL test scan store not loaded: %s", error) - self._cache_dir = None - return self._store - - def save(self) -> None: - """Persist results stored by this compile and report hit and miss counts.""" - - hits, misses = (0, 0) if self._store is None else self._store.counts() - record_compile_metric(metric="sql_test_scan_cache_hits", value=hits) - record_compile_metric(metric="sql_test_scan_cache_misses", value=misses) - if self._store is None: - return - _written, error = self._store.save() - if error is not None: - logging.getLogger(__name__).debug("SQL test scan store not saved: %s", error) diff --git a/src/sqlbuild/compiler/compile/classes/unicode_environment.py b/src/sqlbuild/compiler/compile/classes/unicode_environment.py deleted file mode 100644 index 2b8ef3993c..0000000000 --- a/src/sqlbuild/compiler/compile/classes/unicode_environment.py +++ /dev/null @@ -1,37 +0,0 @@ -"""The process environment as compile text reads it, rejecting values that are not UTF-8.""" - -from __future__ import annotations - -import os -from collections.abc import Iterator, Mapping - -from sqlbuild.compiler.compile.exceptions import CompileInputError - - -class UnicodeEnvironment(Mapping[str, str]): - """`os.environ`, raising a clear error for a value holding undecodable bytes.""" - - def __getitem__(self, name: str) -> str: - value: str = os.environ[name] - try: - _ = value.encode("utf-8") - except UnicodeEncodeError as error: - byte_offset: int = len(value[: error.start].encode("utf-8")) - raise CompileInputError( - f"Environment variable '{name}' is not valid UTF-8 text: byte {byte_offset} " - "starts an invalid UTF-8 sequence", - help=( - f"Set '{name}' to UTF-8 text, since SQLBuild cannot send undecodable bytes " - "to a warehouse; the value is not shown because it may be a secret" - ), - ) from None - return value - - def __contains__(self, name: object) -> bool: - return name in os.environ - - def __iter__(self) -> Iterator[str]: - return iter(os.environ) - - def __len__(self) -> int: - return len(os.environ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index dfc9b660ad..50c48ffdaf 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -2,9 +2,6 @@ from __future__ import annotations -from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( - CompileInputReadRegistry, -) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -92,7 +89,6 @@ DEFAULT_SQL_TEST_MODE: SqlTestMode = SqlTestMode.MODEL COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" -COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" RETIRED_REFERENCE_CACHE_DIRECTORY_NAME: str = "references-v3" SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index d219e66130..b0a97c1a9d 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -10,9 +10,9 @@ from typing import Any, overload import sqlbuild._native as _native +from sqlbuild._native import SqlTestScanCache from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 5831d09637..89d26cad35 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -14,8 +14,11 @@ from typing import TYPE_CHECKING, Any, get_type_hints import sqlbuild._native as _native -from sqlbuild._native import project_relative_path, validate_resource_identity -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild._native import ( + project_relative_path, + record_settings_class_read, + validate_resource_identity, +) from sqlbuild.compiler.discovery.constants import ( PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, @@ -213,7 +216,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ) = cached_event_exporter_extensions(project_dir=project_dir) if cached is not None: for cached_provider in cached[0]: - COMPILE_INPUT_READS.settings_class_read(cached_provider.provider_class) + record_settings_class_read(cached_provider.provider_class) return cached[0] providers_root: Path = project_dir / "providers" if not providers_root.is_dir(): @@ -246,7 +249,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, f"{file_path.relative_to(project_dir)}" ) seen_names[provider_name] = file_path - COMPILE_INPUT_READS.settings_class_read(provider_class) + record_settings_class_read(provider_class) discovered_providers.append( DiscoveredProvider( file_path=file_path, diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 4c3a914582..8b1e1cb324 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -18,9 +18,9 @@ build_compile_inputs, discover_project_inputs, inputs_with, + record_compile_input_reads, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import ( CompileAdapterContext, ) @@ -168,7 +168,7 @@ def recorded_assembly( ) -> tuple[object, tuple[tuple[str, ...], bool]]: """Assemble `inputs`; return its assembly view and the input reads recorded.""" - with COMPILE_INPUT_READS.recording() as reads: + with record_compile_input_reads() as reads: project: CompiledProjectView = assemble_with(inputs=inputs, monkeypatch=monkeypatch) return assembly_view(project), (reads.environment_names, reads.read_run_id) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index bf30c933fc..2c312977cd 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -12,6 +12,7 @@ build_manifest, build_native_declaration_contexts, build_scope_lookup, + collect_compile_diagnostics, discover_project_inputs, expand_sql_macros, infer_native_query_columns, @@ -26,9 +27,6 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - collect_compile_diagnostics, -) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.models import ( CompileAdapterContext, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 64bb9abacf..fcb2625574 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -6,13 +6,12 @@ import pytest -from sqlbuild._native import extract_sql_references +from sqlbuild._native import collect_compile_diagnostics, extract_sql_references from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index b418fe8604..8830420024 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -3,10 +3,10 @@ import pytest from sqlbuild._native import ( + UnicodeEnvironment, expand_effective_vars, expand_template_data, ) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandTemplateDataErrorTestCase, From 66badd5071fc0dc947e3569597c3531a6959a6f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:12:33 +0100 Subject: [PATCH 21/33] refactor: build analysis cache contexts and explicit-reference owners natively Compute the analysis cache context digests and record the analysis cache metrics in the native compile driver, and collect the task, asset, check, loader and hook functions the explicit-reference checks read in the native report entry. Delete the Python helpers they replace. --- .../_helpers/analysis/explicit_references.rs | 298 ++++++++++++++---- .../analysis_session/model_analyses.rs | 12 +- .../_helpers/compile_driver/analysis_cache.rs | 238 ++++++++++++++ .../_helpers/compile_driver/assembly.rs | 3 +- .../compile_driver/assembly_analysis.rs | 19 +- .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/registration.rs | 7 +- .../_helpers/planner_identity/json_values.rs | 13 + src/sqlbuild/_native.pyi | 20 +- .../compile/_helpers/analysis/cache.py | 122 ------- .../_helpers/explicit_references/checks.py | 150 --------- .../compiler/analysis_session/helpers.py | 2 +- 12 files changed, 525 insertions(+), 360 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/cache.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 1436366c08..b7353adbfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -3,8 +3,11 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::exceptions::{PyOSError, PyTypeError}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; @@ -15,12 +18,14 @@ use sqlbuild_analysis::explicit_references::models::{ ReferenceReport, ReferenceRequest, SqlOwner, }; use sqlbuild_analysis::explicit_references::types::ResourceKey; +use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; -use crate::bindings::_helpers::analysis::reference_facts::{ProjectFacts, project_facts, ref_keys}; +use crate::bindings::_helpers::analysis::reference_facts::{ProjectFacts, project_facts}; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; @@ -29,48 +34,106 @@ const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const LOADER_DEFINITION_MODULE: &str = "sqlbuild.python_nodes.main.read_loader_definition"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const BUILTINS_MODULE: &str = "builtins"; +const VALUE_PLACEHOLDER: &str = "__sqlbuild_python_value_"; +const HARD_CODED_ALLOWED: &str = "hard-coded relation names"; +const NODE_OWNERS: [(&str, &str); 3] = [ + ("task", "task_functions"), + ("asset", "asset_functions"), + ("check", "check_functions"), +]; +const NODE_KIND: &str = "node"; +const HOOK_KIND: &str = "hook"; +const LOADER_KIND: &str = "loader"; +const MODEL_REF_KIND: &str = "model"; +const SOURCE_REF_KIND: &str = "source"; const DIAGNOSTIC_PHASE: &str = "compile"; const DIAGNOSTIC_SEVERITY: &str = "error"; -/// One Python SQL owner as Python collects it: the owner, its path order and literal SQL calls. -type OwnerInput<'py> = (Bound<'py, PyAny>, usize, Vec<(String, String, usize)>); +/// One Python function whose literal SQL the checks read, and the references it owns. +struct PythonSqlOwner<'py> { + label: String, + kind: &'static str, + function: Bound<'py, PyAny>, + relative_path: Bound<'py, PyAny>, + own_refs: Vec, + upstream_loader_by_source: Vec<(String, String)>, +} -/// The project-wide settings of one explicit-reference check. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct ReportSettings { - /// Whether the project keeps old-name views unless a model disables them. - retains_old_names: bool, - enforce_explicit: bool, - explicit_help: String, - /// The text Python substitutes for f-string values in literal SQL. - placeholder: String, +impl<'py> PythonSqlOwner<'py> { + fn new(label: String, kind: &'static str, node: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { + label, + kind, + function: node.getattr("function")?, + relative_path: node.getattr("relative_path")?, + own_refs: Vec::new(), + upstream_loader_by_source: Vec::new(), + }) + } } -/// Python's audit-gate, hook-read and Python-SQL checks as a `PythonSqlReferenceReport`. +/// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. #[pyfunction] +#[pyo3(signature = (*, project, discovered_inputs, enforce_explicit))] fn explicit_reference_report<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, - settings: ReportSettings, - owners: Vec>, + discovered_inputs: &Bound<'py, PyAny>, + enforce_explicit: bool, ) -> PyResult> { - let facts: ProjectFacts = project_facts(project, settings.retains_old_names)?; + let owners: Vec> = if enforce_explicit { + python_sql_owners(project, discovered_inputs)? + } else { + Vec::new() + }; + let path_order: Bound<'py, PyDict> = PyDict::new(py); + let unique_paths: Bound<'py, PySet> = + PySet::new(py, owners.iter().map(|owner| owner.relative_path.clone()))?; + for (index, path) in imported(py, BUILTINS_MODULE, "sorted")? + .call1((unique_paths,))? + .try_iter()? + .enumerate() + { + path_order.set_item(path?, index)?; + } + let retains_old_names: bool = !discovered_inputs + .getattr("project_config")? + .getattr("migrations")? + .getattr("old_name_views")? + .is_none(); + let facts: ProjectFacts = project_facts(project, retains_old_names)?; + let mut sql_owners: Vec = Vec::with_capacity(owners.len()); + for owner in &owners { + let order: usize = path_order + .as_any() + .get_item(&owner.relative_path)? + .extract()?; + sql_owners.push(sql_owner( + owner, + order, + literal_sql_calls_of(&owner.function)?, + )); + } let request = ReferenceRequest { - enforce_explicit: settings.enforce_explicit, + enforce_explicit, dialect: project.getattr("sql_analysis_dialect")?.extract()?, - explicit_help: settings.explicit_help, - placeholder: settings.placeholder, + explicit_help: if enforce_explicit { + references_help(HARD_CODED_ALLOWED) + } else { + String::new() + }, + placeholder: VALUE_PLACEHOLDER.to_owned(), resources: facts.resources, models: facts.models, sources: facts.sources, seeds: facts.seeds, audits: facts.audits, hooks: facts.hooks, - owners: owners - .iter() - .map(|(owner, path_order, literals)| sql_owner(owner, *path_order, literals)) - .collect::>()?, + owners: sql_owners, }; match py .compiler_detach(|| Ok(check_explicit_references(&request))) @@ -90,39 +153,168 @@ fn explicit_reference_report<'py>( } } +/// Every task, asset, check, loader and hook function, as Python discovery lists them. +fn python_sql_owners<'py>( + project: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = project.py(); + let mut owners: Vec> = Vec::new(); + for (label, attribute) in NODE_OWNERS { + for node in discovered_inputs.getattr(attribute)?.try_iter()? { + let node: Bound<'py, PyAny> = node?; + let name: String = node.getattr("name")?.extract()?; + owners.push(PythonSqlOwner::new( + format!("{label}:{name}"), + NODE_KIND, + &node, + )?); + } + } + owners.extend(loader_owners(project, discovered_inputs)?); + let hook_names = imported(py, NATIVE_MODULE, "model_python_hook_names")?; + let mut attached: BTreeMap> = BTreeMap::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("model", &model)?; + let model_name: String = model.getattr("name")?.extract()?; + for hook_name in hook_names.call((), Some(&keywords))?.try_iter()? { + attached + .entry(hook_name?.extract()?) + .or_default() + .insert(model_name.clone()); + } + } + for hook in project.getattr("hook_functions")?.try_iter()? { + let hook: Bound<'py, PyAny> = hook?; + let name: String = hook.getattr("name")?.extract()?; + let mut owner = PythonSqlOwner::new(format!("hook:{name}"), HOOK_KIND, &hook)?; + owner.own_refs = attached + .get(&name) + .into_iter() + .flatten() + .map(|model| (MODEL_REF_KIND.to_owned(), model.clone())) + .collect(); + owners.push(owner); + } + Ok(owners) +} + +/// Each loader function, owning its sources and naming the loaders it depends on. +fn loader_owners<'py>( + project: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = project.py(); + let mut sources_by_loader: BTreeMap> = BTreeMap::new(); + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'py, PyAny> = source?; + let loader: Option = source + .getattr("source_entry")? + .getattr("loader")? + .extract()?; + if let Some(loader) = loader { + sources_by_loader + .entry(loader) + .or_default() + .insert(source.getattr("name")?.extract()?); + } + } + let read_definition = imported(py, LOADER_DEFINITION_MODULE, "read_loader_definition")?; + let mut owners: Vec> = Vec::new(); + for loader in discovered_inputs.getattr("loader_functions")?.try_iter()? { + let loader: Bound<'py, PyAny> = loader?; + let name: String = loader.getattr("name")?.extract()?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for dependency in loader.getattr("depends_on")?.try_iter()? { + let dependency: Bound<'py, PyAny> = dependency?; + if !dependency.is_callable() { + continue; + } + let definition: Bound<'py, PyAny> = read_definition.call1((&dependency,))?; + if definition.is_none() { + continue; + } + let definition_name: String = definition.getattr("name")?.extract()?; + let shown: String = match dependency.getattr_opt("__name__")? { + Some(shown) => shown.extract()?, + None => definition_name.clone(), + }; + for source_name in sources_by_loader + .get(&definition_name) + .into_iter() + .flatten() + { + upstream.retain(|(known, _)| known != source_name); + upstream.push((source_name.clone(), shown.clone())); + } + } + let mut owner = PythonSqlOwner::new(format!("loader:{name}"), LOADER_KIND, &loader)?; + owner.own_refs = sources_by_loader + .get(&name) + .into_iter() + .flatten() + .map(|source| (SOURCE_REF_KIND.to_owned(), source.clone())) + .collect(); + owner.upstream_loader_by_source = upstream; + owners.push(owner); + } + Ok(owners) +} + +/// The literal SQL calls of a function's dedented source, or none when its source is unreadable. +fn literal_sql_calls_of(function: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = function.py(); + let (lines, start_line): (Bound<'_, PyAny>, usize) = + match imported(py, "inspect", "getsourcelines")?.call1((function,)) { + Ok(found) => found.extract()?, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + return Ok(Vec::new()); + } + Err(error) => return Err(error), + }; + let joined: Bound<'_, PyAny> = PyString::new(py, "").call_method1("join", (lines,))?; + let source: String = imported(py, "textwrap", "dedent")? + .call1((joined,))? + .extract()?; + Ok(python_literal_sql_calls( + py, + &source, + start_line, + VALUE_PLACEHOLDER, + )) +} + fn sql_owner( - owner: &Bound<'_, PyAny>, + owner: &PythonSqlOwner<'_>, path_order: usize, - literals: &[(String, String, usize)], -) -> PyResult { - Ok(SqlOwner { - label: owner.getattr("label")?.extract()?, - kind: owner.getattr("kind")?.str()?.extract()?, + literals: Vec<(String, String, usize)>, +) -> SqlOwner { + SqlOwner { + label: owner.label.clone(), + kind: owner.kind.to_owned(), path_order, - own_refs: ref_keys(&owner.getattr("own_refs")?)?, - upstream_loaders: owner - .getattr("upstream_loader_by_source")? - .call_method0("items")? - .try_iter()? - .map(|item| item?.extract()) - .collect::>()?, + own_refs: owner.own_refs.clone(), + upstream_loaders: owner.upstream_loader_by_source.clone(), literals: literals - .iter() - .map(|(method, sql, line)| LiteralSql { - method: method.clone(), - sql: sql.clone(), - line: *line, - }) + .into_iter() + .map(|(method, sql, line)| LiteralSql { method, sql, line }) .collect(), - }) + } } /// The report's diagnostics and unmatched relations as Python's compile models. fn python_report<'py>( py: Python<'py>, report: &ReferenceReport, - owners: &[OwnerInput<'py>], + owners: &[PythonSqlOwner<'py>], ) -> PyResult> { + let owner_kind: Bound<'py, PyAny> = + imported(py, REFERENCE_TYPES_MODULE, "HardCodedRelationOwnerKind")?; let compile_models = py.import(COMPILE_MODELS_MODULE)?; let reference_models = py.import(REFERENCE_MODELS_MODULE)?; let source_location: Bound<'py, PyAny> = py @@ -146,7 +338,7 @@ fn python_report<'py>( match row.owner { Some(owner) => { let location: Bound<'py, PyDict> = PyDict::new(py); - location.set_item("path", owners[owner].0.getattr("relative_path")?)?; + location.set_item("path", &owners[owner].relative_path)?; location.set_item("line", row.line)?; location.set_item("column", 1)?; fields.set_item("location", source_location.call((), Some(&location))?)?; @@ -160,15 +352,15 @@ fn python_report<'py>( let relation_name: Bound<'py, PyAny> = reference_models.getattr("RelationName")?; let mut unmatched: Vec> = Vec::with_capacity(report.unmatched.len()); for row in &report.unmatched { - let owner: &Bound<'py, PyAny> = &owners[row.owner].0; + let owner: &PythonSqlOwner<'py> = &owners[row.owner]; let relation: Bound<'py, PyDict> = PyDict::new(py); relation.set_item("name", &row.relation.name)?; relation.set_item("schema", &row.relation.schema)?; relation.set_item("database", &row.relation.database)?; let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("owner_label", owner.getattr("label")?)?; - fields.set_item("owner_kind", owner.getattr("kind")?)?; - fields.set_item("relative_path", owner.getattr("relative_path")?)?; + fields.set_item("owner_label", &owner.label)?; + fields.set_item("owner_kind", owner_kind.call1((owner.kind,))?)?; + fields.set_item("relative_path", &owner.relative_path)?; fields.set_item("line", row.line)?; fields.set_item("method", &row.method)?; fields.set_item("relation", relation_name.call((), Some(&relation))?)?; @@ -183,7 +375,6 @@ fn python_report<'py>( } /// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. -#[pyfunction] fn python_literal_sql_calls( py: Python<'_>, source: &str, @@ -278,7 +469,6 @@ fn hook_row(edge: HookEdge) -> HookRow { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; - module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 7dea30fd8a..911eff0bb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -11,10 +11,10 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; @@ -260,15 +260,7 @@ fn record_cache(python: &[Bound<'_, PyAny>], session: &Bound<'_, PyAny>) -> PyRe /// Record the analysis cache entries a native session read, missed and bypassed. fn record_metrics(py: Python<'_>, (hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { - let arguments: Bound<'_, PyDict> = PyDict::new(py); - arguments.set_item("batch_hits", 0)?; - arguments.set_item("entry_hits", hits)?; - arguments.set_item("misses", misses)?; - arguments.set_item("bypasses", bypasses)?; - py.import(ANALYSIS_CACHE_MODULE)? - .getattr("record_analysis_cache_metrics")? - .call((), Some(&arguments))?; - Ok(()) + record_analysis_cache_metrics(py, (0, hits, misses, bypasses)) } /// Each model's native request row, in analysis order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs new file mode 100644 index 0000000000..28137f8425 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -0,0 +1,238 @@ +//! The analysis cache context and the model analysis cache metrics. + +use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PySlice, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, unicode_sorted_dumps}; + +const ANALYSIS_CACHE_VERSION: i64 = 13; +const ANALYSIS_ALGORITHM_FINGERPRINT: &str = "model-sql-analysis-v21-completed-star-shapes"; +const LOCAL_QUALNAME_MARKER: &str = ""; +const PACKAGE_NAME: &str = "sqlbuild"; +const UNKNOWN_VERSION: &str = "unknown"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +const METRIC_MODULE: &str = "sqlbuild.compiler.profiling.main._metric"; +const METADATA_MODULE: &str = "importlib.metadata"; +const CONTEXT_REQUIRED: [&str; 3] = ["root", "inference_profile", "allow_compact_analysis"]; +const CONTEXT_OPTIONAL: [&str; 2] = ["rich_type_inference", "signature_namespace"]; + +/// A reusable cache context, or `None` when the profile identity is unstable. +pub(crate) fn analysis_cache_context<'py>( + root: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + (allow_compact_analysis, rich_type_inference): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + signature_namespace: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = root.py(); + if root.is_none() { + return Ok(None); + } + let Some(profile) = inference_profile_payload(inference_profile)? else { + return Ok(None); + }; + let shared: Bound<'py, PyDict> = PyDict::new(py); + shared.set_item("algorithm", ANALYSIS_ALGORITHM_FINGERPRINT)?; + shared.set_item("cache_version", ANALYSIS_CACHE_VERSION)?; + shared.set_item("sqlbuild_version", package_version(py, "sqlbuild")?)?; + shared.set_item( + "polyglot_version", + package_version(py, "polyglot-sql-chio")?, + )?; + shared.set_item( + "python_version", + imported(py, "platform", "python_version_tuple")? + .call0()? + .get_item(PySlice::new(py, 0, 2, 1))?, + )?; + shared.set_item("allow_compact_analysis", allow_compact_analysis)?; + shared.set_item("rich_type_inference", rich_type_inference)?; + shared.set_item( + "type_checked_dialects", + imported(py, "builtins", "sorted")?.call1((imported( + py, + SQL_ANALYSIS_CONSTANTS_MODULE, + "TYPE_CHECKED_DIALECTS", + )?,))?, + )?; + shared.set_item("inference_profile", profile)?; + let digests = (|| -> PyResult<(String, String)> { + Ok(( + payload_digest(shared.as_any())?, + payload_digest(signature_namespace)?, + )) + })(); + let (shared_fingerprint, namespace) = match digests { + Ok(digests) => digests, + Err(error) if json_failure(py, &error) => return Ok(None), + Err(error) => return Err(error), + }; + let context = call_with( + &imported(py, COMPILE_MODELS_MODULE, "AnalysisCacheContext")?, + &[ + ("root", root.clone()), + ( + "shared_fingerprint", + PyString::new(py, &shared_fingerprint).into_any(), + ), + ( + "signature_namespace", + PyString::new(py, &namespace).into_any(), + ), + ], + ); + match context { + Ok(context) => Ok(Some(context)), + Err(error) if json_failure(py, &error) => Ok(None), + Err(error) => Err(error), + } +} + +/// Build a reusable cache context, or bypass when profile identity is unstable. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_analysis_cache_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult>> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_analysis_cache_context", + options, + (&CONTEXT_REQUIRED, &CONTEXT_OPTIONAL), + )?; + let rich: Bound<'py, PyAny> = if arguments.passed("rich_type_inference")? { + arguments.value("rich_type_inference")? + } else { + PyBool::new(py, true).to_owned().into_any() + }; + analysis_cache_context( + &arguments.value("root")?, + &arguments.value("inference_profile")?, + (&arguments.value("allow_compact_analysis")?, &rich), + &arguments.value("signature_namespace")?, + ) +} + +/// Record semantic analysis cache outcomes for one invocation. +pub(crate) fn record_analysis_cache_metrics( + py: Python<'_>, + (batch_hits, entry_hits, misses, bypasses): (usize, usize, usize, usize), +) -> PyResult<()> { + let record: Bound<'_, PyAny> = imported(py, METRIC_MODULE, "record_compile_metric")?; + for (metric, value) in [ + ("analysis_batch_cache_hits", batch_hits), + ("analysis_entry_cache_hits", entry_hits), + ("analysis_cache_misses", misses), + ("analysis_cache_bypasses", bypasses), + ] { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("metric", metric)?; + keywords.set_item("value", value)?; + record.call((), Some(&keywords))?; + } + Ok(()) +} + +/// Whether `error` is the `TypeError` or `ValueError` an unstable payload raises. +fn json_failure(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) || error.is_instance_of::(py) +} + +/// The profile identity payload, or `None` when a nullability rule is not a stable module function. +fn inference_profile_payload<'py>( + profile: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = profile.py(); + let inspect = py.import("inspect")?; + let sorted: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?; + let rules: Bound<'py, PyList> = PyList::empty(py); + let items: Bound<'py, PyAny> = sorted.call1((profile + .getattr("function_nullability_rules")? + .call_method0("items")?,))?; + for item in items.try_iter()? { + let (name, rule): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !inspect.call_method1("isfunction", (&rule,))?.is_truthy()? { + return Ok(None); + } + let module: Bound<'py, PyAny> = rule.getattr("__module__")?; + let qualname: Bound<'py, PyAny> = rule.getattr("__qualname__")?; + let stable: bool = module + .extract::()? + .strip_prefix(PACKAGE_NAME) + .is_some_and(|rest| rest.starts_with('.')) + && !qualname + .extract::()? + .contains(LOCAL_QUALNAME_MARKER) + && rule.getattr("__closure__")?.is_none() + && rule.getattr("__defaults__")?.is_none() + && rule.getattr("__kwdefaults__")?.is_none(); + if !stable { + return Ok(None); + } + let source: String = match inspect.call_method1("getsource", (&rule,)) { + Ok(source) => source.extract()?, + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + return Ok(None); + } + Err(error) => return Err(error), + }; + let entry: Bound<'py, PyDict> = PyDict::new(py); + entry.set_item("name", name)?; + entry.set_item("module", module)?; + entry.set_item("qualname", qualname)?; + entry.set_item("source_digest", sha256_hex(source.as_bytes()))?; + rules.append(entry)?; + } + let payload: Bound<'py, PyDict> = PyDict::new(py); + for field in [ + "sql_analysis_dialect", + "quoted_identifiers_ignore_case", + "semantic_known_functions", + "semantic_known_types", + ] { + payload.set_item(field, profile.getattr(field)?)?; + } + payload.set_item("function_nullability_rules", rules)?; + payload.set_item( + "function_return_types", + imported(py, "builtins", "dict")?.call1((sorted.call1((profile + .getattr("function_return_types")? + .call_method0("items")?,))?,))?, + )?; + Ok(Some(payload)) +} + +/// The SHA-256 of `json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True)`. +fn payload_digest(payload: &Bound<'_, PyAny>) -> PyResult { + let encoded: String = unicode_sorted_dumps(payload, COMPACT)?; + Ok(sha256_hex(encoded.as_bytes())) +} + +/// `importlib.metadata.version(package)`, or `unknown` when the package is not installed. +fn package_version(py: Python<'_>, package: &str) -> PyResult { + match imported(py, METADATA_MODULE, "version")?.call1((package,)) { + Ok(version) => version.extract(), + Err(error) + if error.is_instance(py, &imported(py, METADATA_MODULE, "PackageNotFoundError")?) => + { + Ok(UNKNOWN_VERSION.to_owned()) + } + Err(error) => Err(error), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_analysis_cache_context, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 69309e0a44..e21a80bc05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -13,8 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const RECORD_ANALYSES_MODULE: &str = "sqlbuild._native"; -const EXPLICIT_REFERENCES_MODULE: &str = - "sqlbuild.compiler.compile._helpers.explicit_references.checks"; +const EXPLICIT_REFERENCES_MODULE: &str = "sqlbuild._native"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const RUN_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.classes.macro_bridge"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index d88b9f44d6..8114cefb8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -6,6 +6,7 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple} use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; +use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; use crate::bindings::_helpers::compile_driver::python_calls::{ call_with, imported, keywords, within, @@ -13,7 +14,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const PROFILING_CPU_MODULE: &str = "sqlbuild.compiler.profiling.main._record_cpu"; @@ -411,14 +411,11 @@ fn analysis_cache<'py>( ("vars", inputs.getattr("effective_vars")?), ], )?; - call_with( - &imported(py, ANALYSIS_CACHE_MODULE, "build_analysis_cache_context")?, - &[ - ("root", request.analysis_cache_dir.bind(py).clone()), - ("inference_profile", context.profile.clone()), - ("allow_compact_analysis", allow_compact.clone()), - ("rich_type_inference", rich.clone()), - ("signature_namespace", namespace.into_any()), - ], - ) + Ok(analysis_cache_context( + request.analysis_cache_dir.bind(py), + &context.profile, + (allow_compact, rich), + namespace.as_any(), + )? + .unwrap_or_else(|| py.None().into_bound(py))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index d7352be8ec..4d694a8f7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod analysis; +pub(crate) mod analysis_cache; pub(crate) mod assembly; pub(crate) mod assembly_analysis; pub(crate) mod assembly_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 9a3810ab61..0efbc1611b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ - analysis, assembly, command, deferral, diagnostic_collector, diff, discovery, input_reads, - inputs, pipeline, pipeline_entries, project, python_plan_entries, relation_targets, - selection_phases, targets, + analysis, analysis_cache, assembly, command, deferral, diagnostic_collector, diff, discovery, + input_reads, inputs, pipeline, pipeline_entries, project, python_plan_entries, + relation_targets, selection_phases, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::register(module)?; + analysis_cache::register(module)?; assembly::register(module)?; command::register(module)?; project::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs index 382d078218..ee7f791b5d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs @@ -31,6 +31,19 @@ pub(crate) fn sorted_dumps( dumps(&encoded, &dialect).map_err(|error| PyValueError::new_err(format!("{error:?}"))) } +/// `json.dumps(value, ensure_ascii=False, sort_keys=True, separators=separators)`. +pub(crate) fn unicode_sorted_dumps( + value: &Bound<'_, PyAny>, + separators: (&str, &str), +) -> PyResult { + let encoded: JsonValue = Encoder::new(value.py(), false)?.encode(value)?; + let mut options: StdlibJsonOptions = + StdlibJsonOptions::new(None).with_separators(separators.0, separators.1); + options.ensure_ascii = false; + dumps(&encoded, &JsonDialect::Stdlib(options)) + .map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + /// The C encoder's type dispatch, with dictionaries ordered as `sorted(dict.items())`. struct Encoder<'py> { default_str: bool, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 380ec072f4..20672b333b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -77,6 +77,7 @@ from sqlbuild.compiler.auditing.types import ( AuditSeverity, ) from sqlbuild.compiler.compile.models import ( + AnalysisCacheContext, CompileAdapterContext, CompileAnalysisSelection, CompileAuditInput, @@ -3505,6 +3506,14 @@ class CollectedCompileDiagnostics: def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... def collect_compile_diagnostics() -> CollectedCompileDiagnostics: ... +def build_analysis_cache_context( + *, + root: Path | None, + inference_profile: ExpressionInferenceProfile, + allow_compact_analysis: bool, + rich_type_inference: bool = True, + signature_namespace: object = None, +) -> AnalysisCacheContext | None: ... class CompileInputRecording: def __enter__(self) -> CompileInputReads: ... @@ -4606,10 +4615,10 @@ def assemble_native_project_resources( syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], ) -> NativeProjectResources: ... def explicit_reference_report( - project: Any, - settings: dict[str, object], - owners: list[tuple[Any, int, list[tuple[str, str, int]]]], - /, + *, + project: CompiledProjectView, + discovered_inputs: DiscoveredProjectInputs, + enforce_explicit: bool, ) -> PythonSqlReferenceReport: ... def hard_coded_relation_remedy( *, @@ -4715,9 +4724,6 @@ def schema_validations( *, requests: Sequence[SqlSchemaValidationRequest] ) -> tuple[SqlBindingResult, ...]: ... def ignores_quoted_case(*, connection: Mapping[str, object], dialect: str | None) -> bool: ... -def python_literal_sql_calls( - source: str, start_line: int, placeholder: str -) -> list[tuple[str, str, int]]: ... def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( data: bytes, byte_offsets: list[int] diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py b/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py deleted file mode 100644 index 2ae479e965..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py +++ /dev/null @@ -1,122 +0,0 @@ -"""The analysis cache context and the model analysis cache metrics.""" - -from __future__ import annotations - -import hashlib -import inspect -import json -import platform -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import AnalysisCacheContext -from sqlbuild.compiler.profiling.main._metric import record_compile_metric -from sqlbuild.compiler.sql_analysis.constants import TYPE_CHECKED_DIALECTS - -_ANALYSIS_CACHE_VERSION: int = 13 -_ANALYSIS_ALGORITHM_FINGERPRINT: str = "model-sql-analysis-v21-completed-star-shapes" -_LOCAL_QUALNAME_MARKER: str = "" - - -def build_analysis_cache_context( - *, - root: Path | None, - inference_profile: ExpressionInferenceProfile, - allow_compact_analysis: bool, - rich_type_inference: bool = True, - signature_namespace: object = None, -) -> AnalysisCacheContext | None: - """Build a reusable cache context, or bypass when profile identity is unstable.""" - - if root is None: - return None - profile_payload: dict[str, object] | None = _inference_profile_payload(inference_profile) - if profile_payload is None: - return None - shared_payload: dict[str, object] = { - "algorithm": _ANALYSIS_ALGORITHM_FINGERPRINT, - "cache_version": _ANALYSIS_CACHE_VERSION, - "sqlbuild_version": _package_version("sqlbuild"), - "polyglot_version": _package_version("polyglot-sql-chio"), - "python_version": platform.python_version_tuple()[:2], - "allow_compact_analysis": allow_compact_analysis, - "rich_type_inference": rich_type_inference, - "type_checked_dialects": sorted(TYPE_CHECKED_DIALECTS), - "inference_profile": profile_payload, - } - try: - return AnalysisCacheContext( - root=root, - shared_fingerprint=_payload_digest(shared_payload), - signature_namespace=_payload_digest(signature_namespace), - ) - except (TypeError, ValueError): - return None - - -def record_analysis_cache_metrics( - *, batch_hits: int, entry_hits: int, misses: int, bypasses: int -) -> None: - """Record semantic analysis cache outcomes for one invocation.""" - - record_compile_metric(metric="analysis_batch_cache_hits", value=batch_hits) - record_compile_metric(metric="analysis_entry_cache_hits", value=entry_hits) - record_compile_metric(metric="analysis_cache_misses", value=misses) - record_compile_metric(metric="analysis_cache_bypasses", value=bypasses) - - -def _inference_profile_payload( - profile: ExpressionInferenceProfile, -) -> dict[str, object] | None: - rules: list[dict[str, str]] = [] - for name, rule in sorted(profile.function_nullability_rules.items()): - if not inspect.isfunction(rule): - return None - module: str = rule.__module__ - qualname: str = rule.__qualname__ - if ( - not module.startswith("sqlbuild.") - or _LOCAL_QUALNAME_MARKER in qualname - or rule.__closure__ is not None - or rule.__defaults__ is not None - or rule.__kwdefaults__ is not None - ): - return None - try: - source: str = inspect.getsource(rule) - except (OSError, TypeError): - return None - rules.append( - { - "name": name, - "module": module, - "qualname": qualname, - "source_digest": hashlib.sha256(source.encode("utf-8")).hexdigest(), - } - ) - return { - "sql_analysis_dialect": profile.sql_analysis_dialect, - "quoted_identifiers_ignore_case": profile.quoted_identifiers_ignore_case, - "semantic_known_functions": profile.semantic_known_functions, - "semantic_known_types": profile.semantic_known_types, - "function_nullability_rules": rules, - "function_return_types": dict(sorted(profile.function_return_types.items())), - } - - -def _payload_digest(payload: object) -> str: - encoded: bytes = json.dumps( - payload, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - return hashlib.sha256(encoded).hexdigest() - - -def _package_version(package: str) -> str: - try: - return version(package) - except PackageNotFoundError: - return "unknown" diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py deleted file mode 100644 index 6859c703c8..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Compile's explicit-reference checks: Python collects user-code owners, native decides.""" - -from __future__ import annotations - -import inspect -import textwrap -from collections.abc import Callable, Mapping -from dataclasses import dataclass, field -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import explicit_references_help, model_python_hook_names -from sqlbuild.compiler.compile.models import PythonSqlReferenceReport -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind -from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition -from sqlbuild.python_nodes.models import LoaderDefinition, SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - -_VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_" - - -@dataclass(frozen=True) -class _PythonSqlOwner: - label: str - kind: HardCodedRelationOwnerKind - function: Callable[..., object] - relative_path: Path - own_refs: frozenset[SqlResourceRef] = frozenset() - upstream_loader_by_source: Mapping[str, str] = field(default_factory=dict) - - -def explicit_reference_report( - *, - project: CompiledProjectView, - discovered_inputs: DiscoveredProjectInputs, - enforce_explicit: bool, -) -> PythonSqlReferenceReport: - """Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001.""" - - owners: tuple[_PythonSqlOwner, ...] = ( - _owners(project=project, discovered_inputs=discovered_inputs) if enforce_explicit else () - ) - paths: list[Path] = sorted({owner.relative_path for owner in owners}) - path_order: dict[Path, int] = {path: index for index, path in enumerate(paths)} - return _native.explicit_reference_report( - project, - { - "retains_old_names": discovered_inputs.project_config.migrations.old_name_views - is not None, - "enforce_explicit": enforce_explicit, - "explicit_help": ( - explicit_references_help(allowed="hard-coded relation names") - if enforce_explicit - else "" - ), - "placeholder": _VALUE_PLACEHOLDER, - }, - [ - (owner, path_order[owner.relative_path], _literal_sql_calls(owner.function)) - for owner in owners - ], - ) - - -def _owners( - *, project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs -) -> tuple[_PythonSqlOwner, ...]: - owners: list[_PythonSqlOwner] = [] - for label, nodes in ( - ("task", discovered_inputs.task_functions), - ("asset", discovered_inputs.asset_functions), - ("check", discovered_inputs.check_functions), - ): - owners.extend( - _PythonSqlOwner( - label=f"{label}:{node.name}", - kind=HardCodedRelationOwnerKind.NODE, - function=node.function, - relative_path=node.relative_path, - ) - for node in nodes - ) - owners.extend(_loader_owners(project=project, discovered_inputs=discovered_inputs)) - attached_models: dict[str, set[SqlResourceRef]] = {} - for model in project.models: - for hook_name in model_python_hook_names(model=model): - attached_models.setdefault(hook_name, set()).add( - SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=model.name) - ) - owners.extend( - _PythonSqlOwner( - label=f"hook:{hook.name}", - kind=HardCodedRelationOwnerKind.HOOK, - function=hook.function, - relative_path=hook.relative_path, - own_refs=frozenset(attached_models.get(hook.name, ())), - ) - for hook in project.hook_functions - ) - return tuple(owners) - - -def _loader_owners( - *, project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs -) -> tuple[_PythonSqlOwner, ...]: - sources_by_loader: dict[str, set[str]] = {} - for source in project.sources: - if source.source_entry.loader is not None: - sources_by_loader.setdefault(source.source_entry.loader, set()).add(source.name) - owners: list[_PythonSqlOwner] = [] - for loader in discovered_inputs.loader_functions: - upstream_loader_by_source: dict[str, str] = {} - for dependency in loader.depends_on: - definition: LoaderDefinition | None = ( - read_loader_definition(dependency) if callable(dependency) else None - ) - if definition is None: - continue - for source_name in sources_by_loader.get(definition.name, ()): - upstream_loader_by_source[source_name] = getattr( - dependency, "__name__", definition.name - ) - owners.append( - _PythonSqlOwner( - label=f"loader:{loader.name}", - kind=HardCodedRelationOwnerKind.LOADER, - function=loader.function, - relative_path=loader.relative_path, - own_refs=frozenset( - SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=source_name) - for source_name in sources_by_loader.get(loader.name, ()) - ), - upstream_loader_by_source=upstream_loader_by_source, - ) - ) - return tuple(owners) - - -def _literal_sql_calls(function: Callable[..., object]) -> list[tuple[str, str, int]]: - try: - lines: list[str] - start_line: int - lines, start_line = inspect.getsourcelines(function) - except (OSError, TypeError): - return [] - return _native.python_literal_sql_calls( - textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER - ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index cc5bc10b8b..9ad5adf381 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -16,6 +16,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( analyze_native_model_sql, + build_analysis_cache_context, build_compile_inputs, cursor_intrinsics_analysis_sql, discover_project_inputs, @@ -36,7 +37,6 @@ NativePivotTables, ) from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.compile._helpers.analysis.cache import build_analysis_cache_context from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, From aa279b343fc0dcb0e0ba3765fa52d94b9eb84b1c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:23:44 +0100 Subject: [PATCH 22/33] refactor: plan pipelines, open connections and write contracts natively Move the pipeline planning orchestration, hooked connection opening and contract adoption repository writes into the native compile driver and contract command, calling the planner, materialization loader and Python node selection at their existing Python boundaries. Delete the Python modules they replace. --- .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/pipeline.rs | 34 +- .../_helpers/compile_driver/plan_result.rs | 292 ++++++++++++++++++ .../_helpers/compile_driver/registration.rs | 3 +- .../_helpers/contract_command/command.rs | 22 +- .../bindings/_helpers/contract_command/mod.rs | 1 + .../_helpers/contract_command/write.rs | 195 ++++++++++++ .../_helpers/dbt_interop/plan_output.rs | 14 +- .../bindings/_helpers/seed_command/command.rs | 12 +- fensu.toml | 15 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 3 + .../contract_adoption/main/__init__.py | 1 - .../compiler/contract_adoption/main/_write.py | 103 ------ .../compiler/pipeline/main/__init__.py | 1 - .../compiler/pipeline/main/_plan_result.py | 141 --------- .../runtime/contracts/main/__init__.py | 1 - .../runtime/contracts/main/open_connection.py | 31 -- .../main/test_compile_phase_timings.py | 23 +- .../runtime/contracts/test_open_connection.py | 8 +- 20 files changed, 559 insertions(+), 344 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs delete mode 100644 src/sqlbuild/compiler/contract_adoption/main/__init__.py delete mode 100644 src/sqlbuild/compiler/contract_adoption/main/_write.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/__init__.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/_plan_result.py delete mode 100644 src/sqlbuild/runtime/contracts/main/__init__.py delete mode 100644 src/sqlbuild/runtime/contracts/main/open_connection.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 4d694a8f7b..dabff08ac1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod inputs; pub(crate) mod phases; pub(crate) mod pipeline; pub(crate) mod pipeline_entries; +pub(crate) mod plan_result; pub(crate) mod project; pub(crate) mod python_calls; pub(crate) mod python_plan_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 2a71c1c096..4a9604dcb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -8,6 +8,9 @@ use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::{ compile_analysis_selection, project_graph, }; +use crate::bindings::_helpers::compile_driver::plan_result::{ + PlanningInputs, connection_with_hooks, pipeline_result, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -25,9 +28,7 @@ const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_ti const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; -const PLAN_MODULE: &str = "sqlbuild.compiler.pipeline.main._plan_result"; const GENERIC_DIALECT: &str = "generic"; /// The current build's phase timing tracker, when one is recording. @@ -339,25 +340,18 @@ fn planned<'py>( let started: f64 = monotonic(py)?; let mut connection: Option> = None; let result: PyResult> = (|| { - let opened: Bound<'py, PyAny> = call_with( - &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, - &[ - ("adapter", adapter.clone()), - ("connection_config", compiled.getattr("connection_config")?), - ("hooks", hooks.clone()), - ], - )?; + let opened: Bound<'py, PyAny> = + connection_with_hooks(adapter, &compiled.getattr("connection_config")?, hooks)?; connection = Some(opened.clone()); - call_with( - &imported(py, PLAN_MODULE, "build_pipeline_result")?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("connection", opened), - ("project", compiled.getattr("project")?), - ("options", options.clone()), - ("on_progress", hooks.getattr("on_progress")?), - ], + pipeline_result( + &PlanningInputs { + discovered_inputs: discovered, + adapter, + connection: &opened, + }, + &compiled.getattr("project")?, + options, + &hooks.getattr("on_progress")?, ) })(); if let Some(connection) = connection.filter(|connection| !connection.is_none()) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs new file mode 100644 index 0000000000..61cccc946f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -0,0 +1,292 @@ +//! Plan a compiled project over an open warehouse connection, and open that connection. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::phases::monotonic; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; +const RUN_SELECTION_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_selection"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const SINGLE_CONNECTION: i64 = 1; + +/// The deferred relation locations and the warehouse relations behind them, when deferring. +type Deferral<'py> = (Bound<'py, PyAny>, Bound<'py, PyAny>); + +/// The planning inputs: the discovered project, the adapter and its open connection. +pub(crate) struct PlanningInputs<'a, 'py> { + pub(crate) discovered_inputs: &'a Bound<'py, PyAny>, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) connection: &'a Bound<'py, PyAny>, +} + +/// Plan the compiled project over an open connection and attach Python node plans. +pub(crate) fn pipeline_result<'py>( + inputs: &PlanningInputs<'_, 'py>, + project: &Bound<'py, PyAny>, + options: &Bound<'py, PyAny>, + on_progress: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let native = py.import(NATIVE_MODULE)?; + let discovered: &Bound<'py, PyAny> = inputs.discovered_inputs; + let select: Bound<'py, PyAny> = options.getattr("select")?; + let exclude: Bound<'py, PyAny> = options.getattr("exclude")?; + let (deferred_locations, deferred_relations): Deferral<'py> = + deferral(inputs, project, &options.getattr("defer_to")?)?; + let empty = || -> PyResult> { Ok(PyFrozenSet::empty(py)?.into_any()) }; + let mut selected_sql_keys: Bound<'py, PyAny> = py.None().into_bound(py); + let mut selected_python_node_names: Bound<'py, PyAny> = empty()?; + let mut python_read_source_names: Bound<'py, PyAny> = empty()?; + let mut run_selection: Bound<'py, PyAny> = py.None().into_bound(py); + let sql_test_selection: Bound<'py, PyAny> = options.getattr("sql_test_selection")?; + if !sql_test_selection.getattr("models_selected")?.is_truthy()? { + selected_sql_keys = empty()?; + } else if options + .getattr("resolve_python_run_selectors")? + .is_truthy()? + { + let graph: Bound<'py, PyAny> = call_with( + &native.getattr("build_project_graph_from_compiled_project")?, + &[("project", project.clone())], + )?; + run_selection = call_with( + &imported( + py, + RUN_SELECTION_MODULE, + "resolve_python_sql_run_selection_from_inputs", + )?, + &[ + ("select", select.clone()), + ("exclude", exclude.clone()), + ("project_graph", graph), + ("discovered_inputs", discovered.clone()), + ], + )?; + selected_sql_keys = run_selection.getattr("sql_keys")?; + selected_python_node_names = run_selection.getattr("python_node_names")?; + python_read_source_names = call_with( + &native.getattr("python_read_source_names_for")?, + &[ + ("discovered_inputs", discovered.clone()), + ("python_node_names", selected_python_node_names.clone()), + ], + )?; + } + let planner_models = py.import(PLANNER_MODELS_MODULE)?; + let selection: Bound<'py, PyAny> = call_with( + &planner_models.getattr("PlannerSelection")?, + &[ + ("select", select), + ("exclude", exclude), + ("selected_keys", selected_sql_keys), + ("python_read_source_names", python_read_source_names), + ("sql_test_selection", sql_test_selection), + ], + )?; + let overrides: Bound<'py, PyAny> = call_with( + &planner_models.getattr("PlannerOverrides")?, + &option_fields( + options, + &[ + "cursor_overrides", + "full_refresh", + "reload_sources", + "max_microbatches", + ], + )?, + )?; + let mut deferral_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("deferred_locations", deferred_locations), + ("deferred_relations", deferred_relations), + ]; + deferral_fields.extend(option_fields( + options, + &["defer_sources_to", "source_deferral_enabled"], + )?); + let deferral: Bound<'py, PyAny> = + call_with(&planner_models.getattr("DeferralInputs")?, &deferral_fields)?; + let policies: Bound<'py, PyAny> = call_with( + &planner_models.getattr("PlannerPolicies")?, + &option_fields( + options, + &[ + "auto_load_sources", + "selection_diagnostics", + "plan_sql_tests", + "record_migration_fingerprints", + ], + )?, + )?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTION_PLAN_MODULE, "build_execution_plan")?, + &[ + ("project", project.clone()), + ("adapter", inputs.adapter.clone()), + ("connection", inputs.connection.clone()), + ("selection", selection), + ("overrides", overrides), + ("deferral", deferral), + ("policies", policies), + ("on_progress", on_progress.clone()), + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let custom_materializations: Bound<'py, PyAny> = + imported(py, MATERIALIZATIONS_MODULE, "load_custom_materializations")? + .call1((discovered.getattr("materialization_files")?,))?; + let python_outputs: Bound<'py, PyAny> = call_with( + &native.getattr("build_python_run_plan_outputs")?, + &[ + ("discovered_inputs", discovered.clone()), + ("plan_output", plan_output), + ("run_selection", run_selection), + ("selected_python_node_names", selected_python_node_names), + ], + )?; + call_with( + &imported(py, PIPELINE_MODELS_MODULE, "CompilePipelineResult")?, + &[ + ("project", project.clone()), + ("plan_output", python_outputs.getattr("plan_output")?), + ("custom_materializations", custom_materializations), + ( + "python_node_names", + python_outputs.getattr("selected_python_node_names")?, + ), + ( + "python_plan_entries", + python_outputs.getattr("python_plan_entries")?, + ), + ], + ) +} + +/// `(name, options.name)` for each of `names`. +fn option_fields<'a, 'py>( + options: &Bound<'py, PyAny>, + names: &[&'a str], +) -> PyResult)>> { + names + .iter() + .map(|name| Ok((*name, options.getattr(*name)?))) + .collect() +} + +/// The deferred locations and relations of `defer_to`, or two `None`s when not deferring. +fn deferral<'py>( + inputs: &PlanningInputs<'_, 'py>, + project: &Bound<'py, PyAny>, + defer_to: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + if defer_to.is_none() { + return Ok((py.None().into_bound(py), py.None().into_bound(py))); + } + let native = py.import(NATIVE_MODULE)?; + let adapter: &Bound<'py, PyAny> = inputs.adapter; + let target_config: Bound<'py, PyAny> = call_with( + &native.getattr("resolve_deferred_target_config")?, + &[ + ("discovered_inputs", inputs.discovered_inputs.clone()), + ("defer_to", defer_to.clone()), + ( + "current_target_name", + project.getattr("effective_target_name")?, + ), + ], + )?; + let locations: Bound<'py, PyAny> = call_with( + &native.getattr("build_deferred_locations")?, + &[ + ("project", project.clone()), + ("deferred_target_config", target_config), + ("effective_vars", project.getattr("effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ], + )?; + let relations: Bound<'py, PyAny> = call_with( + &native.getattr("gather_deferred_relations")?, + &[ + ("adapter", adapter.clone()), + ("connection", inputs.connection.clone()), + ("deferred_locations", locations.clone()), + ], + )?; + Ok((locations, relations)) +} + +/// Open one adapter connection and report its lifecycle through `hooks`. +pub(crate) fn connection_with_hooks<'py>( + adapter: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, + hooks: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let on_start: Bound<'py, PyAny> = hooks.getattr("on_connection_start")?; + if !on_start.is_none() { + on_start.call1((SINGLE_CONNECTION,))?; + } + let started: f64 = monotonic(py)?; + let elapsed = |started: f64| -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("elapsed_seconds", monotonic(py)? - started)?; + Ok(keywords) + }; + let connection: Bound<'py, PyAny> = match adapter.call_method1("connect", (connection_config,)) + { + Ok(connection) => connection, + Err(error) => { + let on_error: Bound<'py, PyAny> = hooks.getattr("on_connection_error")?; + if !on_error.is_none() { + on_error.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; + } + return Err(error); + } + }; + let on_complete: Bound<'py, PyAny> = hooks.getattr("on_connection_complete")?; + if !on_complete.is_none() { + on_complete.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; + } + Ok(connection) +} + +/// Open one adapter connection and report its lifecycle through hooks. +#[pyfunction] +#[pyo3(signature = (**options))] +fn open_connection_with_hooks<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "open_connection_with_hooks", + options, + (&["adapter", "connection_config", "hooks"], &[]), + )?; + connection_with_hooks( + &arguments.value("adapter")?, + &arguments.value("connection_config")?, + &arguments.value("hooks")?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(open_connection_with_hooks, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 0efbc1611b..917edd104b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_driver::{ analysis, analysis_cache, assembly, command, deferral, diagnostic_collector, diff, discovery, - input_reads, inputs, pipeline, pipeline_entries, project, python_plan_entries, + input_reads, inputs, pipeline, pipeline_entries, plan_result, project, python_plan_entries, relation_targets, selection_phases, targets, }; @@ -19,6 +19,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { discovery::register(module)?; pipeline::register(module)?; pipeline_entries::register(module)?; + plan_result::register(module)?; deferral::register(module)?; diagnostic_collector::register(module)?; diff::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 78a94477a0..0659bfa7f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -21,6 +21,7 @@ use sqlbuild_core::contract_command::models::{ use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -29,7 +30,6 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; -const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; const GENERATE_ACTION: &str = "generate"; /// A `CliUserError` with its code. @@ -258,18 +258,14 @@ fn write_updates<'py>( let status: Bound<'py, PyAny> = progress.status()?; let started: Instant = Instant::now(); status.call_method1("start", ("Updating repository contracts...",))?; - let updated: Bound<'py, PyAny> = match call_with( - &imported(py, WRITE_MODULE, "write_contracts")?, - &[ - ("project_dir", project_dir.clone()), - ("graph", graph.clone()), - ("result", result), - ("overwrite", request.getattr("overwrite")?), - ("adapter", context.getattr("adapter")?), - ("resolved_connection", context.getattr("connection_config")?), - ("cli_vars", request.getattr("cli_vars")?), - ], - ) { + let write: ContractWrite<'py> = ContractWrite { + project_dir: project_dir.clone(), + overwrite: request.getattr("overwrite")?, + adapter: context.getattr("adapter")?, + resolved_connection: context.getattr("connection_config")?, + cli_vars: request.getattr("cli_vars")?, + }; + let updated: Bound<'py, PyAny> = match write_contracts(&write, graph, &result) { Ok(updated) => updated, Err(error) => { if exception(py, &error) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs index 2ef3aae6d5..7e5b22ae11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod command; pub(crate) mod registration; +pub(crate) mod write; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs new file mode 100644 index 0000000000..b5f59917dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -0,0 +1,195 @@ +//! Typed, metadata-preserving repository edits for contract adoption. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const UTF8: &str = "utf-8"; + +/// The repository the edits apply to and how they apply. +pub(crate) struct ContractWrite<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) overwrite: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) resolved_connection: Bound<'py, PyAny>, + pub(crate) cli_vars: Bound<'py, PyAny>, +} + +/// Apply safe source-only edits atomically and validate the resulting project. +pub(crate) fn write_contracts<'py>( + write: &ContractWrite<'py>, + graph: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let native = py.import(NATIVE_MODULE)?; + let project: Bound<'py, PyAny> = graph.getattr("project")?; + let models: Bound<'py, PyDict> = by_name(&project.getattr("models")?)?; + let sources: Bound<'py, PyDict> = by_name(&project.getattr("sources")?)?; + let model_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let updates: Bound<'py, PyDict> = PyDict::new(py); + let updated_evidence: Bound<'py, PyList> = PyList::empty(py); + for evidence in result.getattr("evidence")?.try_iter()? { + let evidence: Bound<'py, PyAny> = evidence?; + let physical_columns: Bound<'py, PyAny> = evidence.getattr("physical_columns")?; + let findings: Bound<'py, PyAny> = evidence.getattr("findings")?; + if physical_columns.is_none() || !findings.is_truthy()? { + updated_evidence.append(evidence)?; + continue; + } + let name: Bound<'py, PyAny> = evidence.getattr("resource_name")?; + let (path, edited): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + if evidence.getattr("resource_type")?.eq(&model_type)? { + let model: Bound<'py, PyAny> = models.as_any().get_item(&name)?; + let edited: Bound<'py, PyAny> = call_with( + &native.getattr("edit_contract_model")?, + &[ + ("model", model.clone()), + ("physical_columns", physical_columns), + ("overwrite", write.overwrite.clone()), + ], + )?; + ( + write.project_dir.div(model.getattr("relative_path")?)?, + edited, + ) + } else { + let source: Bound<'py, PyAny> = sources.as_any().get_item(&name)?; + let path: Bound<'py, PyAny> = + source.getattr("source_file")?.getattr("file_path")?; + let edited: Bound<'py, PyAny> = call_with( + &native.getattr("edit_contract_source")?, + &[ + ("source", source), + ("physical_columns", physical_columns), + ("overwrite", write.overwrite.clone()), + ("contents", updates.as_any().call_method1("get", (&path,))?), + ], + )?; + (path, edited) + }; + let (contents, conflict): (Bound<'py, PyAny>, Bound<'py, PyAny>) = edited.extract()?; + if !contents.is_none() && contents.ne(read_text(&path)?)? { + updates.set_item(&path, contents)?; + } + let remaining: Bound<'py, PyAny> = call_with( + &native.getattr("remaining_contract_findings")?, + &[ + ("findings", findings), + ("overwrite", write.overwrite.clone()), + ("conflict", conflict), + ], + )?; + updated_evidence.append(replaced(&evidence, &[("findings", remaining)])?)?; + } + let evidence: Bound<'py, PyAny> = PyTuple::new(py, updated_evidence)?.into_any(); + if updates.is_empty() { + return replaced(result, &[("evidence", evidence)]); + } + let originals: Bound<'py, PyDict> = PyDict::new(py); + for path in updates.keys() { + originals.set_item(&path, read_text(&path)?)?; + } + if let Err(error) = applied(write, result, &updates) { + if error.is_instance_of::(py) { + write_all(&originals)?; + } + return Err(error); + } + let relative: Vec> = updates + .keys() + .iter() + .map(|path| path.call_method1("relative_to", (&write.project_dir,))) + .collect::>()?; + let written: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((relative,))?; + replaced( + result, + &[ + ("evidence", evidence), + ( + "written_paths", + PyTuple::new(py, written.try_iter()?.collect::>>()?)?.into_any(), + ), + ], + ) +} + +/// Write every update, rediscover the project and compile it when an adapter is available. +fn applied<'py>( + write: &ContractWrite<'py>, + result: &Bound<'py, PyAny>, + updates: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = result.py(); + let native = py.import(NATIVE_MODULE)?; + write_all(updates)?; + let discovered: Bound<'py, PyAny> = call_with( + &native.getattr("discover_project_inputs")?, + &[("project_dir", write.project_dir.clone())], + )?; + if write.adapter.is_none() { + return Ok(()); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("no_cache", true)?; + let selection: Bound<'py, PyAny> = + imported(py, COMPILE_MODELS_MODULE, "CompileAnalysisSelection")? + .call((), Some(&keywords))?; + call_with( + &native.getattr("build_compiled_project")?, + &[ + ("discovered_inputs", discovered), + ("adapter", write.adapter.clone()), + ("selected_target", result.getattr("from_target")?), + ("resolved_connection", write.resolved_connection.clone()), + ("cli_vars", write.cli_vars.clone()), + ("analysis_selection", selection), + ], + )?; + Ok(()) +} + +/// Write each `path: contents` pair atomically, in order. +fn write_all(contents_by_path: &Bound<'_, PyDict>) -> PyResult<()> { + let py: Python<'_> = contents_by_path.py(); + let write_atomically: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "write_atomically")?; + for (path, contents) in contents_by_path.iter() { + call_with(&write_atomically, &[("path", path), ("contents", contents)])?; + } + Ok(()) +} + +fn read_text<'py>(path: &Bound<'py, PyAny>) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(path.py()); + keywords.set_item("encoding", UTF8)?; + path.call_method("read_text", (), Some(&keywords)) +} + +/// `{item.name: item for item in items}`. +fn by_name<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let named: Bound<'py, PyDict> = PyDict::new(items.py()); + for item in items.try_iter()? { + let item: Bound<'py, PyAny> = item?; + named.set_item(item.getattr("name")?, &item)?; + } + Ok(named) +} + +/// `dataclasses.replace(value, **fields)`. +fn replaced<'py>( + value: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = value.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, field) in fields { + keywords.set_item(*name, field)?; + } + imported(py, "dataclasses", "replace")?.call((value,), Some(&keywords)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index a85e301568..570b82d1fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, use crate::bindings::_helpers::command_runtime::connection::{ anchor_duckdb_database, with_dbt_profile, }; +use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; @@ -120,18 +121,7 @@ fn open_connection<'py>( )); } let relayed = call_with(&imported(py, HOOKS, "ConnectionHooks")?, &pairs)?; - call_with( - &imported( - py, - "sqlbuild.runtime.contracts.main.open_connection", - "open_connection_with_hooks", - )?, - &[ - ("adapter", environment.getattr("adapter")?), - ("connection_config", config), - ("hooks", relayed), - ], - ) + connection_with_hooks(&environment.getattr("adapter")?, &config, &relayed) } /// Build the SQLBuild plan from current warehouse facts, closing the custom adapter connection. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 97aa00d23c..2b26d2a393 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -15,6 +15,7 @@ use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; +use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -24,7 +25,6 @@ const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const SEED_PLAN_MODULE: &str = "sqlbuild._native"; const SEED_STATE_MODULE: &str = "sqlbuild._native"; -const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const SUCCESS_STATUS: &str = "success"; @@ -248,14 +248,8 @@ fn compile_seeds<'py>( ("on_connection_error", relay.getattr("on_connection_error")?), ], )?; - let connection: Bound<'py, PyAny> = call_with( - &imported(py, OPEN_CONNECTION_MODULE, "open_connection_with_hooks")?, - &[ - ("adapter", adapter.clone()), - ("connection_config", connection_config), - ("hooks", hooks), - ], - )?; + let connection: Bound<'py, PyAny> = + connection_with_hooks(&adapter, &connection_config, &hooks)?; let read: PyResult> = call_with( &imported(py, SEED_STATE_MODULE, "read_selected_seed_fingerprints")?, &[ diff --git a/fensu.toml b/fensu.toml index 7d94b0e267..7ae6f0f6a5 100644 --- a/fensu.toml +++ b/fensu.toml @@ -78,6 +78,21 @@ rule = "FFR309" path = "src/sqlbuild/compiler/project_assembly/__init__.py" reason = "Native bindings own project assembly and SQL syntax checks; this package only declares the resource fact models the native assembly builds." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/contract_adoption/__init__.py" +reason = "Native bindings own contract comparison, edits and repository writes; this package only declares the adoption models, types and errors the native command builds and raises." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/pipeline/__init__.py" +reason = "Native bindings own the compile-and-plan pipeline; this package declares the pipeline models and keeps the custom materialization loader that imports project code." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/runtime/contracts/__init__.py" +reason = "Native bindings open planning connections and report their lifecycle; this package only declares the connection hook models and types callers pass." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 3107742933..52b9296e20 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 14436 +remaining_lines = 13677 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 20672b333b..8ef10a14b4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3506,6 +3506,9 @@ class CollectedCompileDiagnostics: def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... def collect_compile_diagnostics() -> CollectedCompileDiagnostics: ... +def open_connection_with_hooks( + *, adapter: BaseAdapter, connection_config: dict[str, object], hooks: ConnectionHooks +) -> Any: ... def build_analysis_cache_context( *, root: Path | None, diff --git a/src/sqlbuild/compiler/contract_adoption/main/__init__.py b/src/sqlbuild/compiler/contract_adoption/main/__init__.py deleted file mode 100644 index 443652aac9..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public contract adoption operations.""" diff --git a/src/sqlbuild/compiler/contract_adoption/main/_write.py b/src/sqlbuild/compiler/contract_adoption/main/_write.py deleted file mode 100644 index 7050ced037..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/main/_write.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Typed, metadata-preserving repository edits for contract adoption.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - build_compiled_project, - discover_project_inputs, - edit_contract_model, - edit_contract_source, - remaining_contract_findings, - write_atomically, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledResourceType, - CompiledSourceView, -) -from sqlbuild.compiler.contract_adoption.models import ( - ContractAdoptionResult, - ContractEvidence, - ContractFinding, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph - - -def write_contracts( - *, - project_dir: Path, - graph: ProjectGraph, - result: ContractAdoptionResult, - overwrite: bool, - adapter: BaseAdapter | None = None, - resolved_connection: dict[str, object] | None = None, - cli_vars: dict[str, object] | None = None, -) -> ContractAdoptionResult: - """Apply safe source-only edits atomically and validate the resulting project.""" - - models: dict[str, CompiledModelView] = {model.name: model for model in graph.project.models} - sources: dict[str, CompiledSourceView] = { - source.name: source for source in graph.project.sources - } - updates: dict[Path, str] = {} - updated_evidence: list[ContractEvidence] = [] - for evidence in result.evidence: - if evidence.physical_columns is None or not evidence.findings: - updated_evidence.append(evidence) - continue - if evidence.resource_type == CompiledResourceType.MODEL: - contents, conflict = edit_contract_model( - model=models[evidence.resource_name], - physical_columns=evidence.physical_columns, - overwrite=overwrite, - ) - path: Path = project_dir / models[evidence.resource_name].relative_path - else: - path = sources[evidence.resource_name].source_file.file_path - contents, conflict = edit_contract_source( - source=sources[evidence.resource_name], - physical_columns=evidence.physical_columns, - overwrite=overwrite, - contents=updates.get(path), - ) - if contents is not None and contents != path.read_text(encoding="utf-8"): - updates[path] = contents - remaining: tuple[ContractFinding, ...] = remaining_contract_findings( - findings=evidence.findings, - overwrite=overwrite, - conflict=conflict, - ) - updated_evidence.append(replace(evidence, findings=remaining)) - if not updates: - return replace(result, evidence=tuple(updated_evidence)) - originals: dict[Path, str] = {path: path.read_text(encoding="utf-8") for path in updates} - try: - for path, contents in updates.items(): - write_atomically(path=path, contents=contents) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - if adapter is not None: - _ = build_compiled_project( - discovered_inputs=discovered, - adapter=adapter, - selected_target=result.from_target, - resolved_connection=resolved_connection, - cli_vars=cli_vars, - analysis_selection=CompileAnalysisSelection(no_cache=True), - ) - except Exception: - for path, contents in originals.items(): - write_atomically(path=path, contents=contents) - raise - return replace( - result, - evidence=tuple(updated_evidence), - written_paths=tuple(sorted(path.relative_to(project_dir) for path in updates)), - ) diff --git a/src/sqlbuild/compiler/pipeline/main/__init__.py b/src/sqlbuild/compiler/pipeline/main/__init__.py deleted file mode 100644 index bd9b94f6a1..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Pipeline main package.""" diff --git a/src/sqlbuild/compiler/pipeline/main/_plan_result.py b/src/sqlbuild/compiler/pipeline/main/_plan_result.py deleted file mode 100644 index ad05558578..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_plan_result.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Plan a compiled project over an open warehouse connection.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import build_execution_plan -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline._helpers.materializations import load_custom_materializations -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - PythonRunPlanOutputs, -) -from sqlbuild.compiler.planner.models import ( - DeferralInputs, - PlannerOverrides, - PlannerPolicies, - PlannerSelection, - PlanOutput, -) -from sqlbuild.compiler.python_nodes.main._run_selection import ( - resolve_python_sql_run_selection_from_inputs, -) -from sqlbuild.compiler.python_nodes.models import PythonSqlRunSelection -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_pipeline_result( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection: Any, - project: CompiledProjectView, - options: CompilePipelineOptions, - on_progress: Callable[[str], None] | None = None, -) -> CompilePipelineResult: - """Plan the compiled project over an open connection and attach Python node plans.""" - - select: tuple[str, ...] = options.select - exclude: tuple[str, ...] = options.exclude - deferred_locations: dict[str, CompiledRelationLocation] | None = None - deferred_relations: dict[str, RelationInfo] | None = None - if options.defer_to is not None: - deferred_target_config: TargetConfig = _native.resolve_deferred_target_config( - discovered_inputs=discovered_inputs, - defer_to=options.defer_to, - current_target_name=project.effective_target_name, - ) - deferred_locations = _native.build_deferred_locations( - project=project, - deferred_target_config=deferred_target_config, - effective_vars=project.effective_vars, - default_schema=adapter.default_schema(), - default_database=adapter.default_database(), - render_qualified_name=adapter.render_qualified_name, - ) - deferred_relations = _native.gather_deferred_relations( - adapter=adapter, - connection=connection, - deferred_locations=deferred_locations, - ) - - selected_sql_keys: frozenset[CompiledObjectKey] | None = None - selected_python_node_names: frozenset[str] = frozenset() - python_read_source_names: frozenset[str] = frozenset() - run_selection: PythonSqlRunSelection | None = None - if not options.sql_test_selection.models_selected: - selected_sql_keys = frozenset() - elif options.resolve_python_run_selectors: - run_selection = resolve_python_sql_run_selection_from_inputs( - select=select, - exclude=exclude, - project_graph=_native.build_project_graph_from_compiled_project(project=project), - discovered_inputs=discovered_inputs, - ) - selected_sql_keys = run_selection.sql_keys - selected_python_node_names = run_selection.python_node_names - python_read_source_names = _native.python_read_source_names_for( - discovered_inputs=discovered_inputs, python_node_names=selected_python_node_names - ) - - plan_output: PlanOutput = build_execution_plan( - project=project, - adapter=adapter, - connection=connection, - selection=PlannerSelection( - select=select, - exclude=exclude, - selected_keys=selected_sql_keys, - python_read_source_names=python_read_source_names, - sql_test_selection=options.sql_test_selection, - ), - overrides=PlannerOverrides( - cursor_overrides=options.cursor_overrides, - full_refresh=options.full_refresh, - reload_sources=options.reload_sources, - max_microbatches=options.max_microbatches, - ), - deferral=DeferralInputs( - deferred_locations=deferred_locations, - deferred_relations=deferred_relations, - defer_sources_to=options.defer_sources_to, - source_deferral_enabled=options.source_deferral_enabled, - ), - policies=PlannerPolicies( - auto_load_sources=options.auto_load_sources, - selection_diagnostics=options.selection_diagnostics, - plan_sql_tests=options.plan_sql_tests, - record_migration_fingerprints=options.record_migration_fingerprints, - ), - on_progress=on_progress, - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - custom_materializations: dict[str, Any] = load_custom_materializations( - discovered_inputs.materialization_files - ) - python_outputs: PythonRunPlanOutputs = _native.build_python_run_plan_outputs( - discovered_inputs=discovered_inputs, - plan_output=plan_output, - run_selection=run_selection, - selected_python_node_names=selected_python_node_names, - ) - - return CompilePipelineResult( - project=project, - plan_output=python_outputs.plan_output, - custom_materializations=custom_materializations, - python_node_names=python_outputs.selected_python_node_names, - python_plan_entries=python_outputs.python_plan_entries, - ) diff --git a/src/sqlbuild/runtime/contracts/main/__init__.py b/src/sqlbuild/runtime/contracts/main/__init__.py deleted file mode 100644 index e5537acbae..0000000000 --- a/src/sqlbuild/runtime/contracts/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Runtime contract operations.""" diff --git a/src/sqlbuild/runtime/contracts/main/open_connection.py b/src/sqlbuild/runtime/contracts/main/open_connection.py deleted file mode 100644 index 8c92accf53..0000000000 --- a/src/sqlbuild/runtime/contracts/main/open_connection.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Callback-aware adapter connection opening.""" - -from __future__ import annotations - -import time -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def open_connection_with_hooks( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - hooks: ConnectionHooks, -) -> Any: - """Open one adapter connection and report its lifecycle through hooks.""" - - if hooks.on_connection_start is not None: - hooks.on_connection_start(1) - started_at: float = time.monotonic() - try: - connection: Any = adapter.connect(connection_config) - except Exception: - if hooks.on_connection_error is not None: - hooks.on_connection_error(1, elapsed_seconds=time.monotonic() - started_at) - raise - if hooks.on_connection_complete is not None: - hooks.on_connection_complete(1, elapsed_seconds=time.monotonic() - started_at) - return connection diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index deee9d90ef..689af9cae3 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -11,14 +11,14 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.main import _plan_result as plan_result +from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult +from sqlbuild.compiler.planner.main.execution import execution from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from sqlbuild.rule_engine._helpers.run import rule_threads from sqlbuild.rule_engine.models import RulesRunResult -from sqlbuild.runtime.contracts.main import open_connection from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase @@ -28,14 +28,14 @@ [ PipelinePhaseTimingTestCase( description="slow planning connection is excluded from direct compilation", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 13.0, 13.0), + clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 13.0, 13.0), expected_compile_seconds=2.0, expected_planning_seconds=10.0, expected_total_seconds=13.0, ), PipelinePhaseTimingTestCase( description="rules wall time is reported and excluded from compile and planning", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 13.0, 13.0), + clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 3.0, 13.0, 13.0), expected_compile_seconds=2.0, expected_planning_seconds=10.0, expected_total_seconds=13.0, @@ -66,8 +66,19 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di adapter: Mock = Mock() monkeypatch.setattr(time, "monotonic", monotonic) monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) - monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) - monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) + monkeypatch.setattr(execution, "build_execution_plan", Mock(return_value=PlanOutput())) + monkeypatch.setattr(materializations, "load_custom_materializations", Mock(return_value={})) + monkeypatch.setattr( + _native, + "build_python_run_plan_outputs", + Mock( + return_value=Mock( + plan_output=pipeline_result.plan_output, + selected_python_node_names=frozenset(), + python_plan_entries=(), + ) + ), + ) (tmp_path / "sqlbuild_project.toml").write_text('name = "timing"\n', encoding="utf-8") monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py index 79daaa59c6..aa03c92f52 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py @@ -1,11 +1,11 @@ from __future__ import annotations +import time from collections.abc import Iterator import pytest -import sqlbuild.runtime.contracts.main.open_connection as open_connection_module -from sqlbuild.runtime.contracts.main.open_connection import open_connection_with_hooks +from sqlbuild._native import open_connection_with_hooks from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.runtime.contracts._test_types import ( OpenConnectionFailureTestCase, @@ -44,7 +44,7 @@ def test_given_connection_hooks_when_opening_succeeds_then_reports_order_and_ela connection=connection, ) monotonic_times: Iterator[float] = iter(test_case.monotonic_times) - monkeypatch.setattr(open_connection_module.time, "monotonic", lambda: next(monotonic_times)) + monkeypatch.setattr(time, "monotonic", lambda: next(monotonic_times)) def on_start(connection_count: int) -> None: events.append("start") @@ -98,7 +98,7 @@ def test_given_connection_hooks_when_opening_fails_then_reports_error_and_rerais error=test_case.expected_error, ) monotonic_times: Iterator[float] = iter(test_case.monotonic_times) - monkeypatch.setattr(open_connection_module.time, "monotonic", lambda: next(monotonic_times)) + monkeypatch.setattr(time, "monotonic", lambda: next(monotonic_times)) def on_start(connection_count: int) -> None: events.append("start") From a36666d1ec5b3bca18a508f62313168cb832aa15 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:35:15 +0100 Subject: [PATCH 23/33] refactor: host setting help, expected errors, lint stops and phase trackers natively Render setting and snippet help in sqlbuild-config, read expected-error codes, messages and help, scope the speculative lint stop signal, and track build phase timings and process resource deltas in the native bindings. Delete the Python helpers they replace and the help renderers only tests used. --- .../src/project/_helpers/help.rs | 3 + .../sqlbuild-config/src/project/main/mod.rs | 2 + .../src/project/main/setting_help_text.rs | 23 ++ .../src/project/main/snippet_help_text.rs | 13 ++ crates/sqlbuild-config/src/project/models.rs | 17 ++ .../src/project/tests/test_guidance.rs | 44 ++++ .../_helpers/boundary/expected_errors.rs | 38 +++ .../src/bindings/_helpers/boundary/mod.rs | 1 + .../_helpers/build_command/command.rs | 2 +- .../_helpers/compile_driver/pipeline.rs | 2 +- .../_helpers/compile_driver/targets.rs | 2 +- .../src/bindings/_helpers/diagnostics/mod.rs | 1 + .../bindings/_helpers/diagnostics/trackers.rs | 220 ++++++++++++++++++ .../bindings/_helpers/discovery/guidance.rs | 57 ++++- .../_helpers/rules/lint_preparation.rs | 6 +- .../src/bindings/_helpers/rules/lint_stop.rs | 92 ++++++++ .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 6 +- .../src/bindings/_helpers/rules/run_lint.rs | 8 +- .../src/bindings/main/register.rs | 2 + fensu.toml | 5 + python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 43 ++++ .../_helpers/diagnostics/python_resources.py | 2 +- .../classes/build_phase_timing_tracker.py | 58 ----- .../classes/process_resource_tracker.py | 35 --- .../main/process_resource_reporting.py | 2 +- .../errors/contracts/main/__init__.py | 1 - .../errors/contracts/main/error_code.py | 9 - .../errors/contracts/main/error_help.py | 10 - .../errors/contracts/main/error_message.py | 9 - src/sqlbuild/errors/setting_help/__init__.py | 0 .../errors/setting_help/_helpers/__init__.py | 0 .../errors/setting_help/_helpers/text.py | 70 ------ src/sqlbuild/errors/setting_help/constants.py | 7 - .../errors/setting_help/main/__init__.py | 0 .../errors/setting_help/main/join_helps.py | 11 - .../setting_help/main/model_header_help.py | 11 - .../errors/setting_help/main/setting_help.py | 16 -- .../errors/setting_help/main/setting_note.py | 16 -- .../errors/setting_help/main/snippet_help.py | 11 - src/sqlbuild/errors/setting_help/types.py | 5 - .../executor/functions/main/_execute.py | 6 +- .../executor/pipeline/_helpers/connections.py | 2 +- .../executor/pipeline/_helpers/scenario.py | 6 +- src/sqlbuild/executor/pipeline/main/run.py | 3 +- .../run/_helpers/execution/results.py | 5 +- src/sqlbuild/executor/run/main/_execute.py | 2 +- .../scenario/_helpers/local/execution.py | 4 +- .../scenario/_helpers/local/snapshots.py | 3 +- .../scenario/_helpers/snapshots/core.py | 3 +- .../executor/scenario/main/_capture.py | 2 +- src/sqlbuild/lint/classes/__init__.py | 1 - src/sqlbuild/lint/classes/stop_context.py | 21 -- .../rule_engine/_helpers/run/rule_threads.py | 8 +- .../sqlbuild/cli/commands/main/test_rules.py | 5 +- .../main/test_compile_phase_timings.py | 2 +- .../diagnostics/test_process_resources.py | 2 +- .../setting_help/main/test_setting_help.py | 41 +--- 59 files changed, 603 insertions(+), 376 deletions(-) create mode 100644 crates/sqlbuild-config/src/project/main/setting_help_text.rs create mode 100644 crates/sqlbuild-config/src/project/main/snippet_help_text.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs delete mode 100644 src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py delete mode 100644 src/sqlbuild/diagnostics/classes/process_resource_tracker.py delete mode 100644 src/sqlbuild/errors/contracts/main/__init__.py delete mode 100644 src/sqlbuild/errors/contracts/main/error_code.py delete mode 100644 src/sqlbuild/errors/contracts/main/error_help.py delete mode 100644 src/sqlbuild/errors/contracts/main/error_message.py delete mode 100644 src/sqlbuild/errors/setting_help/__init__.py delete mode 100644 src/sqlbuild/errors/setting_help/_helpers/__init__.py delete mode 100644 src/sqlbuild/errors/setting_help/_helpers/text.py delete mode 100644 src/sqlbuild/errors/setting_help/constants.py delete mode 100644 src/sqlbuild/errors/setting_help/main/__init__.py delete mode 100644 src/sqlbuild/errors/setting_help/main/join_helps.py delete mode 100644 src/sqlbuild/errors/setting_help/main/model_header_help.py delete mode 100644 src/sqlbuild/errors/setting_help/main/setting_help.py delete mode 100644 src/sqlbuild/errors/setting_help/main/setting_note.py delete mode 100644 src/sqlbuild/errors/setting_help/main/snippet_help.py delete mode 100644 src/sqlbuild/errors/setting_help/types.py delete mode 100644 src/sqlbuild/lint/classes/__init__.py delete mode 100644 src/sqlbuild/lint/classes/stop_context.py diff --git a/crates/sqlbuild-config/src/project/_helpers/help.rs b/crates/sqlbuild-config/src/project/_helpers/help.rs index 14bf4a9fe6..9cdb500837 100644 --- a/crates/sqlbuild-config/src/project/_helpers/help.rs +++ b/crates/sqlbuild-config/src/project/_helpers/help.rs @@ -13,6 +13,8 @@ use crate::project::errors::ConfigFailure; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum SettingValue<'a> { Bool(bool), + /// An integer, as its decimal text. + Integer(&'a str), Text(&'a str), } @@ -29,6 +31,7 @@ fn toml_value(value: SettingValue<'_>) -> String { match value { SettingValue::Bool(true) => "true".to_owned(), SettingValue::Bool(false) => "false".to_owned(), + SettingValue::Integer(digits) => digits.to_owned(), SettingValue::Text(text) => { format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) } diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 6a2fc8fe75..a3e65d82e8 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -4,5 +4,7 @@ pub mod load_project_config; pub mod local_sql_analysis_failure; pub mod microbatch_guidance; pub mod resolve_config_file; +pub mod setting_help_text; +pub mod snippet_help_text; pub mod sql_analysis_off_guidance; pub mod unsupported_target_warehouse; diff --git a/crates/sqlbuild-config/src/project/main/setting_help_text.rs b/crates/sqlbuild-config/src/project/main/setting_help_text.rs new file mode 100644 index 0000000000..734be6fbc4 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/setting_help_text.rs @@ -0,0 +1,23 @@ +//! Help that shows the exact setting to write. + +use crate::project::_helpers::help::{Setting, SettingValue, setting_help}; +use crate::project::models::{SettingSite, SettingText}; + +/// `, set this in :` followed by the exact TOML section and key. +#[must_use] +pub fn setting_help_text(purpose: &str, site: SettingSite<'_>, value: &SettingText) -> String { + let value: SettingValue<'_> = match value { + SettingText::Bool(flag) => SettingValue::Bool(*flag), + SettingText::Integer(digits) => SettingValue::Integer(digits), + SettingText::Text(text) => SettingValue::Text(text), + }; + setting_help( + purpose, + &Setting { + file_name: site.file_name, + section: site.section, + key: site.key, + value, + }, + ) +} diff --git a/crates/sqlbuild-config/src/project/main/snippet_help_text.rs b/crates/sqlbuild-config/src/project/main/snippet_help_text.rs new file mode 100644 index 0000000000..c21bb6bc3c --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/snippet_help_text.rs @@ -0,0 +1,13 @@ +//! Help that shows an exact authored snippet to add. + +use crate::project::constants::SETTING_SNIPPET_INDENT; + +/// `, :` followed by the exact lines to write, one per line. +#[must_use] +pub fn snippet_help_text(purpose: &str, target: &str, lines: &[String]) -> String { + let snippet: Vec = lines + .iter() + .map(|line| format!("{SETTING_SNIPPET_INDENT}{line}")) + .collect(); + format!("{purpose}, {target}:\n{}", snippet.join("\n")) +} diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 2874ba17bb..ff2a018aa6 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -38,3 +38,20 @@ pub enum Built { /// `Decimal(str(value))` of an integer or float. Decimal(ConfigValue), } + +/// A setting value as help text shows it in TOML. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SettingText { + Bool(bool), + /// An integer, as its decimal text. + Integer(String), + Text(String), +} + +/// Where a setting lives: its file name, TOML section and key. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SettingSite<'a> { + pub file_name: &'a str, + pub section: &'a str, + pub key: &'a str, +} diff --git a/crates/sqlbuild-config/src/project/tests/test_guidance.rs b/crates/sqlbuild-config/src/project/tests/test_guidance.rs index fe68a81e20..f8277f5fb6 100644 --- a/crates/sqlbuild-config/src/project/tests/test_guidance.rs +++ b/crates/sqlbuild-config/src/project/tests/test_guidance.rs @@ -1,6 +1,9 @@ use crate::project::main::explicit_references_help::explicit_references_help; use crate::project::main::microbatch_guidance::microbatch_guidance; +use crate::project::main::setting_help_text::setting_help_text; +use crate::project::main::snippet_help_text::snippet_help_text; use crate::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance; +use crate::project::models::{SettingSite, SettingText}; use crate::project::tests::test_types::GuidanceTestCase; #[test] @@ -24,6 +27,47 @@ fn given_setting_guidance_when_rendering_then_text_matches_python() { expected_note: "the current value is [references] enforce_explicit = true (from sqlbuild_project.toml or its default); while migrating a project, to allow X, set this in sqlbuild_project.toml:\n [references]\n enforce_explicit = false", expected_help: "", }, + GuidanceTestCase { + description: "a quoted string setting and an integer setting", + rendered: ( + setting_help_text( + "to stage", + SettingSite { + file_name: "sqlbuild_project.toml", + section: "settings", + key: "table_promotion_mode", + }, + &SettingText::Text("st\"aged\\".to_owned()), + ), + setting_help_text( + "to wait", + SettingSite { + file_name: "sqlbuild_local.toml", + section: "settings", + key: "timeout", + }, + &SettingText::Integer("30".to_owned()), + ), + ), + expected_note: "to stage, set this in sqlbuild_project.toml:\n [settings]\n table_promotion_mode = \"st\\\"aged\\\\\"", + expected_help: "to wait, set this in sqlbuild_local.toml:\n [settings]\n timeout = 30", + }, + GuidanceTestCase { + description: "a snippet help shows every line to write", + rendered: ( + snippet_help_text( + "to describe it", + "add this to seeds/lookups.yml", + &[ + "- name: product_types".to_owned(), + " description: Types".to_owned(), + ], + ), + String::new(), + ), + expected_note: "to describe it, add this to seeds/lookups.yml:\n - name: product_types\n description: Types", + expected_help: "", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs new file mode 100644 index 0000000000..275cde1b9c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/expected_errors.rs @@ -0,0 +1,38 @@ +//! The stable code, message and help of an expected exception-like object. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; + +/// Return a stable code from an expected exception-like object. +#[pyfunction] +#[pyo3(signature = (*, error, fallback_code))] +fn error_code(error: &Bound<'_, PyAny>, fallback_code: &Bound<'_, PyAny>) -> PyResult { + match error.getattr_opt("code")? { + Some(code) => Ok(code.str()?.to_string()), + None => Ok(fallback_code.str()?.to_string()), + } +} + +/// Return expected-error help text when present. +#[pyfunction] +fn error_help<'py>(error: &Bound<'py, PyAny>) -> PyResult>> { + Ok(error + .getattr_opt("help")? + .filter(|help| help.is_instance_of::())) +} + +/// Return a stable message from an expected exception-like object. +#[pyfunction] +fn error_message(error: &Bound<'_, PyAny>) -> PyResult { + match error.getattr_opt("message")? { + Some(message) => Ok(message.str()?.to_string()), + None => Ok(error.str()?.to_string()), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(error_code, module)?)?; + module.add_function(wrap_pyfunction!(error_help, module)?)?; + module.add_function(wrap_pyfunction!(error_message, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 7c0e808fc6..56be30d4df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,5 +1,6 @@ mod allocator; pub(crate) mod config_values; +pub(crate) mod expected_errors; pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 593374b7c0..9069c440bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::build_command::planning::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; -const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; const RESOURCE_REPORTING_MODULE: &str = "sqlbuild.diagnostics.main.process_resource_reporting"; const DEADLINE_MODULE: &str = "sqlbuild.runtime.execution_limits.main.execution_deadline_scope"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 4a9604dcb2..8bf92a1d30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -24,7 +24,7 @@ const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index ab172b2417..d64171676c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -30,7 +30,7 @@ const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; -const SETTING_HELP_MODULE: &str = "sqlbuild.errors.setting_help.main.setting_help"; +const SETTING_HELP_MODULE: &str = "sqlbuild._native"; /// Connection settings that identify the warehouse two targets write to. const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ "account", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index 5dfdec4811..14fc125d3d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1 +1,2 @@ pub(crate) mod redaction; +pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs new file mode 100644 index 0000000000..1682c57e86 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -0,0 +1,220 @@ +//! Invocation-scoped partial build phase timings and process resource deltas. + +use std::sync::{Mutex, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyType}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild.diagnostics.models"; +const PROCESS_RESOURCES_MODULE: &str = "sqlbuild.diagnostics._helpers.process_resources"; +const TRACKER_VARIABLE: &str = "sqlbuild_build_phase_timing_tracker"; +const PHASES: [&str; 6] = [ + "compile_seconds", + "planning_seconds", + "connection_preparation_seconds", + "schema_preparation_seconds", + "execution_seconds", + "cost_collection_seconds", +]; + +static CURRENT_TRACKER: PyOnceLock> = PyOnceLock::new(); + +/// The context variable holding the active build timing tracker. +fn current_variable(py: Python<'_>) -> PyResult> { + CURRENT_TRACKER + .get_or_try_init(py, || { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")? + .call((TRACKER_VARIABLE,), Some(&keywords)) + .map(Bound::unbind) + }) + .map(|variable| variable.bind(py).clone()) +} + +/// `monotonic`, or `time.monotonic` when none is given. +fn clock<'py>( + py: Python<'py>, + monotonic: Option>, +) -> PyResult> { + match monotonic { + Some(monotonic) => Ok(monotonic), + None => imported(py, "time", "monotonic"), + } +} + +/// `max(0.0, later - earlier)`. +fn non_negative(later: f64, earlier: f64) -> f64 { + (later - earlier).max(0.0) +} + +/// Retain disjoint monotonic phase durations across exceptional exits. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct BuildPhaseTimingTracker { + monotonic: Py, + started_at: f64, + #[pyo3(get, set)] + compile_seconds: Option, + #[pyo3(get, set)] + planning_seconds: Option, + #[pyo3(get, set)] + connection_preparation_seconds: Option, + #[pyo3(get, set)] + schema_preparation_seconds: Option, + #[pyo3(get, set)] + execution_seconds: Option, + #[pyo3(get, set)] + cost_collection_seconds: Option, + tokens: Mutex>>, +} + +#[pymethods] +impl BuildPhaseTimingTracker { + #[new] + #[pyo3(signature = (*, monotonic = None))] + fn new(py: Python<'_>, monotonic: Option>) -> PyResult { + let monotonic: Bound<'_, PyAny> = clock(py, monotonic)?; + let started_at: f64 = monotonic.call0()?.extract()?; + Ok(Self { + monotonic: monotonic.unbind(), + started_at, + compile_seconds: None, + planning_seconds: None, + connection_preparation_seconds: None, + schema_preparation_seconds: None, + execution_seconds: None, + cost_collection_seconds: None, + tokens: Mutex::new(Vec::new()), + }) + } + + /// Return currently available phase durations and total wall time. + fn snapshot<'py>(&self, py: Python<'py>) -> PyResult> { + let now: f64 = self.monotonic.bind(py).call0()?.extract()?; + let values: [Option; 6] = [ + self.compile_seconds, + self.planning_seconds, + self.connection_preparation_seconds, + self.schema_preparation_seconds, + self.execution_seconds, + self.cost_collection_seconds, + ]; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (phase, value) in PHASES.into_iter().zip(values) { + keywords.set_item(phase, value)?; + } + keywords.set_item("total_seconds", non_negative(now, self.started_at))?; + imported(py, DIAGNOSTICS_MODELS_MODULE, "PartialBuildPhaseTimings")? + .call((), Some(&keywords)) + } + + /// Return the active build timing tracker. + #[classmethod] + fn current<'py>(cls: &Bound<'py, PyType>) -> PyResult> { + current_variable(cls.py())?.call_method0("get") + } + + /// Install this tracker for one command invocation and reset it afterward. + fn scope(slf: &Bound<'_, Self>) -> PhaseTimingScope { + PhaseTimingScope { + tracker: slf.clone().unbind(), + } + } +} + +impl BuildPhaseTimingTracker { + fn tokens(&self) -> std::sync::MutexGuard<'_, Vec>> { + self.tokens.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +/// Makes one tracker the active build timing tracker while entered. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct PhaseTimingScope { + tracker: Py, +} + +#[pymethods] +impl PhaseTimingScope { + fn __enter__(&self, py: Python<'_>) -> PyResult<()> { + let token: Bound<'_, PyAny> = + current_variable(py)?.call_method1("set", (self.tracker.bind(py),))?; + self.tracker.bind(py).borrow().tokens().push(token.unbind()); + Ok(()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self.tracker.bind(py).borrow().tokens().pop(); + if let Some(token) = token { + current_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// Capture wall and process resource deltas around one command. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ProcessResourceTracker { + monotonic: Py, + resource_reader: Py, + started_at: f64, + started_user: f64, + started_system: f64, +} + +#[pymethods] +impl ProcessResourceTracker { + #[new] + #[pyo3(signature = (*, monotonic = None, resource_reader = None))] + fn new( + py: Python<'_>, + monotonic: Option>, + resource_reader: Option>, + ) -> PyResult { + let monotonic: Bound<'_, PyAny> = clock(py, monotonic)?; + let resource_reader: Bound<'_, PyAny> = match resource_reader { + Some(reader) => reader, + None => imported(py, PROCESS_RESOURCES_MODULE, "read_process_resources")?, + }; + let started_at: f64 = monotonic.call0()?.extract()?; + let (started_user, started_system, _): (f64, f64, Bound<'_, PyAny>) = + resource_reader.call0()?.extract()?; + Ok(Self { + monotonic: monotonic.unbind(), + resource_reader: resource_reader.unbind(), + started_at, + started_user, + started_system, + }) + } + + /// Return elapsed wall/CPU deltas and process peak RSS. + fn finish<'py>(&self, py: Python<'py>) -> PyResult> { + let (user_cpu, system_cpu, max_rss_bytes): (f64, f64, Bound<'py, PyAny>) = + self.resource_reader.bind(py).call0()?.extract()?; + let now: f64 = self.monotonic.bind(py).call0()?.extract()?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("wall_seconds", non_negative(now, self.started_at))?; + keywords.set_item( + "user_cpu_seconds", + non_negative(user_cpu, self.started_user), + )?; + keywords.set_item( + "system_cpu_seconds", + non_negative(system_cpu, self.started_system), + )?; + keywords.set_item("max_rss_bytes", max_rss_bytes)?; + imported(py, DIAGNOSTICS_MODELS_MODULE, "ProcessResourceUsage")?.call((), Some(&keywords)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs index 7c94d567db..5ed0391081 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -1,11 +1,17 @@ //! Setting guidance shown by diagnostics that depend on project settings. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyInt, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::project::main::explicit_references_help::explicit_references_help as references_help; use sqlbuild_config::project::main::microbatch_guidance::microbatch_guidance as concurrency_guidance; +use sqlbuild_config::project::main::setting_help_text::setting_help_text; +use sqlbuild_config::project::main::snippet_help_text::snippet_help_text; use sqlbuild_config::project::main::sql_analysis_off_guidance::sql_analysis_off_guidance as analysis_off; +use sqlbuild_config::project::models::{SettingSite, SettingText}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// The note and help for turning SQL analysis on, or `None` when it is effectively on. @@ -52,7 +58,56 @@ fn explicit_references_help(allowed: &str) -> String { references_help(allowed) } +/// A Python `bool | int | str` setting value as help text shows it. +fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(flag) = value.cast::() { + return Ok(SettingText::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(SettingText::Integer(value.str()?.extract()?)); + } + if value.is_instance_of::() { + return Ok(SettingText::Text(value.extract()?)); + } + Err(PyTypeError::new_err( + "setting value must be a bool, int or str", + )) +} + +/// Render `, set this in :` followed by the exact TOML section and key. +#[pyfunction] +#[pyo3(signature = (**options))] +fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "setting_help", + options, + (&["purpose", "file_name", "section", "key", "value"], &[]), + )?; + let text = |name: &str| -> PyResult { arguments.value(name)?.extract() }; + let (file_name, section, key): (String, String, String) = + (text("file_name")?, text("section")?, text("key")?); + Ok(setting_help_text( + &text("purpose")?, + SettingSite { + file_name: &file_name, + section: §ion, + key: &key, + }, + &setting_text(&arguments.value("value")?)?, + )) +} + +/// Render `, :` followed by the exact lines to write, one per line. +#[pyfunction] +#[pyo3(signature = (*, purpose, target, lines))] +fn snippet_help(purpose: &str, target: &str, lines: Vec) -> String { + snippet_help_text(purpose, target, &lines) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(setting_help, module)?)?; + module.add_function(wrap_pyfunction!(snippet_help, module)?)?; module.add_function(wrap_pyfunction!(sql_analysis_off_guidance, module)?)?; module.add_function(wrap_pyfunction!(microbatch_guidance, module)?)?; module.add_function(wrap_pyfunction!(explicit_references_help, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index ce6ed95913..15c2c97235 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -10,13 +10,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::rules::lint_headers::{lint_body_ranges, lint_file_role}; +use crate::bindings::_helpers::rules::lint_stop::check_stop; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild.lint.models"; const EXPANSION_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; -const STOP: &str = "sqlbuild.lint.classes.stop_context"; /// Expand a single body and finish its native SQL preparation. #[pyfunction] @@ -344,7 +344,7 @@ pub(crate) fn expand_file_bodies<'py>( &[], ), )?; - imported(py, STOP, "LintStopContext")?.call_method0("check")?; + check_stop(py)?; let path = args.value("file_path")?; let role = lint_file_role( &path, @@ -459,7 +459,7 @@ pub(crate) fn prepare_expanded_files<'py>( ) -> PyResult> { let _ = dialect; let py = expanded_files.py(); - imported(py, STOP, "LintStopContext")?.call_method0("check")?; + check_stop(py)?; let files: Vec> = expanded_files.try_iter()?.collect::>()?; let mut pending: Vec> = Vec::new(); for file in &files { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs new file mode 100644 index 0000000000..c97fbc88bb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs @@ -0,0 +1,92 @@ +//! The stop signal that lets a speculative lint run be abandoned between files. + +use std::sync::{Mutex, PoisonError}; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const STOP_VARIABLE: &str = "active_lint_stop"; + +static ACTIVE_STOP: PyOnceLock> = PyOnceLock::new(); + +/// The context variable holding the active lint run's stop event, if any. +fn active_stop_variable(py: Python<'_>) -> PyResult> { + ACTIVE_STOP + .get_or_try_init(py, || { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")? + .call((STOP_VARIABLE,), Some(&keywords)) + .map(Bound::unbind) + }) + .map(|variable| variable.bind(py).clone()) +} + +/// Abandon a speculative lint run on this thread once its owner no longer needs it. +pub(crate) fn check_stop(py: Python<'_>) -> PyResult<()> { + let stop: Bound<'_, PyAny> = active_stop_variable(py)?.call_method0("get")?; + if !stop.is_none() && stop.call_method0("is_set")?.is_truthy()? { + return Err(PyErr::from_value( + imported(py, LINT_EXCEPTIONS_MODULE, "LintStoppedError")?.call0()?, + )); + } + Ok(()) +} + +/// Makes `stop` the active lint stop event while entered. +#[pyclass(frozen, module = "sqlbuild._native")] +struct LintStopScope { + stop: Py, + token: Mutex>>, +} + +#[pymethods] +impl LintStopScope { + fn __enter__(&self, py: Python<'_>) -> PyResult<()> { + let token: Bound<'_, PyAny> = + active_stop_variable(py)?.call_method1("set", (self.stop.bind(py),))?; + *self.token.lock().unwrap_or_else(PoisonError::into_inner) = Some(token.unbind()); + Ok(()) + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self + .token + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + if let Some(token) = token { + active_stop_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// A scope that makes `stop` the active lint stop event while entered. +#[pyfunction] +fn lint_stop_scope(stop: &Bound<'_, PyAny>) -> LintStopScope { + LintStopScope { + stop: stop.clone().unbind(), + token: Mutex::new(None), + } +} + +/// The active lint run's stop event, or `None` outside a stop scope. +#[pyfunction] +fn active_lint_stop(py: Python<'_>) -> PyResult> { + active_stop_variable(py)?.call_method0("get") +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(lint_stop_scope, module)?)?; + module.add_function(wrap_pyfunction!(active_lint_stop, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index f13e0bc557..f87e3ba6ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod lint_expansion; pub(crate) mod lint_headers; pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; +pub(crate) mod lint_stop; pub(crate) mod lint_text; pub(crate) mod registration; pub(crate) mod relation_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 5abc574c40..adcb2564b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -6,8 +6,9 @@ use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, - early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_text, - results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, sql_rules, + early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_stop, + lint_text, results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, + sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -59,6 +60,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; + lint_stop::register(module)?; selection::register(module)?; results::register(module)?; guidance::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 2e32ce2c90..8c4bad037c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -17,10 +17,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; +use crate::bindings::_helpers::rules::lint_stop::check_stop; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXPANSION_MODULE: &str = "sqlbuild._native"; -const STOP_CONTEXT_MODULE: &str = "sqlbuild.lint.classes.stop_context"; const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; fn resolved(path: &Path) -> PathBuf { @@ -84,12 +84,6 @@ fn collect_lint_project_files<'py>( ) } -/// `LintStopContext.check()`: abandon a speculative run its owner no longer needs. -fn check_stop(py: Python<'_>) -> PyResult<()> { - imported(py, STOP_CONTEXT_MODULE, "LintStopContext")?.call_method0("check")?; - Ok(()) -} - /// The keyword arguments of one lint run. struct LintRun<'py> { project_dir: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 46f3b76be5..1f4e40a899 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -29,6 +29,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { analysis::sql_tests::register(module)?; sqltext::registration::register(module)?; boundary::oracles::register(module)?; + boundary::expected_errors::register(module)?; discovery::registration::register(module)?; scopes::registration::register(module)?; model_config::registration::register(module)?; @@ -52,6 +53,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cursor::registration::register(module)?; state_rows::registration::register(module)?; diagnostics::redaction::register(module)?; + diagnostics::trackers::register(module)?; Ok(()) } diff --git a/fensu.toml b/fensu.toml index 7ae6f0f6a5..70d9e82f59 100644 --- a/fensu.toml +++ b/fensu.toml @@ -93,6 +93,11 @@ rule = "FFR309" path = "src/sqlbuild/runtime/contracts/__init__.py" reason = "Native bindings open planning connections and report their lifecycle; this package only declares the connection hook models and types callers pass." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/errors/contracts/__init__.py" +reason = "Native bindings read expected-error codes, messages and help; this package only declares the shared and executor input errors callers raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 52b9296e20..10627e530c 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 13677 +remaining_lines = 13386 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8ef10a14b4..bb80faf2fc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -8,6 +8,8 @@ from datetime import date, datetime from decimal import Decimal from pathlib import Path, PurePath from typing import Any, Literal, TextIO, TypedDict +from threading import Event +from typing import Any, Literal, TextIO, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -383,6 +385,7 @@ from sqlbuild.cursor_algebra.models import ( TimestampValue, ) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar +from sqlbuild.diagnostics.models import PartialBuildPhaseTimings, ProcessResourceUsage from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildExecutionResult, @@ -3506,6 +3509,46 @@ class CollectedCompileDiagnostics: def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... def collect_compile_diagnostics() -> CollectedCompileDiagnostics: ... + +class BuildPhaseTimingTracker: + compile_seconds: float | None + planning_seconds: float | None + connection_preparation_seconds: float | None + schema_preparation_seconds: float | None + execution_seconds: float | None + cost_collection_seconds: float | None + def __init__(self, *, monotonic: Callable[[], float] | None = None) -> None: ... + def snapshot(self) -> PartialBuildPhaseTimings: ... + @classmethod + def current(cls) -> BuildPhaseTimingTracker | None: ... + def scope(self) -> PhaseTimingScope: ... + +class PhaseTimingScope: + def __enter__(self) -> None: ... + def __exit__(self, *exception: object) -> bool: ... + +class ProcessResourceTracker: + def __init__( + self, + *, + monotonic: Callable[[], float] | None = None, + resource_reader: Callable[[], tuple[float, float, int | None]] | None = None, + ) -> None: ... + def finish(self) -> ProcessResourceUsage: ... + +class LintStopScope: + def __enter__(self) -> None: ... + def __exit__(self, *exception: object) -> bool: ... + +def lint_stop_scope(stop: Event, /) -> LintStopScope: ... +def active_lint_stop() -> Event | None: ... +def error_code(*, error: BaseException, fallback_code: str) -> str: ... +def error_help(error: BaseException, /) -> str | None: ... +def error_message(error: BaseException, /) -> str: ... +def setting_help( + *, purpose: str, file_name: str, section: str, key: str, value: bool | int | str +) -> str: ... +def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: ... def open_connection_with_hooks( *, adapter: BaseAdapter, connection_config: dict[str, object], hooks: ConnectionHooks ) -> Any: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index 18f136d0d9..778eaaefb4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -7,9 +7,9 @@ from functools import partial from pathlib import Path +from sqlbuild._native import snippet_help from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider -from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from sqlbuild.spec.contracts.models import SourceLocation diff --git a/src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py b/src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py deleted file mode 100644 index dd9316c3aa..0000000000 --- a/src/sqlbuild/diagnostics/classes/build_phase_timing_tracker.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Invocation-scoped partial build timing tracker.""" - -from __future__ import annotations - -import time -from collections.abc import Callable, Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token -from typing import ClassVar - -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings - - -class BuildPhaseTimingTracker: - """Retain disjoint monotonic phase durations across exceptional exits.""" - - def __init__(self, *, monotonic: Callable[[], float] = time.monotonic) -> None: - self._monotonic: Callable[[], float] = monotonic - self._started_at: float = monotonic() - self.compile_seconds: float | None = None - self.planning_seconds: float | None = None - self.connection_preparation_seconds: float | None = None - self.schema_preparation_seconds: float | None = None - self.execution_seconds: float | None = None - self.cost_collection_seconds: float | None = None - - def snapshot(self) -> PartialBuildPhaseTimings: - """Return currently available phase durations and total wall time.""" - - return PartialBuildPhaseTimings( - compile_seconds=self.compile_seconds, - planning_seconds=self.planning_seconds, - connection_preparation_seconds=self.connection_preparation_seconds, - schema_preparation_seconds=self.schema_preparation_seconds, - execution_seconds=self.execution_seconds, - cost_collection_seconds=self.cost_collection_seconds, - total_seconds=max(0.0, self._monotonic() - self._started_at), - ) - - @classmethod - def current(cls) -> BuildPhaseTimingTracker | None: - """Return the active build timing tracker.""" - - return cls._current.get() - - @contextmanager - def scope(self) -> Iterator[None]: - """Install this tracker for one command invocation and reset it afterward.""" - - token: Token[BuildPhaseTimingTracker | None] = self._current.set(self) - try: - yield - finally: - self._current.reset(token) - - _current: ClassVar[ContextVar[BuildPhaseTimingTracker | None]] = ContextVar( - "sqlbuild_build_phase_timing_tracker", default=None - ) diff --git a/src/sqlbuild/diagnostics/classes/process_resource_tracker.py b/src/sqlbuild/diagnostics/classes/process_resource_tracker.py deleted file mode 100644 index 90fdb4a7ea..0000000000 --- a/src/sqlbuild/diagnostics/classes/process_resource_tracker.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Monotonic process resource tracker.""" - -from __future__ import annotations - -import time -from collections.abc import Callable - -from sqlbuild.diagnostics._helpers.process_resources import read_process_resources -from sqlbuild.diagnostics.models import ProcessResourceUsage - - -class ProcessResourceTracker: - """Capture wall and process resource deltas around one command.""" - - def __init__( - self, - *, - monotonic: Callable[[], float] = time.monotonic, - resource_reader: Callable[[], tuple[float, float, int | None]] = read_process_resources, - ) -> None: - self._monotonic: Callable[[], float] = monotonic - self._resource_reader: Callable[[], tuple[float, float, int | None]] = resource_reader - self._started_at: float = monotonic() - self._started_user, self._started_system, _ = resource_reader() - - def finish(self) -> ProcessResourceUsage: - """Return elapsed wall/CPU deltas and process peak RSS.""" - - user_cpu, system_cpu, max_rss_bytes = self._resource_reader() - return ProcessResourceUsage( - wall_seconds=max(0.0, self._monotonic() - self._started_at), - user_cpu_seconds=max(0.0, user_cpu - self._started_user), - system_cpu_seconds=max(0.0, system_cpu - self._started_system), - max_rss_bytes=max_rss_bytes, - ) diff --git a/src/sqlbuild/diagnostics/main/process_resource_reporting.py b/src/sqlbuild/diagnostics/main/process_resource_reporting.py index dd88f006e4..f5bcc28310 100644 --- a/src/sqlbuild/diagnostics/main/process_resource_reporting.py +++ b/src/sqlbuild/diagnostics/main/process_resource_reporting.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from contextlib import contextmanager -from sqlbuild.diagnostics.classes.process_resource_tracker import ProcessResourceTracker +from sqlbuild._native import ProcessResourceTracker from sqlbuild.diagnostics.main._log_process_resources import log_process_resources from sqlbuild.diagnostics.models import ProcessResourceUsage diff --git a/src/sqlbuild/errors/contracts/main/__init__.py b/src/sqlbuild/errors/contracts/main/__init__.py deleted file mode 100644 index 3dafe77f29..0000000000 --- a/src/sqlbuild/errors/contracts/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Expected-error contract entries.""" diff --git a/src/sqlbuild/errors/contracts/main/error_code.py b/src/sqlbuild/errors/contracts/main/error_code.py deleted file mode 100644 index 5a686c4411..0000000000 --- a/src/sqlbuild/errors/contracts/main/error_code.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Expected-error code extraction entrypoint.""" - -from __future__ import annotations - - -def error_code(*, error: BaseException, fallback_code: str) -> str: - """Return a stable code from an expected exception-like object.""" - - return str(getattr(error, "code", fallback_code)) diff --git a/src/sqlbuild/errors/contracts/main/error_help.py b/src/sqlbuild/errors/contracts/main/error_help.py deleted file mode 100644 index f30417e028..0000000000 --- a/src/sqlbuild/errors/contracts/main/error_help.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Expected-error help extraction entrypoint.""" - -from __future__ import annotations - - -def error_help(error: BaseException) -> str | None: - """Return expected-error help text when present.""" - - help_text: object | None = getattr(error, "help", None) - return help_text if isinstance(help_text, str) else None diff --git a/src/sqlbuild/errors/contracts/main/error_message.py b/src/sqlbuild/errors/contracts/main/error_message.py deleted file mode 100644 index 0685ac3a46..0000000000 --- a/src/sqlbuild/errors/contracts/main/error_message.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Expected-error message extraction entrypoint.""" - -from __future__ import annotations - - -def error_message(error: BaseException) -> str: - """Return a stable message from an expected exception-like object.""" - - return str(getattr(error, "message", str(error))) diff --git a/src/sqlbuild/errors/setting_help/__init__.py b/src/sqlbuild/errors/setting_help/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/errors/setting_help/_helpers/__init__.py b/src/sqlbuild/errors/setting_help/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/errors/setting_help/_helpers/text.py b/src/sqlbuild/errors/setting_help/_helpers/text.py deleted file mode 100644 index 3bbe56e013..0000000000 --- a/src/sqlbuild/errors/setting_help/_helpers/text.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Exact TOML and MODEL-header snippets for diagnostics that name a setting.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help.constants import ( - ADDITIONAL_HELP_SEPARATOR, - HELP_CONTINUATION_INDENT, - SETTING_SNIPPET_INDENT, -) -from sqlbuild.errors.setting_help.types import SettingValue - - -def toml_value(value: SettingValue) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, int): - return str(value) - escaped: str = value.replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' - - -def setting_snippet(*, section: str, key: str, value: SettingValue) -> str: - return "\n".join( - ( - f"{SETTING_SNIPPET_INDENT}[{section}]", - f"{SETTING_SNIPPET_INDENT}{key} = {toml_value(value)}", - ) - ) - - -def setting_help( - *, purpose: str, file_name: str, section: str, key: str, value: SettingValue -) -> str: - snippet: str = setting_snippet(section=section, key=key, value=value) - return f"{purpose}, set this in {file_name}:\n{snippet}" - - -def setting_note( - *, file_name: str, section: str, key: str, value: SettingValue, explicit: bool | None -) -> str: - rendered: str = toml_value(value) - if explicit is None: - return ( - f"the current value is [{section}] {key} = {rendered} (from {file_name} or its default)" - ) - if explicit: - return f"{file_name} sets [{section}] {key} = {rendered}" - return f"{file_name} does not set [{section}] {key}, so it defaults to {rendered}" - - -def model_header_help(*, purpose: str, entry: str, follow_up: str | None) -> str: - lines: list[str] = [ - f"{purpose}, add this to the MODEL header:", - f"{SETTING_SNIPPET_INDENT}MODEL (", - f"{SETTING_SNIPPET_INDENT} {entry},", - f"{SETTING_SNIPPET_INDENT} ...", - f"{SETTING_SNIPPET_INDENT});", - ] - if follow_up is not None: - lines.append(f"{HELP_CONTINUATION_INDENT}{follow_up}") - return "\n".join(lines) - - -def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: - snippet: str = "\n".join(f"{SETTING_SNIPPET_INDENT}{line}" for line in lines) - return f"{purpose}, {target}:\n{snippet}" - - -def join_helps(helps: tuple[str, ...]) -> str: - return ADDITIONAL_HELP_SEPARATOR.join(helps) diff --git a/src/sqlbuild/errors/setting_help/constants.py b/src/sqlbuild/errors/setting_help/constants.py deleted file mode 100644 index 216f80e559..0000000000 --- a/src/sqlbuild/errors/setting_help/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Layout of help text that shows an exact setting snippet.""" - -from __future__ import annotations - -SETTING_SNIPPET_INDENT: str = " " * 12 -HELP_CONTINUATION_INDENT: str = " " * 10 -ADDITIONAL_HELP_SEPARATOR: str = "\n = help: " diff --git a/src/sqlbuild/errors/setting_help/main/__init__.py b/src/sqlbuild/errors/setting_help/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/errors/setting_help/main/join_helps.py b/src/sqlbuild/errors/setting_help/main/join_helps.py deleted file mode 100644 index e63e7214a0..0000000000 --- a/src/sqlbuild/errors/setting_help/main/join_helps.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for joining several help paragraphs.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import join_helps as _join_helps - - -def join_helps(*helps: str) -> str: - """Join several help paragraphs into one help text, one `= help:` label per paragraph.""" - - return _join_helps(helps) diff --git a/src/sqlbuild/errors/setting_help/main/model_header_help.py b/src/sqlbuild/errors/setting_help/main/model_header_help.py deleted file mode 100644 index 8ad91217dd..0000000000 --- a/src/sqlbuild/errors/setting_help/main/model_header_help.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for help that shows an exact MODEL header entry.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import model_header_help as _model_header_help - - -def model_header_help(*, purpose: str, entry: str, follow_up: str | None = None) -> str: - """Render `, add this to the MODEL header:` followed by the exact header entry.""" - - return _model_header_help(purpose=purpose, entry=entry, follow_up=follow_up) diff --git a/src/sqlbuild/errors/setting_help/main/setting_help.py b/src/sqlbuild/errors/setting_help/main/setting_help.py deleted file mode 100644 index b802654af8..0000000000 --- a/src/sqlbuild/errors/setting_help/main/setting_help.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public entry for help that shows the exact setting to write.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import setting_help as _setting_help -from sqlbuild.errors.setting_help.types import SettingValue - - -def setting_help( - *, purpose: str, file_name: str, section: str, key: str, value: SettingValue -) -> str: - """Render `, set this in :` followed by the exact TOML section and key.""" - - return _setting_help( - purpose=purpose, file_name=file_name, section=section, key=key, value=value - ) diff --git a/src/sqlbuild/errors/setting_help/main/setting_note.py b/src/sqlbuild/errors/setting_help/main/setting_note.py deleted file mode 100644 index 3bd64b34fe..0000000000 --- a/src/sqlbuild/errors/setting_help/main/setting_note.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public entry for the current-value note of a setting.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import setting_note as _setting_note -from sqlbuild.errors.setting_help.types import SettingValue - - -def setting_note( - *, file_name: str, section: str, key: str, value: SettingValue, explicit: bool | None = True -) -> str: - """Render a setting's current value; `explicit=None` when its provenance is unknown.""" - - return _setting_note( - file_name=file_name, section=section, key=key, value=value, explicit=explicit - ) diff --git a/src/sqlbuild/errors/setting_help/main/snippet_help.py b/src/sqlbuild/errors/setting_help/main/snippet_help.py deleted file mode 100644 index 6f5025d24a..0000000000 --- a/src/sqlbuild/errors/setting_help/main/snippet_help.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for help that shows an exact authored snippet to add.""" - -from __future__ import annotations - -from sqlbuild.errors.setting_help._helpers.text import snippet_help as _snippet_help - - -def snippet_help(*, purpose: str, target: str, lines: tuple[str, ...]) -> str: - """Render `, :` followed by the exact lines to write, one per line.""" - - return _snippet_help(purpose=purpose, target=target, lines=lines) diff --git a/src/sqlbuild/errors/setting_help/types.py b/src/sqlbuild/errors/setting_help/types.py deleted file mode 100644 index f3060d3fb9..0000000000 --- a/src/sqlbuild/errors/setting_help/types.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Types for setting help text.""" - -from __future__ import annotations - -type SettingValue = bool | int | str diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index b90c1b264e..f00bba06d6 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -8,6 +8,9 @@ from sqlbuild._native import ( compute_function_definition_hash, + error_code, + error_help, + error_message, function_node_type, write_fingerprint, ) @@ -18,9 +21,6 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.constants import ( FUNCTION_EXECUTION_FAILED_CODE, diff --git a/src/sqlbuild/executor/pipeline/_helpers/connections.py b/src/sqlbuild/executor/pipeline/_helpers/connections.py index 90cc7ae2b2..adcbb3d091 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/connections.py +++ b/src/sqlbuild/executor/pipeline/_helpers/connections.py @@ -8,9 +8,9 @@ from contextvars import copy_context from typing import Any +from sqlbuild._native import BuildPhaseTimingTracker from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.executor.build.models import BuildCallbacks from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from sqlbuild.executor.pipeline.models import BuildConnectionPreparation diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index f6acce603f..60f819b29f 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -18,6 +18,9 @@ from sqlbuild._native import ( NativeProgressProjector, current_native_progress_projector, + error_code, + error_help, + error_message, scenario_local_type_overrides_for_dialect, ) from sqlbuild._native import build_cli_scenario_plan as build_scenario_plan @@ -28,9 +31,6 @@ from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, open_worker_connections, diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index 3c6faeaf0d..e2b61b7ec7 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -6,10 +6,9 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import BuildPhaseTimingTracker, CostContext from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.main._external_source_loads import ( diff --git a/src/sqlbuild/executor/run/_helpers/execution/results.py b/src/sqlbuild/executor/run/_helpers/execution/results.py index 3266c9e301..502bb15301 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/results.py +++ b/src/sqlbuild/executor/run/_helpers/execution/results.py @@ -4,7 +4,7 @@ from dataclasses import replace -from sqlbuild._native import format_waiting_on_empty_inputs +from sqlbuild._native import error_code, error_help, error_message, format_waiting_on_empty_inputs from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.planner.models import ( CursorBounds, @@ -12,9 +12,6 @@ MaximumStartSafetyEvidence, ModelPlanEntry, ) -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.constants import ( RUN_AUDIT_FAILED_CODE, diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 9843ae0a42..f4a286b14f 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -5,6 +5,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import setting_help from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -19,7 +20,6 @@ from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.setting_help.main.setting_help import setting_help from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.final_audits import run_final_model_audits from sqlbuild.executor.run._helpers.execution.hook_phases import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index d3173a0a38..90a20fb144 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import error_code, error_help, error_message from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import TablePromotionMode @@ -20,9 +21,6 @@ ScenarioRelationPlan, ) from sqlbuild.compiler.planner.types import ScenarioArtifactKind -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_help import error_help -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index b2a1233172..e24624c68a 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -5,10 +5,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import error_code, error_message from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.scenario._helpers.snapshots.core import ( is_scenario_snapshot_fresh, read_scenario_snapshot_jsonl, diff --git a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py index 3d276078dd..78e7096b66 100644 --- a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py @@ -11,6 +11,7 @@ from typing import Any from uuid import UUID +from sqlbuild._native import error_code, error_message from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, ScenarioFixturePlan, @@ -18,8 +19,6 @@ ) from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.errors.contracts.main.error_code import error_code -from sqlbuild.errors.contracts.main.error_message import error_message from sqlbuild.executor.scenario.constants import ( JSONL_BLANK_LINE, SCENARIO_EXEC_CAPTURE_LIMIT_EXCEEDED, diff --git a/src/sqlbuild/executor/scenario/main/_capture.py b/src/sqlbuild/executor/scenario/main/_capture.py index 7cd3dbbbaa..568a1e995c 100644 --- a/src/sqlbuild/executor/scenario/main/_capture.py +++ b/src/sqlbuild/executor/scenario/main/_capture.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import error_code from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.errors.contracts.main.error_code import error_code from sqlbuild.executor.scenario._helpers.capture.relation import ( capture_scenario_snapshot_relation, scenario_capture_source_relation_name, diff --git a/src/sqlbuild/lint/classes/__init__.py b/src/sqlbuild/lint/classes/__init__.py deleted file mode 100644 index 29801d9b58..0000000000 --- a/src/sqlbuild/lint/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Runtime classes for the lint and format layer.""" diff --git a/src/sqlbuild/lint/classes/stop_context.py b/src/sqlbuild/lint/classes/stop_context.py deleted file mode 100644 index 3b6215ebd6..0000000000 --- a/src/sqlbuild/lint/classes/stop_context.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Thread-local stop signal for speculative lint runs.""" - -from contextvars import ContextVar -from threading import Event -from typing import ClassVar - -from sqlbuild.lint.exceptions import LintStoppedError - - -class LintStopContext: - """Own the stop signal that lets a speculative lint run be abandoned between files.""" - - active: ClassVar[ContextVar[Event | None]] = ContextVar("active_lint_stop", default=None) - - @staticmethod - def check() -> None: - """Abandon a speculative lint run on this thread once its owner no longer needs it.""" - - stop: Event | None = LintStopContext.active.get() - if stop is not None and stop.is_set(): - raise LintStoppedError diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 32a8c500c5..7b7d3ad7e4 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -3,7 +3,6 @@ from __future__ import annotations from concurrent.futures import Executor, Future, ThreadPoolExecutor -from contextvars import Token from pathlib import Path from threading import Event @@ -11,6 +10,7 @@ build_lint_expansion_context, early_lint_expansions, finish_rule_run, + lint_stop_scope, plan_rule_run, run_lint, run_planned_sql_rules, @@ -25,7 +25,6 @@ from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules from sqlbuild.rule_engine.models import ( @@ -133,8 +132,7 @@ def _prepare_sql_lint( compiled_expansions: dict[Path, CompiledSqlExpansion], stop: Event, ) -> PreparedSqlLintResult: - stop_token: Token[Event | None] = LintStopContext.active.set(stop) - try: + with lint_stop_scope(stop): context: SqlExpansionContext = build_lint_expansion_context( project_dir=project_dir, discovered_inputs=discovered_inputs, @@ -147,6 +145,4 @@ def _prepare_sql_lint( compiled_expansions=compiled_expansions, expansion_context=context, ) - finally: - LintStopContext.active.reset(stop_token) return PreparedSqlLintResult(result=result, context=context) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 289af32bd6..158d3ee9d7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -15,12 +15,11 @@ import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module -from sqlbuild._native import rules_bulk_cache_path +from sqlbuild._native import active_lint_stop, rules_bulk_cache_path from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintRunResult from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, @@ -1037,7 +1036,7 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: lint_started.set() - stop: Event | None = LintStopContext.active.get() + stop: Event | None = active_lint_stop() stop_observed.append(stop is not None and stop.wait(timeout=_EARLY_LINT_STOP_TIMEOUT_S)) return original_run_lint(**kwargs) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 689af9cae3..23285b9f3e 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -9,13 +9,13 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import BuildPhaseTimingTracker from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.main.execution import execution from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from sqlbuild.rule_engine._helpers.run import rule_threads from sqlbuild.rule_engine.models import RulesRunResult diff --git a/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py b/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py index 692793be69..3191ef6e1a 100644 --- a/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py +++ b/tests/unit/src/sqlbuild/diagnostics/test_process_resources.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import ProcessResourceTracker from sqlbuild.diagnostics._helpers.process_resources import read_process_resources -from sqlbuild.diagnostics.classes.process_resource_tracker import ProcessResourceTracker from sqlbuild.diagnostics.models import ProcessResourceUsage from tests.unit.src.sqlbuild.diagnostics._test_types import ProcessResourceTestCase diff --git a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py index a2dd5fd97d..1c26ec50b9 100644 --- a/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py +++ b/tests/unit/src/sqlbuild/errors/setting_help/main/test_setting_help.py @@ -9,13 +9,10 @@ from sqlbuild._native import ( explicit_references_help, microbatch_guidance, + setting_help, + snippet_help, sql_analysis_opt_out_help, ) -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.model_header_help import model_header_help -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from tests.unit.src.sqlbuild.errors.setting_help.main._test_types import ( SettingCatalogueTestCase, SettingHelpTestCase, @@ -52,32 +49,15 @@ ' table_promotion_mode = "staged"', ), SettingHelpTestCase( - "an explicit note states the current value", - setting_note( + "an integer setting is bare", + setting_help( + purpose="to wait", file_name="sqlbuild_local.toml", section="settings", - key="sql_analysis", - value=False, - ), - "sqlbuild_local.toml sets [settings] sql_analysis = false", - ), - SettingHelpTestCase( - "a default note says the value is the default", - setting_note( - file_name="sqlbuild_project.toml", - section="references", - key="enforce_explicit", - value=True, - explicit=False, + key="timeout", + value=30, ), - "sqlbuild_project.toml does not set [references] enforce_explicit, " - "so it defaults to true", - ), - SettingHelpTestCase( - "a MODEL header help shows the exact entry", - model_header_help(purpose="to skip it", entry="sql_analysis false"), - "to skip it, add this to the MODEL header:\n MODEL (\n" - " sql_analysis false,\n ...\n );", + "to wait, set this in sqlbuild_local.toml:\n [settings]\n timeout = 30", ), SettingHelpTestCase( "a snippet help shows every line to write", @@ -90,11 +70,6 @@ " - name: product_types\n" " description: Product type catalogue", ), - SettingHelpTestCase( - "several helps keep one label each", - join_helps("first", "second"), - "first\n = help: second", - ), ), ids=lambda case: case.description, ) From b4df8f31fa56b276ce3d936c53d97be6b25567f9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:44:41 +0100 Subject: [PATCH 24/33] refactor: host CLI styling, terminal width and the transient line coordinator natively Expose the core CLI style, the shared transient line coordinator and the terminal width read from the native bindings, delete the Python presentation classes and helpers, the dead theme models and status constants, and the lint rule that guarded the removed terminal capability module. --- crates/sqlbuild-core/src/cli_text/models.rs | 15 ++ .../bindings/_helpers/build_progress/mod.rs | 1 + .../_helpers/build_progress/registration.rs | 4 +- .../_helpers/build_progress/terminal.rs | 15 +- .../build_progress/transient_lines.rs | 141 ++++++++++++++++++ .../src/bindings/_helpers/cli/entry.rs | 2 +- .../src/bindings/_helpers/cli/progress.rs | 2 +- .../_helpers/dbt_interop/runtime_execution.rs | 2 +- .../_helpers/dbt_interop/sqlbuild_work.rs | 2 +- .../bindings/_helpers/load/load_progress.rs | 2 +- .../_helpers/presentation_text/mod.rs | 1 + .../presentation_text/registration.rs | 3 +- .../_helpers/presentation_text/style.rs | 129 ++++++++++++++++ fensu.toml | 5 + python-boundary.toml | 2 +- scripts/fensu_policy/constants.py | 5 - .../rules/canonical_operations.py | 26 ---- src/sqlbuild/_native.pyi | 41 ++++- .../_helpers/logging_formatters.py | 2 +- .../classes/diagnostics_console_formatter.py | 2 +- .../clone/_helpers/prephase_progress.py | 2 +- .../scenario/classes/scenario_interrupts.py | 4 +- .../presentation/_helpers/__init__.py | 1 - .../_helpers/terminal_capabilities.py | 13 -- .../presentation/_helpers/transient_line.py | 13 -- src/sqlbuild/presentation/classes/__init__.py | 1 - .../presentation/classes/cli_style.py | 134 ----------------- .../classes/transient_line_coordinator.py | 65 -------- src/sqlbuild/presentation/constants.py | 6 - src/sqlbuild/presentation/main/__init__.py | 1 - .../presentation/main/terminal_columns.py | 13 -- .../main/transient_line_coordinator.py | 14 -- src/sqlbuild/presentation/models.py | 44 ------ .../rules/test_canonical_operations.py | 40 ----- .../_helpers/test_connection_progress.py | 2 +- .../shared/_helpers/test_planning_progress.py | 2 +- .../commands/shared/_helpers/test_progress.py | 8 +- .../classes/test_native_progress_projector.py | 3 +- .../_helpers/migrations/test_old_name_rows.py | 3 +- .../presentation/classes/test_cli_document.py | 3 +- .../presentation/classes/test_cli_style.py | 2 +- .../test_transient_line_coordinator.py | 2 +- .../presentation/main/test_count_header.py | 3 +- 43 files changed, 364 insertions(+), 417 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs delete mode 100644 src/sqlbuild/presentation/_helpers/__init__.py delete mode 100644 src/sqlbuild/presentation/_helpers/terminal_capabilities.py delete mode 100644 src/sqlbuild/presentation/_helpers/transient_line.py delete mode 100644 src/sqlbuild/presentation/classes/__init__.py delete mode 100644 src/sqlbuild/presentation/classes/cli_style.py delete mode 100644 src/sqlbuild/presentation/classes/transient_line_coordinator.py delete mode 100644 src/sqlbuild/presentation/main/__init__.py delete mode 100644 src/sqlbuild/presentation/main/terminal_columns.py delete mode 100644 src/sqlbuild/presentation/main/transient_line_coordinator.py diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 249fcb0745..a7edb05016 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -85,6 +85,21 @@ impl CliStyle { self.apply(BOLD, text) } + #[must_use] + pub fn dbt_object_name(self, text: &str) -> String { + self.apply("", text) + } + + #[must_use] + pub fn dbt_execution_label(self, text: &str) -> String { + self.apply(ORANGE, text) + } + + #[must_use] + pub fn log_label(self, text: &str) -> String { + self.apply(DIM, text) + } + #[must_use] pub fn success(self, text: &str) -> String { self.apply(GREEN, text) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs index e0efa09893..10bc878014 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/mod.rs @@ -6,4 +6,5 @@ pub(crate) mod plan_facts; pub(crate) mod registration; pub(crate) mod result_text; pub(crate) mod terminal; +pub(crate) mod transient_lines; pub(crate) mod transient_status; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 27abb010e4..4330892110 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, terminal, transient_status, + audit_progress, build_callbacks, footer, nested_progress, terminal, transient_lines, + transient_status, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +13,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { footer::register(module)?; nested_progress::register(module)?; terminal::register(module)?; + transient_lines::register(module)?; transient_status::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs index c0ddf2193d..ebf0e651f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs @@ -4,10 +4,10 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::build_progress::transient_lines::shared_coordinator; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const COLUMNS_MODULE: &str = "sqlbuild.presentation.main.terminal_columns"; -const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const DEFAULT_TERMINAL_SIZE: (i64, i64) = (120, 24); const SPINNER_JOIN_SECONDS: f64 = 0.2; const EXIT_JOIN_SECONDS: f64 = 1.0; const SPINNER_THREAD_NAME: &str = "sqlbuild-progress-spinner"; @@ -21,17 +21,20 @@ pub(crate) const SPINNER_FRAMES: [&str; 10] = [ "\u{2807}", "\u{280f}", ]; -/// The current terminal width. +/// The current terminal width, `shutil.get_terminal_size(fallback=(120, 24)).columns`. pub(crate) fn terminal_columns(py: Python<'_>) -> PyResult { - let columns: i64 = imported(py, COLUMNS_MODULE, "terminal_columns")? - .call0()? + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("fallback", DEFAULT_TERMINAL_SIZE)?; + let columns: i64 = imported(py, "shutil", "get_terminal_size")? + .call((), Some(&options))? + .getattr("columns")? .extract()?; Ok(usize::try_from(columns).unwrap_or(0)) } /// The process-wide coordinator of the one live transient line. pub(crate) fn shared_lines(py: Python<'_>) -> PyResult> { - imported(py, LINES_MODULE, "shared_transient_line_coordinator")?.call0() + Ok(shared_coordinator(py)?.into_any()) } /// Whether `stream` reports a terminal. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs new file mode 100644 index 0000000000..e746a615ab --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_lines.rs @@ -0,0 +1,141 @@ +//! The coordinator of one live transient terminal line and persistent writes. + +use std::sync::{Mutex, MutexGuard, PoisonError}; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::build_progress::terminal::tty_flag; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +/// The stream a transient line is drawn on and the renderer that owns it. +type ActiveLine = (Py, Py); + +static SHARED_COORDINATOR: PyOnceLock> = PyOnceLock::new(); + +/// Keep persistent terminal lines from landing on a live spinner or status line. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct TransientLineCoordinator { + /// The re-entrant lock every writer holds while it draws. + #[pyo3(get)] + lock: Py, + active: Mutex>, +} + +impl TransientLineCoordinator { + fn active(&self) -> MutexGuard<'_, Option> { + self.active.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Run `body` while holding the coordinator lock. + fn locked(&self, py: Python<'_>, body: impl FnOnce() -> PyResult) -> PyResult { + let lock: &Bound<'_, PyAny> = self.lock.bind(py); + lock.call_method0("acquire")?; + let result: PyResult = body(); + lock.call_method0("release")?; + result + } + + /// The owner whose line a write to `stream` would land on. + fn interrupted_owner<'py>( + &self, + stream: &Bound<'py, PyAny>, + ) -> PyResult>> { + let py: Python<'py> = stream.py(); + let Some((owner_stream, owner)) = self + .active() + .as_ref() + .map(|(owner_stream, owner)| (owner_stream.bind(py).clone(), owner.bind(py).clone())) + else { + return Ok(None); + }; + if owner_stream.is(stream) || (tty_flag(&owner_stream)? && tty_flag(stream)?) { + return Ok(Some(owner)); + } + Ok(None) + } +} + +#[pymethods] +impl TransientLineCoordinator { + #[new] + fn new(py: Python<'_>) -> PyResult { + Ok(Self { + lock: imported(py, "threading", "RLock")?.call0()?.unbind(), + active: Mutex::new(None), + }) + } + + /// Record the owner of the transient line currently drawn on a terminal stream. + #[pyo3(signature = (*, stream, owner))] + fn claim(&self, stream: &Bound<'_, PyAny>, owner: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(stream.py(), || { + *self.active() = Some((stream.clone().unbind(), owner.clone().unbind())); + Ok(()) + }) + } + + /// Forget a transient owner once its line is no longer drawn. + #[pyo3(signature = (*, owner))] + fn release(&self, owner: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(owner.py(), || { + let mut active = self.active(); + if active + .as_ref() + .is_some_and(|(_, current)| current.is(owner)) + { + *active = None; + } + Ok(()) + }) + } + + /// Erase and forget any live transient line, so nothing redraws it afterwards. + fn clear_active(&self, py: Python<'_>) -> PyResult<()> { + self.locked(py, || { + let taken: Option = self.active().take(); + if let Some((_, owner)) = taken { + owner.bind(py).call_method0("clear_transient_line")?; + } + Ok(()) + }) + } + + /// Write lasting text, clearing and then redrawing any live transient line. + #[pyo3(signature = (*, stream, text))] + fn write_persistent(&self, stream: &Bound<'_, PyAny>, text: &Bound<'_, PyAny>) -> PyResult<()> { + self.locked(stream.py(), || { + let owner: Option> = self.interrupted_owner(stream)?; + if let Some(owner) = &owner { + owner.call_method0("clear_transient_line")?; + } + stream.call_method1("write", (text,))?; + stream.call_method0("flush")?; + if let Some(owner) = &owner { + owner.call_method0("redraw_transient_line")?; + } + Ok(()) + }) + } +} + +/// The coordinator shared by every terminal writer in this process. +pub(crate) fn shared_coordinator(py: Python<'_>) -> PyResult> { + SHARED_COORDINATOR + .get_or_try_init(py, || Py::new(py, TransientLineCoordinator::new(py)?)) + .map(|coordinator| coordinator.bind(py).clone()) +} + +/// Return the coordinator shared by every terminal writer in this process. +#[pyfunction] +fn shared_transient_line_coordinator( + py: Python<'_>, +) -> PyResult> { + shared_coordinator(py) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(shared_transient_line_coordinator, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index b5e2c3f20d..483a944ba9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -33,7 +33,7 @@ const EXTERNAL_SCOPE_MODULE: &str = const INVOCATION_CONTEXT_MODULE: &str = "sqlbuild.runtime.output_capture.main.invocation_context"; const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exceptions"; const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; -const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const TRANSIENT_LINE_MODULE: &str = "sqlbuild._native"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs index a74d0e08c5..9348de724d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; const CONTEXT_ATTRIBUTE: &str = "_native_progress_projector_context"; const CONTEXT_NAME: &str = "sqlbuild_native_progress_projector"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COORDINATOR_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const COORDINATOR_MODULE: &str = "sqlbuild._native"; const COORDINATOR_FUNCTION: &str = "shared_transient_line_coordinator"; const CLAIM_OPTIONS: (&[&str], &[&str]) = ( &["resource_name"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs index 75905f28f0..c62b84ea12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs @@ -108,7 +108,7 @@ fn execute_dbt_commands<'py>( let argv = arguments.value("merged_argv")?; let output = arguments.value("progress_stream")?; let style = call_with( - &imported(py, "sqlbuild.presentation.classes.cli_style", "CliStyle")?, + &imported(py, "sqlbuild._native", "CliStyle")?, &[("use_color", arguments.value("use_color")?)], )?; let result_type = imported( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 2746bcd6c0..53267ead8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -116,7 +116,7 @@ fn execute_dbt_sqlbuild_work<'py>( ], )?; let style = call_with( - &imported(py, "sqlbuild.presentation.classes.cli_style", "CliStyle")?, + &imported(py, "sqlbuild._native", "CliStyle")?, &[("use_color", color.clone())], )?; emit( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs index 506e5e996c..1ae444823e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::build_progress::terminal::start_spinner; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; +const LINES_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; const LOADER_NODE_KEY: &str = "sqlbuild_loader_node"; const SPINNER_TICK_SECONDS: f64 = 0.1; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs index 42554e58ff..a3c118adf5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod document; pub(crate) mod functions; pub(crate) mod registration; +pub(crate) mod style; pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs index fd3b43e9ff..df45c7b2ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::presentation_text::{document, functions, terminal}; +use crate::bindings::_helpers::presentation_text::{document, functions, style, terminal}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { document::register(module)?; functions::register(module)?; + style::register(module)?; terminal::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs new file mode 100644 index 0000000000..9a6bb6d0b2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs @@ -0,0 +1,129 @@ +//! Semantic styling for human CLI output, as a Python class. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle as NativeStyle; + +/// Semantic CLI styling facade used by human-output formatters. +#[pyclass(name = "CliStyle", module = "sqlbuild._native", frozen)] +pub(crate) struct PythonCliStyle { + #[pyo3(get)] + use_color: bool, +} + +impl PythonCliStyle { + fn native(&self) -> NativeStyle { + NativeStyle { + use_color: self.use_color, + } + } +} + +#[pymethods] +impl PythonCliStyle { + #[new] + #[pyo3(signature = (*, use_color))] + fn new(use_color: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + use_color: use_color.is_truthy()?, + }) + } + + fn title(&self, text: &str) -> String { + self.native().title(text) + } + + fn section(&self, text: &str) -> String { + self.native().section(text) + } + + fn label(&self, text: &str) -> String { + self.native().label(text) + } + + fn value(&self, text: &str) -> String { + self.native().value(text) + } + + fn accent(&self, text: &str) -> String { + self.native().accent(text) + } + + fn accent_strong(&self, text: &str) -> String { + self.native().accent_strong(text) + } + + fn plan_section(&self, text: &str) -> String { + self.native().plan_section(text) + } + + fn object_name(&self, text: &str) -> String { + self.native().object_name(text) + } + + fn command(&self, text: &str) -> String { + self.native().command(text) + } + + fn muted(&self, text: &str) -> String { + self.native().muted(text) + } + + fn success(&self, text: &str) -> String { + self.native().success(text) + } + + fn success_strong(&self, text: &str) -> String { + self.native().success_strong(text) + } + + fn warning(&self, text: &str) -> String { + self.native().warning(text) + } + + fn warning_strong(&self, text: &str) -> String { + self.native().warning_strong(text) + } + + fn error(&self, text: &str) -> String { + self.native().error(text) + } + + fn error_strong(&self, text: &str) -> String { + self.native().error_strong(text) + } + + fn error_muted(&self, text: &str) -> String { + self.native().error_muted(text) + } + + fn log_label(&self, text: &str) -> String { + self.native().log_label(text) + } + + fn dbt_section(&self, text: &str) -> String { + self.native().dbt_section(text) + } + + fn dbt_label(&self, text: &str) -> String { + self.native().dbt_label(text) + } + + fn dbt_object_name(&self, text: &str) -> String { + self.native().dbt_object_name(text) + } + + fn dbt_execution_label(&self, text: &str) -> String { + self.native().dbt_execution_label(text) + } + + /// Style status text according to the status word. + #[pyo3(signature = (*, status, text = None))] + fn status(&self, status: &str, text: Option<&str>) -> String { + self.native().status_text(status, text.unwrap_or(status)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/fensu.toml b/fensu.toml index 70d9e82f59..1738998284 100644 --- a/fensu.toml +++ b/fensu.toml @@ -98,6 +98,11 @@ rule = "FFR309" path = "src/sqlbuild/errors/contracts/__init__.py" reason = "Native bindings read expected-error codes, messages and help; this package only declares the shared and executor input errors callers raise." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/presentation/__init__.py" +reason = "Native bindings own CLI styling, terminal width and the transient line coordinator; this package only declares the display options and presentation types callers pass." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" diff --git a/python-boundary.toml b/python-boundary.toml index 10627e530c..d723754897 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 13386 +remaining_lines = 13081 [[stays]] path = "executor" diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 085a2d85f9..950db72806 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -186,11 +186,6 @@ PROVIDER_CLASS_NAME: str = "Provider" PROVIDER_MODULE_PARTS: tuple[str, ...] = ("src", "sqlbuild", "providers.py") PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) -PUBLIC_COLOR_ENTRY_PARTS: frozenset[tuple[str, ...]] = frozenset( - { - ("src", "sqlbuild", "presentation", "main", "terminal_columns.py"), - } -) ROOT_SCOPE_NAME: str = "root" RUNTIME_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild") SELECTOR_MARKER: str = "+" diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index 8968f85d6d..de2907a050 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -19,38 +19,12 @@ NAME_REFERENCE_KIND, PLANNER_PATH_PREFIX, POLICY_EVALUATION_SCOPES, - PUBLIC_COLOR_ENTRY_PARTS, ROOT_SCOPE_NAME, SOURCE_FRESHNESS_MARKERS, SOURCE_FRESHNESS_SINGULAR_WRITER, ) -@rule( - code="XSB041", - family=Family.CUSTOM, - slug="color-capability-entry", - message="terminal capability imports must use the presentation main entry", - remediation="Import terminal_columns from sqlbuild.presentation.main.terminal_columns.", -) -def color_capability_entry(*, module: object, ctx: RuleContext) -> list[Fault]: - del module - if ctx.repo_relative_parts() in PUBLIC_COLOR_ENTRY_PARTS: - return [] - raw_module_parts: tuple[str, ...] = tuple( - "sqlbuild.presentation._helpers.terminal_capabilities".split(".") - ) - faults: list[Fault] = [] - imported: ImportFact - for imported in ctx.facts.references().imports: - imports_raw_module: bool = imported.module_parts == raw_module_parts or any( - alias.imported_parts == raw_module_parts for alias in imported.aliases - ) - if imports_raw_module: - faults.append(ctx.fault_at(location=imported.location)) - return faults - - @rule( code="XSB052", family=Family.CUSTOM, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bb80faf2fc..ed0b846a4f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,5 +1,6 @@ """Private SQLBuild native engine bindings.""" +import threading from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set from concurrent.futures import Future from contextlib import AbstractContextManager @@ -443,9 +444,8 @@ from sqlbuild.lint.models import ( RuleFixResult, ) from sqlbuild.lint.types import RelationKeyRow, RelationKeys -from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions -from sqlbuild.presentation.types import CompletionState +from sqlbuild.presentation.types import CompletionState, TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import ( @@ -3508,8 +3508,45 @@ class CollectedCompileDiagnostics: @property def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... +class CliStyle: + use_color: bool + def __init__(self, *, use_color: bool) -> None: ... + def title(self, text: str) -> str: ... + def section(self, text: str) -> str: ... + def label(self, text: str) -> str: ... + def value(self, text: str) -> str: ... + def accent(self, text: str) -> str: ... + def accent_strong(self, text: str) -> str: ... + def plan_section(self, text: str) -> str: ... + def object_name(self, text: str) -> str: ... + def command(self, text: str) -> str: ... + def muted(self, text: str) -> str: ... + def success(self, text: str) -> str: ... + def success_strong(self, text: str) -> str: ... + def warning(self, text: str) -> str: ... + def warning_strong(self, text: str) -> str: ... + def error(self, text: str) -> str: ... + def error_strong(self, text: str) -> str: ... + def error_muted(self, text: str) -> str: ... + def log_label(self, text: str) -> str: ... + def dbt_section(self, text: str) -> str: ... + def dbt_label(self, text: str) -> str: ... + def dbt_object_name(self, text: str) -> str: ... + def dbt_execution_label(self, text: str) -> str: ... + def status(self, *, status: str, text: str | None = None) -> str: ... + def collect_compile_diagnostics() -> CollectedCompileDiagnostics: ... +class TransientLineCoordinator: + lock: threading.RLock + def __init__(self) -> None: ... + def claim(self, *, stream: TextIO, owner: TransientLineOwner) -> None: ... + def release(self, *, owner: TransientLineOwner) -> None: ... + def clear_active(self) -> None: ... + def write_persistent(self, *, stream: TextIO, text: str) -> None: ... + +def shared_transient_line_coordinator() -> TransientLineCoordinator: ... + class BuildPhaseTimingTracker: compile_seconds: float | None planning_seconds: float | None diff --git a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py index 39a986cf75..5705251f17 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py +++ b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py @@ -4,7 +4,7 @@ import logging -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle _LOGGER_ROOT_NAME: str = "sqlbuild" _INLINE_SQL_TRANSACTION_STATEMENTS: tuple[str, ...] = ("BEGIN", "COMMIT", "ROLLBACK") diff --git a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py index f09c973f14..2c1a556162 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py +++ b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py @@ -4,13 +4,13 @@ import logging +from sqlbuild._native import CliStyle from sqlbuild.diagnostics._helpers.logging_formatters import ( _append_exception, _format_console_header, _get_record_sql, _render_sql_inline, ) -from sqlbuild.presentation.classes.cli_style import CliStyle _SQL_SEPARATOR: str = "-" * 80 diff --git a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py index 5276e7a13f..9ff8ce45cc 100644 --- a/src/sqlbuild/executor/clone/_helpers/prephase_progress.py +++ b/src/sqlbuild/executor/clone/_helpers/prephase_progress.py @@ -9,13 +9,13 @@ from functools import partial from typing import TextIO +from sqlbuild._native import CliStyle from sqlbuild.executor.clone.classes.prephase_progress_reporter import ( ClonePrephaseProgressReporter, ) from sqlbuild.executor.clone.classes.progress_state import CloneProgressState from sqlbuild.executor.clone.models import CloneItemResult, PrephaseProgressRow from sqlbuild.executor.clone.types import CloneAction, CloneItemCallback, CloneStatus -from sqlbuild.presentation.classes.cli_style import CliStyle _TYPE_WIDTH: int = 10 _NAME_WIDTH: int = 40 diff --git a/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py b/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py index c6a1829b3c..33874f98b6 100644 --- a/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py +++ b/src/sqlbuild/executor/scenario/classes/scenario_interrupts.py @@ -8,9 +8,7 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager -from sqlbuild.presentation.main.transient_line_coordinator import ( - shared_transient_line_coordinator, -) +from sqlbuild._native import shared_transient_line_coordinator _STOP_NOTICE: str = "Interrupted; cleaning up running scenarios...\n" _ABANDON_NOTICE: str = ( diff --git a/src/sqlbuild/presentation/_helpers/__init__.py b/src/sqlbuild/presentation/_helpers/__init__.py deleted file mode 100644 index c43f9eed92..0000000000 --- a/src/sqlbuild/presentation/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Presentation implementations.""" diff --git a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py b/src/sqlbuild/presentation/_helpers/terminal_capabilities.py deleted file mode 100644 index ec3b5e3cfa..0000000000 --- a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Terminal capability implementations.""" - -from __future__ import annotations - -import shutil - -_DEFAULT_TERMINAL_COLUMNS: int = 120 - - -def terminal_columns() -> int: - """Return the current terminal width in columns.""" - - return shutil.get_terminal_size(fallback=(_DEFAULT_TERMINAL_COLUMNS, 24)).columns diff --git a/src/sqlbuild/presentation/_helpers/transient_line.py b/src/sqlbuild/presentation/_helpers/transient_line.py deleted file mode 100644 index 7a74e98f60..0000000000 --- a/src/sqlbuild/presentation/_helpers/transient_line.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Process-wide transient terminal line coordination.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator - -_SHARED_COORDINATOR: TransientLineCoordinator = TransientLineCoordinator() - - -def shared_transient_line_coordinator() -> TransientLineCoordinator: - """Return the coordinator shared by every terminal writer in this process.""" - - return _SHARED_COORDINATOR diff --git a/src/sqlbuild/presentation/classes/__init__.py b/src/sqlbuild/presentation/classes/__init__.py deleted file mode 100644 index da0b1c0690..0000000000 --- a/src/sqlbuild/presentation/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Presentation classes.""" diff --git a/src/sqlbuild/presentation/classes/cli_style.py b/src/sqlbuild/presentation/classes/cli_style.py deleted file mode 100644 index 4981b60221..0000000000 --- a/src/sqlbuild/presentation/classes/cli_style.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Semantic styling class for human CLI output.""" - -from __future__ import annotations - -from sqlbuild.presentation.constants import ( - ERROR_STATUS_LABELS, - SKIPPED_STATUS_LABELS, - SUCCESS_STATUS_LABELS, - WARNING_STATUS_LABELS, -) -from sqlbuild.presentation.models import CliTheme, TextStyle - -_BOLD: str = "\033[1m" -_DIM: str = "\033[2m" -_RED: str = "\033[38;5;167m" -_GREEN: str = "\033[32m" -_YELLOW: str = "\033[33m" -_BLUE: str = "\033[34m" -_ORANGE: str = "\033[38;5;208m" - -default_cli_theme: CliTheme = CliTheme( - title=TextStyle(_BLUE + _BOLD), - section=TextStyle(_BOLD), - label=TextStyle(_DIM), - value=TextStyle(_BLUE), - accent=TextStyle(_BLUE), - accent_strong=TextStyle(_BLUE + _BOLD), - plan_section=TextStyle(_BOLD), - object_name=TextStyle(""), - command=TextStyle(_DIM), - success=TextStyle(_GREEN), - success_strong=TextStyle(_GREEN + _BOLD), - warning=TextStyle(_YELLOW), - warning_strong=TextStyle(_YELLOW + _BOLD), - error=TextStyle(_RED), - error_strong=TextStyle(_RED + _BOLD), - error_muted=TextStyle(_RED + _DIM), - log_label=TextStyle(_DIM), - skipped=TextStyle(_DIM), - muted=TextStyle(_DIM), - dbt_section=TextStyle(_BOLD), - dbt_label=TextStyle(_ORANGE), - dbt_object_name=TextStyle(""), - dbt_execution_label=TextStyle(_ORANGE), -) - - -class CliStyle: - """Semantic CLI styling facade used by human-output formatters.""" - - def __init__(self, *, use_color: bool, theme: CliTheme = default_cli_theme) -> None: - self.use_color: bool = use_color - self.theme: CliTheme = theme - - def title(self, text: str) -> str: - return self.theme.title.apply(text=text, use_color=self.use_color) - - def section(self, text: str) -> str: - return self.theme.section.apply(text=text, use_color=self.use_color) - - def label(self, text: str) -> str: - return self.theme.label.apply(text=text, use_color=self.use_color) - - def value(self, text: str) -> str: - return self.theme.value.apply(text=text, use_color=self.use_color) - - def accent(self, text: str) -> str: - return self.theme.accent.apply(text=text, use_color=self.use_color) - - def accent_strong(self, text: str) -> str: - return self.theme.accent_strong.apply(text=text, use_color=self.use_color) - - def plan_section(self, text: str) -> str: - return self.theme.plan_section.apply(text=text, use_color=self.use_color) - - def object_name(self, text: str) -> str: - return self.theme.object_name.apply(text=text, use_color=self.use_color) - - def command(self, text: str) -> str: - return self.theme.command.apply(text=text, use_color=self.use_color) - - def muted(self, text: str) -> str: - return self.theme.muted.apply(text=text, use_color=self.use_color) - - def success(self, text: str) -> str: - return self.theme.success.apply(text=text, use_color=self.use_color) - - def success_strong(self, text: str) -> str: - return self.theme.success_strong.apply(text=text, use_color=self.use_color) - - def warning(self, text: str) -> str: - return self.theme.warning.apply(text=text, use_color=self.use_color) - - def warning_strong(self, text: str) -> str: - return self.theme.warning_strong.apply(text=text, use_color=self.use_color) - - def error(self, text: str) -> str: - return self.theme.error.apply(text=text, use_color=self.use_color) - - def error_strong(self, text: str) -> str: - return self.theme.error_strong.apply(text=text, use_color=self.use_color) - - def error_muted(self, text: str) -> str: - return self.theme.error_muted.apply(text=text, use_color=self.use_color) - - def log_label(self, text: str) -> str: - return self.theme.log_label.apply(text=text, use_color=self.use_color) - - def dbt_section(self, text: str) -> str: - return self.theme.dbt_section.apply(text=text, use_color=self.use_color) - - def dbt_label(self, text: str) -> str: - return self.theme.dbt_label.apply(text=text, use_color=self.use_color) - - def dbt_object_name(self, text: str) -> str: - return self.theme.dbt_object_name.apply(text=text, use_color=self.use_color) - - def dbt_execution_label(self, text: str) -> str: - return self.theme.dbt_execution_label.apply(text=text, use_color=self.use_color) - - def status(self, *, status: str, text: str | None = None) -> str: - """Style status text according to the status word.""" - - rendered: str = status if text is None else text - normalized: str = status.upper() - if normalized in SUCCESS_STATUS_LABELS: - return self.theme.success.apply(text=rendered, use_color=self.use_color) - if normalized in WARNING_STATUS_LABELS: - return self.theme.warning.apply(text=rendered, use_color=self.use_color) - if normalized in ERROR_STATUS_LABELS: - return self.theme.error.apply(text=rendered, use_color=self.use_color) - if normalized in SKIPPED_STATUS_LABELS: - return self.theme.skipped.apply(text=rendered, use_color=self.use_color) - return rendered diff --git a/src/sqlbuild/presentation/classes/transient_line_coordinator.py b/src/sqlbuild/presentation/classes/transient_line_coordinator.py deleted file mode 100644 index 9975f4cf9a..0000000000 --- a/src/sqlbuild/presentation/classes/transient_line_coordinator.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Coordinator for one live transient terminal line and persistent writes.""" - -from __future__ import annotations - -import threading -from typing import TextIO - -from sqlbuild.presentation.types import TransientLineOwner - - -class TransientLineCoordinator: - """Keep persistent terminal lines from landing on a live spinner or status line.""" - - def __init__(self) -> None: - self.lock: threading.RLock = threading.RLock() - self._active: tuple[TextIO, TransientLineOwner] | None = None - - def claim(self, *, stream: TextIO, owner: TransientLineOwner) -> None: - """Record the owner of the transient line currently drawn on a terminal stream.""" - - with self.lock: - self._active = (stream, owner) - - def release(self, *, owner: TransientLineOwner) -> None: - """Forget a transient owner once its line is no longer drawn.""" - - with self.lock: - if self._active is not None and self._active[1] is owner: - self._active = None - - def clear_active(self) -> None: - """Erase and forget any live transient line, so nothing redraws it afterwards.""" - - with self.lock: - if self._active is None: - return - owner: TransientLineOwner = self._active[1] - self._active = None - owner.clear_transient_line() - - def write_persistent(self, *, stream: TextIO, text: str) -> None: - """Write lasting text, clearing and then redrawing any live transient line.""" - - with self.lock: - owner: TransientLineOwner | None = self._interrupted_owner(stream=stream) - if owner is not None: - owner.clear_transient_line() - stream.write(text) - stream.flush() - if owner is not None: - owner.redraw_transient_line() - - def _interrupted_owner(self, *, stream: TextIO) -> TransientLineOwner | None: - if self._active is None: - return None - owner_stream: TextIO - owner: TransientLineOwner - owner_stream, owner = self._active - if owner_stream is stream or (_is_tty(owner_stream) and _is_tty(stream)): - return owner - return None - - -def _is_tty(stream: TextIO) -> bool: - return hasattr(stream, "isatty") and stream.isatty() diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index b31cf9d1f9..ef8eccfe8f 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -3,9 +3,3 @@ from __future__ import annotations DEFAULT_MAX_DISPLAY_ENTRIES: int = 20 -ERROR_STATUS_LABELS: frozenset[str] = frozenset({"ERROR", "FAIL", "FAILED"}) -SKIPPED_STATUS_LABELS: frozenset[str] = frozenset({"SKIP", "SKIPPED"}) -SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) -TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" -TREE_LAST_GLYPH: str = "\u2514\u2500\u2500" -WARNING_STATUS_LABELS: frozenset[str] = frozenset({"WARN", "WARNING"}) diff --git a/src/sqlbuild/presentation/main/__init__.py b/src/sqlbuild/presentation/main/__init__.py deleted file mode 100644 index 48e5caacda..0000000000 --- a/src/sqlbuild/presentation/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Presentation function entries.""" diff --git a/src/sqlbuild/presentation/main/terminal_columns.py b/src/sqlbuild/presentation/main/terminal_columns.py deleted file mode 100644 index cbbfdd61a2..0000000000 --- a/src/sqlbuild/presentation/main/terminal_columns.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public terminal width entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.terminal_capabilities import ( - terminal_columns as _terminal_columns, -) - - -def terminal_columns() -> int: - """Return the current terminal width in columns.""" - - return _terminal_columns() diff --git a/src/sqlbuild/presentation/main/transient_line_coordinator.py b/src/sqlbuild/presentation/main/transient_line_coordinator.py deleted file mode 100644 index 8a97433a08..0000000000 --- a/src/sqlbuild/presentation/main/transient_line_coordinator.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public shared transient line coordinator entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.transient_line import ( - shared_transient_line_coordinator as _shared_transient_line_coordinator, -) -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator - - -def shared_transient_line_coordinator() -> TransientLineCoordinator: - """Return the coordinator shared by every terminal writer in this process.""" - - return _shared_transient_line_coordinator() diff --git a/src/sqlbuild/presentation/models.py b/src/sqlbuild/presentation/models.py index 18264c6368..9dc8455ead 100644 --- a/src/sqlbuild/presentation/models.py +++ b/src/sqlbuild/presentation/models.py @@ -13,47 +13,3 @@ class DisplayOptions: max_entries_per_section: int | None = DEFAULT_MAX_DISPLAY_ENTRIES overflow_flag: str = "--verbose" - - -@dataclass(frozen=True) -class TextStyle: - """One ANSI style role in the CLI theme.""" - - prefix: str - suffix: str = "\033[0m" - - def apply(self, *, text: str, use_color: bool) -> str: - """Apply this style when color is enabled.""" - - if not use_color or not self.prefix or not text: - return text - return f"{self.prefix}{text}{self.suffix}" - - -@dataclass(frozen=True) -class CliTheme: - """Semantic style roles for human CLI output.""" - - title: TextStyle - section: TextStyle - label: TextStyle - value: TextStyle - accent: TextStyle - accent_strong: TextStyle - plan_section: TextStyle - object_name: TextStyle - command: TextStyle - success: TextStyle - success_strong: TextStyle - warning: TextStyle - warning_strong: TextStyle - error: TextStyle - error_strong: TextStyle - error_muted: TextStyle - log_label: TextStyle - skipped: TextStyle - muted: TextStyle - dbt_section: TextStyle - dbt_label: TextStyle - dbt_object_name: TextStyle - dbt_execution_label: TextStyle diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index 491d20a309..3a9794b84f 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -4,7 +4,6 @@ from fensu import RuleCase, RuleResult, evaluate_rule from scripts.fensu_policy.rules.canonical_operations import ( - color_capability_entry, dbt_graph_projection, dbt_reference_resolution, selector_marker_parsing, @@ -15,45 +14,6 @@ from tests.unit.scripts.fensu_policy.rules._test_types import CustomRuleTestCase -@pytest.mark.parametrize( - "test_case", - [ - CustomRuleTestCase( - description="raw terminal capability import faults", - path="src/sqlbuild/example/main/build.py", - source=( - "from sqlbuild.presentation._helpers.terminal_capabilities import terminal_columns\n" - ), - expected_fault_count=1, - ), - CustomRuleTestCase( - description="public terminal capability import passes", - path="src/sqlbuild/example/main/build.py", - source="from sqlbuild.presentation.main.terminal_columns import terminal_columns\n", - expected_fault_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_import_when_checking_color_entry_then_matches_contract( - test_case: CustomRuleTestCase, -) -> None: - result: RuleResult = evaluate_rule( - rule=color_capability_entry, - test_case=RuleCase( - description=test_case.description, - source=test_case.source, - expected_fault_count=test_case.expected_fault_count, - path=test_case.path, - scope=test_case.scope, - scope_root=test_case.scope_root, - files=test_case.files, - ), - ) - - assert result.fault_count == test_case.expected_fault_count - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py index f39b25c692..d2daea412f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_connection_progress.py @@ -7,9 +7,9 @@ import pytest from sqlbuild._native import ( + CliStyle, ConnectionProgressReporter, ) -from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( ConnectionProgressTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py index 0813bd636d..059fdfa5e3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_planning_progress.py @@ -7,9 +7,9 @@ import pytest from sqlbuild._native import ( + CliStyle, PlanningProgressReporter, ) -from sqlbuild.presentation.classes.cli_style import CliStyle from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( PlanningFinishTestCase, PlanningProgressTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 77e77250e7..535810c1d7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -3,7 +3,6 @@ from __future__ import annotations import time -from functools import partial from io import StringIO import pytest @@ -36,7 +35,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.presentation.main import terminal_columns as terminal_columns_module from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( BuildFooterTestCase, @@ -695,11 +693,7 @@ def test_given_failed_top_level_node_when_reporting_progress_then_writes_error_d ) -> None: stream: StringIO = StringIO() monkeypatch.setattr("sys.stdout", stream) - monkeypatch.setattr( - terminal_columns_module, - "terminal_columns", - partial(int, test_case.terminal_width), - ) + monkeypatch.setenv("COLUMNS", str(test_case.terminal_width)) callbacks: BuildProgressCallbacks = BuildProgressCallbacks( plan=PlanOutput(), use_color=test_case.use_color, diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py index 8206664aec..856756ee1c 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_native_progress_projector.py @@ -8,7 +8,7 @@ import pytest -from sqlbuild._native import NativeProgressProjector +from sqlbuild._native import NativeProgressProjector, shared_transient_line_coordinator from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, @@ -18,7 +18,6 @@ invocation_scope, run_scope, ) -from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator from sqlbuild.runtime.observability.classes.operation_lifecycle import publish_retry_scheduled from tests.unit.src.sqlbuild.cli.progress.classes._test_types import ( CursorCleanupCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py index 443abdb3a2..5029617c24 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_old_name_rows.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import old_name_rows -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle, old_name_rows from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( OldNameRowsTestCase, ) diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py b/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py index 16e50370c2..b5ef96a5ff 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_cli_document.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliDocument, CliStyle from tests.unit.src.sqlbuild.presentation.classes._test_types import CliDocumentTestCase diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py b/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py index 54d7cc13f7..1658d74565 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_cli_style.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle from tests.unit.src.sqlbuild.presentation.classes._test_types import CliStyleTestCase diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py b/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py index 8cf7e30bc6..42f18c13f2 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_transient_line_coordinator.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.presentation.classes.transient_line_coordinator import TransientLineCoordinator +from sqlbuild._native import TransientLineCoordinator from tests.unit.src.sqlbuild.presentation.classes._test_types import ( TransientLineCoordinatorTestCase, ) diff --git a/tests/unit/src/sqlbuild/presentation/main/test_count_header.py b/tests/unit/src/sqlbuild/presentation/main/test_count_header.py index b3dde32fbe..93c3e01faf 100644 --- a/tests/unit/src/sqlbuild/presentation/main/test_count_header.py +++ b/tests/unit/src/sqlbuild/presentation/main/test_count_header.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import count_header_style -from sqlbuild.presentation.classes.cli_style import CliStyle +from sqlbuild._native import CliStyle, count_header_style from tests.unit.src.sqlbuild.presentation.main._test_types import CountHeaderTestCase From 7c4e981401d4b680ae9fd10366911ab8503d42f6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:46:41 +0100 Subject: [PATCH 25/33] refactor: drop empty lint packages and the dead compile context key enum --- src/sqlbuild/_native.pyi | 3 +-- src/sqlbuild/compiler/compile/types.py | 13 ------------- src/sqlbuild/lint/_helpers/__init__.py | 1 - src/sqlbuild/lint/main/__init__.py | 1 - 4 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 src/sqlbuild/lint/_helpers/__init__.py delete mode 100644 src/sqlbuild/lint/main/__init__.py diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ed0b846a4f..2bc52aa41f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -8,9 +8,8 @@ from contextvars import Token from datetime import date, datetime from decimal import Decimal from pathlib import Path, PurePath -from typing import Any, Literal, TextIO, TypedDict from threading import Event -from typing import Any, Literal, TextIO, TypedDict, overload +from typing import Any, Literal, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 01c2391b5d..38ba355622 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -614,19 +614,6 @@ def resource_type(self) -> CompiledResourceType: return CompiledResourceType(self.value) -class CompileContextKey(StrEnum): - RUN_ID = "run.id" - RUN_TARGET = "run.target" - MODEL_NAME = "model.name" - MODEL_DATABASE = "model.database" - MODEL_SCHEMA = "model.schema" - MODEL_ALIAS = "model.alias" - DESTINATION_DATABASE = "destination.database" - DESTINATION_SCHEMA = "destination.schema" - DESTINATION_TABLE = "destination.table" - DESTINATION_QUALIFIED = "destination.qualified" - - class CompiledResourceType(StrEnum): MODEL = "model" SOURCE = "source" diff --git a/src/sqlbuild/lint/_helpers/__init__.py b/src/sqlbuild/lint/_helpers/__init__.py deleted file mode 100644 index 70254bb5af..0000000000 --- a/src/sqlbuild/lint/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for the lint and format layer.""" diff --git a/src/sqlbuild/lint/main/__init__.py b/src/sqlbuild/lint/main/__init__.py deleted file mode 100644 index 43780eb795..0000000000 --- a/src/sqlbuild/lint/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Orchestration for the lint and format layer.""" From f00707d2fc06a4574d9033dd434bf8d54e51cd4b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:56:17 +0100 Subject: [PATCH 26/33] refactor: share native context variable creation --- .../src/contracts/_helpers/setting_help.rs | 2 +- .../_helpers/compile_attachment/target.rs | 2 +- .../compile_driver/diagnostic_collector.rs | 12 ++---------- .../_helpers/compile_driver/python_calls.rs | 19 ++++++++++++++++++- .../bindings/_helpers/diagnostics/trackers.rs | 12 ++---------- .../src/bindings/_helpers/rules/lint_stop.rs | 14 +++----------- 6 files changed, 27 insertions(+), 34 deletions(-) diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs index b3e3de3d5e..ef77d39392 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/setting_help.rs @@ -1,4 +1,4 @@ -//! Python's setting help texts (`sqlbuild.errors.setting_help`) for string settings. +//! Setting help texts for string settings, matching the shared native setting help. use crate::contracts::constants::{ADDITIONAL_HELP_SEPARATOR, SETTING_SNIPPET_INDENT}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 7b6c02757f..df0f86093d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -92,7 +92,7 @@ fn compile_cache_root<'py>( { return Ok(py.None().into_bound(py)); } - let root: Bound<'py, PyAny> = cache_directory(&project_dir)?; + let root: Bound<'py, PyAny> = cache_directory(project_dir)?; remove_retired_caches(&root)?; Ok(root) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs index 7d0777d104..5b151c0955 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -7,7 +7,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_collected_compile_diagnostics"; @@ -16,15 +16,7 @@ static ACTIVE_COLLECTOR: PyOnceLock> = PyOnceLock::new(); /// The context variable holding the collection diagnostics are reported into, if any. fn active_collector(py: Python<'_>) -> PyResult> { - ACTIVE_COLLECTOR - .get_or_try_init(py, || { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("default", py.None())?; - imported(py, "contextvars", "ContextVar")? - .call((CONTEXT_VARIABLE_NAME,), Some(&keywords)) - .map(Bound::unbind) - }) - .map(|variable| variable.bind(py).clone()) + context_variable(py, &ACTIVE_COLLECTOR, CONTEXT_VARIABLE_NAME) } /// Deduplicated compile diagnostics gathered while the collection is entered. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs index 1a6c91e380..7d9561859a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_calls.rs @@ -1,7 +1,8 @@ //! Calling the Python stages the native driver still runs: imports, keyword calls and `with`. use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; /// `module.name`, imported. @@ -142,3 +143,19 @@ impl<'py> KeywordArguments<'py> { self.values.contains(name) } } + +/// The `contextvars.ContextVar(name, default=None)` kept in `cell`, created once per interpreter. +pub(crate) fn context_variable<'py>( + py: Python<'py>, + cell: &'static PyOnceLock>, + name: &str, +) -> PyResult> { + cell.get_or_try_init(py, || { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + imported(py, "contextvars", "ContextVar")? + .call((name,), Some(&keywords)) + .map(Bound::unbind) + }) + .map(|variable| variable.bind(py).clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs index 1682c57e86..33f860fdac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -7,7 +7,7 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyType}; use pyo3::{pyclass, pymethods}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild.diagnostics.models"; const PROCESS_RESOURCES_MODULE: &str = "sqlbuild.diagnostics._helpers.process_resources"; @@ -25,15 +25,7 @@ static CURRENT_TRACKER: PyOnceLock> = PyOnceLock::new(); /// The context variable holding the active build timing tracker. fn current_variable(py: Python<'_>) -> PyResult> { - CURRENT_TRACKER - .get_or_try_init(py, || { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("default", py.None())?; - imported(py, "contextvars", "ContextVar")? - .call((TRACKER_VARIABLE,), Some(&keywords)) - .map(Bound::unbind) - }) - .map(|variable| variable.bind(py).clone()) + context_variable(py, &CURRENT_TRACKER, TRACKER_VARIABLE) } /// `monotonic`, or `time.monotonic` when none is given. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs index c97fbc88bb..2dd45f982e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs @@ -6,10 +6,10 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::PyTuple; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; const STOP_VARIABLE: &str = "active_lint_stop"; @@ -18,15 +18,7 @@ static ACTIVE_STOP: PyOnceLock> = PyOnceLock::new(); /// The context variable holding the active lint run's stop event, if any. fn active_stop_variable(py: Python<'_>) -> PyResult> { - ACTIVE_STOP - .get_or_try_init(py, || { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("default", py.None())?; - imported(py, "contextvars", "ContextVar")? - .call((STOP_VARIABLE,), Some(&keywords)) - .map(Bound::unbind) - }) - .map(|variable| variable.bind(py).clone()) + context_variable(py, &ACTIVE_STOP, STOP_VARIABLE) } /// Abandon a speculative lint run on this thread once its owner no longer needs it. From 8deb23d67125c3da67a6c43097fe5248db37bb56 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:16:48 +0100 Subject: [PATCH 27/33] test: observe planning connections through the adapter in the rules pipeline test --- .../src/sqlbuild/compiler/pipeline/test_main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 047e73d94f..66bd89520c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -22,7 +22,6 @@ from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.runtime.contracts.main import open_connection as compile_pipeline from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, CompileProgressIntegrationTestCase, @@ -141,13 +140,14 @@ def test_given_rule_finding_when_running_shared_compile_pipeline_then_planning_i 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n', encoding="utf-8", ) - open_connection: Mock = Mock() - monkeypatch.setattr(compile_pipeline, "open_connection_with_hooks", open_connection) + adapter: DuckDbAdapter = DuckDbAdapter() + connect: Mock = Mock() + monkeypatch.setattr(adapter, "connect", connect) with pytest.raises(RulesError, match=test_case.expected_error_pattern): - _ = run_compile_pipeline_for_project(project_dir=tmp_path, adapter=DuckDbAdapter()) + _ = run_compile_pipeline_for_project(project_dir=tmp_path, adapter=adapter) - assert open_connection.call_count == test_case.expected_connection_calls + assert connect.call_count == test_case.expected_connection_calls @pytest.mark.parametrize( From 738751d6e8682103e71aa815e5c29a863a0eb5d2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:32:44 +0100 Subject: [PATCH 28/33] chore: reconcile integrated Phase 1 boundary --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index d723754897..ddce5dfd34 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 13081 +remaining_lines = 13066 [[stays]] path = "executor" From ad929a9d1109e62a2787749b18b186d8f1ff0e70 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:41:15 +0100 Subject: [PATCH 29/33] test: patch the native execution plan in phase timing tests --- .../compiler/pipeline/main/test_compile_phase_timings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 23285b9f3e..87815a8441 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -14,7 +14,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult -from sqlbuild.compiler.planner.main.execution import execution from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from sqlbuild.rule_engine._helpers.run import rule_threads @@ -66,7 +65,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di adapter: Mock = Mock() monkeypatch.setattr(time, "monotonic", monotonic) monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) - monkeypatch.setattr(execution, "build_execution_plan", Mock(return_value=PlanOutput())) + monkeypatch.setattr(_native, "build_execution_plan", Mock(return_value=PlanOutput())) monkeypatch.setattr(materializations, "load_custom_materializations", Mock(return_value={})) monkeypatch.setattr( _native, From 2df9aee8c06eece9f98022df0cb40d6a0f72e684 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:46:15 +0100 Subject: [PATCH 30/33] test: drop scan entries for modules moved to Rust --- scripts/compiler_differential/constants.py | 1 - .../sqlbuild/adapter/relations/test_metadata_sql_ownership.py | 1 - 2 files changed, 2 deletions(-) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 9207da6f01..d05df9c59f 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -658,7 +658,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ), } ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( - "compile/_helpers/analysis", "compile/_helpers/diagnostics", "contracts", "sql_analysis", diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index f762fe4391..1375a23885 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -23,7 +23,6 @@ "adapters/snowflake/classes/snowflake_adapter.py", "adapters/sqlserver/classes/sqlserver_adapter.py", "adapters/sqlserver/constants.py", - "cost/_helpers/snowflake.py", "cost/models.py", } ) From 48252ab465fa4c41944117057432bab4681f2a72 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:31:55 +0100 Subject: [PATCH 31/33] fix: report connection errors only for ordinary exceptions --- .../_helpers/compile_driver/plan_result.rs | 3 ++- .../sqlbuild/runtime/contracts/_test_types.py | 6 +++--- .../src/sqlbuild/runtime/contracts/helpers.py | 2 +- .../runtime/contracts/test_open_connection.py | 19 ++++++++++++++----- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 61cccc946f..2b188c660f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -1,5 +1,6 @@ //! Plan a compiled project over an open warehouse connection, and open that connection. +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -254,7 +255,7 @@ pub(crate) fn connection_with_hooks<'py>( Ok(connection) => connection, Err(error) => { let on_error: Bound<'py, PyAny> = hooks.getattr("on_connection_error")?; - if !on_error.is_none() { + if error.is_instance_of::(hooks.py()) && !on_error.is_none() { on_error.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; } return Err(error); diff --git a/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py b/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py index 5a55b6c11e..7b431a5856 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/_test_types.py @@ -16,7 +16,7 @@ class OpenConnectionFailureTestCase: description: str connection_config: dict[str, object] monotonic_times: tuple[float, float] - expected_error: Exception + expected_error: BaseException expected_event_order: tuple[str, ...] - expected_elapsed_seconds: float - expected_connection_count: int + expected_elapsed_seconds: tuple[float, ...] + expected_connection_counts: tuple[int, ...] diff --git a/tests/unit/src/sqlbuild/runtime/contracts/helpers.py b/tests/unit/src/sqlbuild/runtime/contracts/helpers.py index 36d233e974..d83ac1b730 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/helpers.py @@ -29,7 +29,7 @@ def _execute(self, connection: Any, sql: str) -> object: class FailingConnectionAdapter(BaseAdapter): adapter_name: ClassVar[str] = "failing-connection-test" - def __init__(self, *, events: list[str], error: Exception) -> None: + def __init__(self, *, events: list[str], error: BaseException) -> None: self.events = events self.error = error self.connected_config: dict[str, object] | None = None diff --git a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py index aa03c92f52..5e8083d0e7 100644 --- a/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py +++ b/tests/unit/src/sqlbuild/runtime/contracts/test_open_connection.py @@ -80,9 +80,18 @@ def on_complete(connection_count: int, *, elapsed_seconds: int | float) -> None: monotonic_times=(20.0, 21.25), expected_error=RuntimeError("connection unavailable"), expected_event_order=("start", "connect", "error"), - expected_elapsed_seconds=1.25, - expected_connection_count=1, - ) + expected_elapsed_seconds=(1.25,), + expected_connection_counts=(1, 1), + ), + OpenConnectionFailureTestCase( + description="interrupted connection propagates without an error event", + connection_config={"database": "unavailable.duckdb"}, + monotonic_times=(20.0, 21.25), + expected_error=KeyboardInterrupt(), + expected_event_order=("start", "connect"), + expected_elapsed_seconds=(), + expected_connection_counts=(1,), + ), ], ids=lambda case: case.description, ) @@ -122,5 +131,5 @@ def on_error(connection_count: int, *, elapsed_seconds: int | float) -> None: assert exc_info.value is test_case.expected_error assert adapter.connected_config is test_case.connection_config assert tuple(events) == test_case.expected_event_order - assert elapsed_seconds_values == [test_case.expected_elapsed_seconds] - assert connection_counts == [test_case.expected_connection_count] * 2 + assert tuple(elapsed_seconds_values) == test_case.expected_elapsed_seconds + assert tuple(connection_counts) == test_case.expected_connection_counts From 9e277eb3e30bdc8fe3b97c3bbb3ac8cbca592065 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:28:34 +0100 Subject: [PATCH 32/33] test: keep only asserted fields from compile benchmark payloads --- .../_helpers/varied_benchmark.py | 15 +++++++++++++-- .../sqlbuild/cli/commands/main/compile/helpers.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index 7fe7dd708c..651959d413 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -11,6 +11,7 @@ from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + _RETAINED_BENCHMARK_PAYLOAD_KEYS, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, ) @@ -23,7 +24,10 @@ def run_varied_cache_benchmark( _assert_complete_rules(project_dir) measurements: dict[str, FreshProcessCompileBenchmarkResult] = {} measurements["cold"] = _measure( - project_dir=project_dir, label="cold", budget=test_case.expected_cold_max_seconds + project_dir=project_dir, + label="cold", + budget=test_case.expected_cold_max_seconds, + retained_payload_keys=(*_RETAINED_BENCHMARK_PAYLOAD_KEYS, "resources"), ) measurements["warm"] = _measure( project_dir=project_dir, label="warm", budget=test_case.expected_warm_max_seconds @@ -109,10 +113,17 @@ def _measure_edit( return measurements -def _measure(*, project_dir: Path, label: str, budget: float) -> FreshProcessCompileBenchmarkResult: +def _measure( + *, + project_dir: Path, + label: str, + budget: float, + retained_payload_keys: tuple[str, ...] = _RETAINED_BENCHMARK_PAYLOAD_KEYS, +) -> FreshProcessCompileBenchmarkResult: return _run_fresh_process_compile_benchmark( project_dir=project_dir, label=f"varied-{label}", expected_max_wall_seconds=budget, compile_args=(), + retained_payload_keys=retained_payload_keys, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 3bde194cc4..15334996b8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -161,6 +161,14 @@ class SemanticCompileBenchmarkResult(NamedTuple): warm: CompileBenchmarkMeasurement +_RETAINED_BENCHMARK_PAYLOAD_KEYS: tuple[str, ...] = ( + "compile_timings", + "has_errors", + "diagnostics", + "summary", +) + + class FreshProcessCompileBenchmarkResult(NamedTuple): elapsed_seconds: float peak_rss_bytes: int @@ -660,6 +668,7 @@ def _run_fresh_process_compile_benchmark( label: str, expected_max_wall_seconds: float, compile_args: tuple[str, ...], + retained_payload_keys: tuple[str, ...] = _RETAINED_BENCHMARK_PAYLOAD_KEYS, ) -> FreshProcessCompileBenchmarkResult: run: _TimedSqbRun = _run_timed_sqb( project_dir=project_dir, @@ -677,7 +686,7 @@ def _run_fresh_process_compile_benchmark( elapsed_seconds=run.elapsed_seconds, peak_rss_bytes=run.peak_rss_bytes, semantic_fingerprint=semantic_fingerprint, - payload=payload, + payload={key: payload.get(key) for key in retained_payload_keys}, cpu_seconds=run.cpu_seconds, major_page_faults=run.major_page_faults, minor_page_faults=run.minor_page_faults, From d026f24d9b0127e56d3105a1a4cd9bfdd6528bad Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:21:14 +0100 Subject: [PATCH 33/33] test: guard that compiles after a project config edit keep unchanged artifacts --- .../sqlbuild/cli/commands/main/_test_types.py | 9 +++++ .../commands/main/test_compile_warm_target.py | 40 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index d4cc7e7c3b..8e0a72a9d2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -939,6 +939,15 @@ class WarmTargetEditTestCase: expected_exit_code: int = 0 +@dataclass(frozen=True) +class WarmTargetProjectConfigEditTestCase: + description: str + edited_project_config: str + expected_edit_changed: tuple[str, ...] = () + expected_after_edit_changed: tuple[str, ...] = () + expected_exit_codes: tuple[int, int] = (0, 0) + + @dataclass(frozen=True) class RuleExceptionPolicyTestCase: """One `[rules] allow_exceptions` value with one escape hatch, compiled through the CLI.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py index 6af6a5fbb1..45dc6dac01 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_warm_target.py @@ -17,6 +17,7 @@ ) from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( WarmTargetEditTestCase, + WarmTargetProjectConfigEditTestCase, WarmTargetTestCase, ) @@ -96,5 +97,44 @@ def test_given_one_edited_model_when_compiling_warm_then_rewrites_only_its_artif ) +@pytest.mark.parametrize( + "test_case", + [ + WarmTargetProjectConfigEditTestCase( + "unused_var_added", + edited_project_config=('name = "shop"\nadapter = "duckdb"\n\n[vars]\nrevision = "1"\n'), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_config_edit_when_compiling_twice_then_rewrites_no_unchanged_artifact( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + test_case: WarmTargetProjectConfigEditTestCase, +) -> None: + write_warm_target_project(project_dir=tmp_path) + _ = compile_project(project_dir=tmp_path, compile_args=()) + compiled_dir: Path = tmp_path / "target" / "compiled" + stamp_artifacts( + artifacts=compiled_artifacts(compiled_dir=compiled_dir), mtime_ns=_UNCHANGED_MTIME_NS + ) + (tmp_path / "sqlbuild_project.toml").write_text( + test_case.edited_project_config, encoding="utf-8" + ) + + edit_exit_code: int = compile_project(project_dir=tmp_path, compile_args=()) + edit_changed: tuple[str, ...] = changed_artifacts( + compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS + ) + after_edit_exit_code: int = compile_project(project_dir=tmp_path, compile_args=()) + _ = capsys.readouterr() + + assert (edit_exit_code, after_edit_exit_code) == test_case.expected_exit_codes + assert edit_changed == test_case.expected_edit_changed + assert changed_artifacts(compiled_dir=compiled_dir, mtime_ns=_UNCHANGED_MTIME_NS) == ( + test_case.expected_after_edit_changed + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"])