diff --git a/Cargo.lock b/Cargo.lock index dbfac92b5c..0882c97e86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,6 +1213,7 @@ name = "sqlbuild-core" version = "0.156.1" dependencies = [ "regex", + "sha2", ] [[package]] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs index 6d937aa4e6..3fff43f207 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/cte_facts.rs @@ -32,6 +32,9 @@ use crate::assembly::analysis_session::constants::{ use crate::assembly::analysis_session::models::NullabilityCallback; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; +use crate::nullability_rules::main::first_arg_nullability::first_arg_nullability; +use crate::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::{NormalizedType, TypeFamily}; @@ -1020,36 +1023,13 @@ fn combined(node_kind: &str, rule: Option<&str>, arguments: &[&'static str]) -> return UNKNOWN_NULLABILITY; } match rule { - Some(FIRST_ARG_RULE) => arguments.first().copied().unwrap_or(UNKNOWN_NULLABILITY), - Some(CONDITIONAL_RESULT_RULE) => conditional_result(arguments), - Some(SAFE_CAST_RULE) => safe_cast(arguments), + Some(FIRST_ARG_RULE) => first_arg_nullability(arguments), + Some(CONDITIONAL_RESULT_RULE) => conditional_result_nullability(arguments), + Some(SAFE_CAST_RULE) => safe_cast_nullability(arguments), _ => UNKNOWN_NULLABILITY, } } -/// Python's `safe_cast_nullability`. -fn safe_cast(arguments: &[&'static str]) -> &'static str { - if arguments.first() == Some(&NULLABLE_NULLABILITY) { - NULLABLE_NULLABILITY - } else { - UNKNOWN_NULLABILITY - } -} - -/// Python's `conditional_result_nullability`. -fn conditional_result(arguments: &[&'static str]) -> &'static str { - let [_, when_true, when_false, ..] = arguments else { - return UNKNOWN_NULLABILITY; - }; - if *when_true == NON_NULL_NULLABILITY && *when_false == NON_NULL_NULLABILITY { - return NON_NULL_NULLABILITY; - } - if *when_true == NULLABLE_NULLABILITY || *when_false == NULLABLE_NULLABILITY { - return NULLABLE_NULLABILITY; - } - UNKNOWN_NULLABILITY -} - /// Python's `_infer_polyglot_column_nullability`. fn column_nullability( column: &Expression, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index d1589b8930..b37e6c4e4b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -13,15 +13,15 @@ use crate::assembly::analysis_session::_helpers::dict_walk::{ parse_one_or_error, payload, relation_name, render_type, to_dict, truthy, upper, }; use crate::assembly::analysis_session::constants::{ - ALIAS_AST_KIND, CAST_AST_KIND, CTE_AST_KIND, DUCKDB_DIALECT, DYNAMIC_VALUE_SOURCE_KINDS, - MOTHERDUCK_DIALECT, PIVOT_AST_KIND, QUERY_AST_KINDS, SELECT_AST_KIND, - SIMPLIFIED_PIVOT_DIALECTS, SUPPORTED_PIVOT_DIALECTS, TYPE_PRESERVING_AGGREGATES, - UNKNOWN_NULLABILITY, UNKNOWN_TYPE, + ALIAS_AST_KIND, CAST_AST_KIND, DUCKDB_DIALECT, DYNAMIC_VALUE_SOURCE_KINDS, MOTHERDUCK_DIALECT, + PIVOT_AST_KIND, QUERY_AST_KINDS, SELECT_AST_KIND, SIMPLIFIED_PIVOT_DIALECTS, + SUPPORTED_PIVOT_DIALECTS, TYPE_PRESERVING_AGGREGATES, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, }; use crate::assembly::analysis_session::models::{ ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables, }; use crate::assembly::analysis_session::types::Shapes; +use sqlbuild_core::constants::CTE_AST_KIND; /// What an internal native failure of a pivot proof names. const PIVOT_CONTEXT: &str = "native dynamic pivot proof"; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index c9a57102f7..2b786482a0 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -72,7 +72,6 @@ pub(crate) const COLUMN_AST_KIND: &str = "column"; pub(crate) const TABLE_AST_KIND: &str = "table"; pub(crate) const FUNCTION_AST_KIND: &str = "function"; pub(crate) const ALIAS_AST_KIND: &str = "alias"; -pub(crate) const CTE_AST_KIND: &str = "cte"; pub(crate) const STAR_AST_KIND: &str = "star"; pub(crate) const DYNAMIC_VALUE_SOURCE_KINDS: [&str; 3] = ["pivot_any", "select", "query"]; pub(crate) const DEPENDENCY_FUNCTION_NAMES: [&str; 3] = ["__ref", "__source", "__seed"]; diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs index a445125ab0..270450768e 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/promotion.rs @@ -5,10 +5,12 @@ use crate::contracts::_helpers::setting_help::{ }; use crate::contracts::constants::{ CONTRACT_ENFORCED, IMMEDIATE_PROMOTION_MODE, MICROBATCH_INCREMENTAL_MODE, - PROMOTION_CONFLICT_CODE, PROMOTION_REMOVE_CONTRACT_HELP, SETTINGS_SECTION, - STAGED_LIFECYCLE_MATERIALIZATIONS, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, + PROMOTION_CONFLICT_CODE, PROMOTION_REMOVE_CONTRACT_HELP, STAGED_LIFECYCLE_MATERIALIZATIONS, }; use crate::contracts::models::{PromotionConflict, PromotionModel, PromotionRequest}; +use sqlbuild_core::constants::SETTINGS_SECTION; +use sqlbuild_core::constants::STAGED_PROMOTION_MODE; +use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; /// One conflict per enforced-contract table model when the effective promotion is immediate. pub(crate) fn promotion_conflicts(request: &PromotionRequest) -> Vec { diff --git a/crates/sqlbuild-analysis/src/contracts/constants.rs b/crates/sqlbuild-analysis/src/contracts/constants.rs index 112d6458f2..26e6e274b7 100644 --- a/crates/sqlbuild-analysis/src/contracts/constants.rs +++ b/crates/sqlbuild-analysis/src/contracts/constants.rs @@ -9,14 +9,11 @@ pub(crate) const MISSING_DECLARATIONS_CODE: &str = "K006"; pub(crate) const DYNAMIC_OUTPUT_NOT_PROVEN_CODE: &str = "K011"; pub(crate) const PROMOTION_CONFLICT_CODE: &str = "K011"; -pub(crate) const CONTRACT_ENFORCED: &str = "enforced"; +pub const CONTRACT_ENFORCED: &str = "enforced"; pub(crate) const CONTRACT_NONE: &str = "none"; pub(crate) const STAGED_LIFECYCLE_MATERIALIZATIONS: [&str; 2] = ["table", "incremental"]; pub(crate) const MICROBATCH_INCREMENTAL_MODE: &str = "microbatch"; pub(crate) const IMMEDIATE_PROMOTION_MODE: &str = "immediate"; -pub(crate) const STAGED_PROMOTION_MODE: &str = "staged"; -pub(crate) const SETTINGS_SECTION: &str = "settings"; -pub(crate) const TABLE_PROMOTION_MODE_SETTING_KEY: &str = "table_promotion_mode"; pub(crate) const NO_COMPILER_EVIDENCE: &str = "no compiler evidence"; pub(crate) const DYNAMIC_OUTPUT_NOT_PROVEN_HELP: &str = "project one supported runtime-dynamic pivot through a wildcard output boundary and declare every generated family"; diff --git a/crates/sqlbuild-analysis/src/contracts/mod.rs b/crates/sqlbuild-analysis/src/contracts/mod.rs index 4ca56896ce..5fb131efc2 100644 --- a/crates/sqlbuild-analysis/src/contracts/mod.rs +++ b/crates/sqlbuild-analysis/src/contracts/mod.rs @@ -1,7 +1,7 @@ //! Model column contracts and promotion conflicts. pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs index 8d9c11ec79..6d4d2d0a6c 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod checks; pub(crate) mod edges; pub(crate) mod nodes; +pub(crate) mod projection; pub(crate) mod selection; pub(crate) mod values; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs new file mode 100644 index 0000000000..8860bcf778 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/_helpers/projection.rs @@ -0,0 +1,27 @@ +//! Node lookups the orchestrator projections share. + +use std::collections::HashMap; + +use crate::dag_artifact::models::DagProjectionGraph; + +/// Node index by id; a later node with a repeated id replaces the earlier one. +pub(crate) fn nodes_by_id(graph: &DagProjectionGraph) -> HashMap<&str, usize> { + graph + .nodes + .iter() + .enumerate() + .map(|(index, node)| (node.id.as_str(), index)) + .collect() +} + +/// Upstream node ids by downstream node id, in edge order. +pub(crate) fn upstream_by_id(graph: &DagProjectionGraph) -> HashMap<&str, Vec<&str>> { + let mut upstream: HashMap<&str, Vec<&str>> = HashMap::new(); + for (from_id, to_id) in &graph.edges { + upstream + .entry(to_id.as_str()) + .or_default() + .push(from_id.as_str()); + } + upstream +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs index 77a41ef817..5de360110e 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/constants.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/constants.rs @@ -34,3 +34,5 @@ pub(crate) const SELECTOR_KIND_CHECK: &str = "check"; pub(crate) const SELECTOR_KIND_TEST: &str = "test"; pub(crate) const SELECTOR_KIND_SEPARATOR: &str = ":"; pub(crate) const SELECTOR_MODELS_PREFIX: &str = "models/"; +/// The orchestrator kind every SQLBuild asset carries. +pub(crate) const SQLBUILD_ASSET_KIND: &str = "sqlbuild"; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs new file mode 100644 index 0000000000..d187feb77d --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_plans.rs @@ -0,0 +1,37 @@ +//! The asset nodes an orchestrator definition owns and their dependencies. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::projection::{nodes_by_id, upstream_by_id}; +use crate::dag_artifact::models::{AssetNodePlan, AssetProjectionPolicy, DagProjectionGraph}; + +/// Each node of an asset kind, in artifact order, with its upstream nodes in edge order. +#[must_use] +pub fn dag_asset_plans( + graph: &DagProjectionGraph, + policy: &AssetProjectionPolicy, +) -> Vec { + let nodes: HashMap<&str, usize> = nodes_by_id(graph); + let upstream: HashMap<&str, Vec<&str>> = upstream_by_id(graph); + let is_asset = |index: usize| policy.asset_kinds.contains(&graph.nodes[index].kind); + let mut plans: Vec = Vec::new(); + for (index, node) in graph.nodes.iter().enumerate() { + if !is_asset(index) { + continue; + } + let mut dependencies: Vec = Vec::new(); + for upstream_id in upstream.get(node.id.as_str()).into_iter().flatten() { + match nodes.get(upstream_id) { + Some(dependency) if !policy.upstream_asset_kinds_only || is_asset(*dependency) => { + dependencies.push(*dependency); + } + _ => {} + } + } + plans.push(AssetNodePlan { + node: index, + upstream: dependencies, + }); + } + plans +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs new file mode 100644 index 0000000000..e7bf3c9116 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_asset_spec_kinds.rs @@ -0,0 +1,27 @@ +//! The orchestrator kinds of one asset. + +use crate::dag_artifact::constants::SQLBUILD_ASSET_KIND; +use crate::dag_artifact::models::AssetSpecKindPolicy; + +/// `sqlbuild` plus `view`/`table` for models, the node kind for direct kinds, else nothing more. +#[must_use] +pub fn dag_asset_spec_kinds( + (kind, materialization_type): (&str, &str), + policy: &AssetSpecKindPolicy, +) -> Vec { + let detail: Option<&str> = if kind == policy.model_kind { + Some(if materialization_type == policy.view_materialization { + "view" + } else { + "table" + }) + } else if policy.direct_kinds.contains(kind) { + Some(kind) + } else { + None + }; + std::iter::once(SQLBUILD_ASSET_KIND) + .chain(detail) + .map(str::to_owned) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs new file mode 100644 index 0000000000..540a8b0201 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_check_plans.rs @@ -0,0 +1,42 @@ +//! The checks an orchestrator check definition exposes and the nodes they check. + +use std::collections::HashMap; + +use crate::dag_artifact::_helpers::projection::nodes_by_id; +use crate::dag_artifact::models::{ + CheckKindFilter, CheckPlan, DagProjectionCheck, DagProjectionGraph, +}; + +/// Whether `filter` exposes checks of `kind`. +fn exposed(filter: &CheckKindFilter, kind: &str) -> bool { + !filter.excluded.contains(kind) + && filter + .required + .as_ref() + .is_none_or(|required| required.contains(kind)) +} + +/// The indexes of the known nodes `check` names, in its order. +fn checked_nodes(nodes: &HashMap<&str, usize>, check: &DagProjectionCheck) -> Vec { + check + .checked_asset_ids + .iter() + .filter_map(|id| nodes.get(id.as_str()).copied()) + .collect() +} + +/// Every check in artifact order, whether `filter` exposes its kind, and its known checked nodes. +#[must_use] +pub fn dag_check_plans(graph: &DagProjectionGraph, filter: &CheckKindFilter) -> Vec { + let nodes: HashMap<&str, usize> = nodes_by_id(graph); + graph + .checks + .iter() + .enumerate() + .map(|(index, check)| CheckPlan { + check: index, + included: exposed(filter, &check.kind), + nodes: checked_nodes(&nodes, check), + }) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs new file mode 100644 index 0000000000..db335097e7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/dag_selected_asset_nodes.rs @@ -0,0 +1,21 @@ +//! The selected nodes an orchestrator owns as assets. + +use std::collections::BTreeSet; + +use crate::dag_artifact::models::DagProjectionGraph; + +/// Indexes of nodes whose id is selected and whose kind is an asset kind, in artifact order. +#[must_use] +pub fn dag_selected_asset_nodes( + graph: &DagProjectionGraph, + selected_ids: &BTreeSet, + asset_kinds: &BTreeSet, +) -> Vec { + graph + .nodes + .iter() + .enumerate() + .filter(|(_, node)| selected_ids.contains(&node.id) && asset_kinds.contains(&node.kind)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs index 80514c0a8a..2d2ed30464 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/main/mod.rs @@ -1,2 +1,6 @@ pub mod build_dag_json; +pub mod dag_asset_plans; +pub mod dag_asset_spec_kinds; +pub mod dag_check_plans; +pub mod dag_selected_asset_nodes; pub mod resolve_dag_selection; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/models.rs b/crates/sqlbuild-analysis/src/dag_artifact/models.rs index 7ec5cf89a3..5e4fdee3a9 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/models.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/models.rs @@ -1,5 +1,7 @@ //! The compiled project, Python node graph and authored values the DAG artifact is built from. +use std::collections::BTreeSet; + use crate::assembly::project::types::ObjectKey; use crate::dag_artifact::types::{DagColumnLineage, DagValue}; @@ -243,6 +245,69 @@ pub struct DagSelectionGraph { pub checks: Vec, } +/// One DAG artifact node as orchestrator asset projection reads it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagProjectionNode { + pub id: String, + pub kind: String, +} + +/// One DAG artifact check as orchestrator check projection reads it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagProjectionCheck { + pub kind: String, + pub checked_asset_ids: Vec, +} + +/// The DAG artifact's nodes, `(from, to)` edges and checks, in artifact order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DagProjectionGraph { + pub nodes: Vec, + pub edges: Vec<(String, String)>, + pub checks: Vec, +} + +/// Which nodes become orchestrator assets and which upstream nodes they depend on. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetProjectionPolicy { + /// Node kinds the orchestrator owns as assets. + pub asset_kinds: BTreeSet, + /// Upstream nodes of other kinds are left out of an asset's dependencies. + pub upstream_asset_kinds_only: bool, +} + +/// One asset node and the nodes it depends on, as indexes into the graph's nodes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetNodePlan { + pub node: usize, + pub upstream: Vec, +} + +/// Which check kinds a check definition exposes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CheckKindFilter { + pub excluded: BTreeSet, + /// When set, only these kinds are exposed. + pub required: Option>, +} + +/// One check, whether its kind is exposed, and the nodes it checks as graph indexes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CheckPlan { + pub check: usize, + pub included: bool, + pub nodes: Vec, +} + +/// How an asset's orchestrator kinds follow from its node kind and materialization. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssetSpecKindPolicy { + pub model_kind: String, + pub view_materialization: String, + /// Node kinds reported as their own orchestrator kind. + pub direct_kinds: BTreeSet, +} + /// The node lookup and edge indexes every selector token resolves against. #[derive(Debug)] pub(crate) struct DagSelectionIndex<'a> { diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs index e7bae12d29..ff81f5efd0 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests.rs @@ -2,5 +2,7 @@ mod build_dag_json; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_projection.rs"] +mod projection; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs index c41a2a4f7f..a6ab1a2e78 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs @@ -96,3 +96,42 @@ pub(crate) fn team_meta(value: DagValue) -> DagValue { (text_key("team"), value), ]) } + +/// Nodes `seed.a`, `model.b`, `check.c`, `model.b` again (the later one wins) and edges into them. +pub(crate) fn projection_graph() -> crate::dag_artifact::models::DagProjectionGraph { + use crate::dag_artifact::models::{DagProjectionCheck, DagProjectionGraph, DagProjectionNode}; + let node = |id: &str, kind: &str| DagProjectionNode { + id: id.to_owned(), + kind: kind.to_owned(), + }; + let edge = |from: &str, to: &str| (from.to_owned(), to.to_owned()); + DagProjectionGraph { + nodes: vec![ + node("seed.a", "seed"), + node("model.b", "model"), + node("check.c", "check"), + node("model.b", "model"), + ], + edges: vec![ + edge("missing", "model.b"), + edge("check.c", "model.b"), + edge("seed.a", "model.b"), + edge("model.b", "check.c"), + ], + checks: vec![ + DagProjectionCheck { + kind: "scenario".to_owned(), + checked_asset_ids: vec!["model.b".to_owned(), "missing".to_owned()], + }, + DagProjectionCheck { + kind: "audit".to_owned(), + checked_asset_ids: vec!["seed.a".to_owned()], + }, + ], + } +} + +/// The Dagster and Rivers asset node kinds of the projection graph. +pub(crate) fn asset_kinds() -> std::collections::BTreeSet { + ["seed", "model"].map(str::to_owned).into() +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs new file mode 100644 index 0000000000..749a854954 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_projection.rs @@ -0,0 +1,115 @@ +//! Orchestrator asset and check projections follow the integrations' artifact indexing. + +use std::collections::BTreeSet; + +use crate::dag_artifact::main::dag_asset_plans::dag_asset_plans; +use crate::dag_artifact::main::dag_asset_spec_kinds::dag_asset_spec_kinds; +use crate::dag_artifact::main::dag_check_plans::dag_check_plans; +use crate::dag_artifact::models::{ + AssetNodePlan, AssetProjectionPolicy, AssetSpecKindPolicy, CheckKindFilter, CheckPlan, +}; +use crate::dag_artifact::tests::helpers::{asset_kinds, projection_graph}; +use crate::dag_artifact::tests::test_types::{ + AssetPlansTestCase, CheckPlansTestCase, SpecKindsTestCase, +}; + +#[test] +fn given_artifact_graph_when_planning_assets_then_keeps_edge_order_and_last_duplicate() { + let test_cases = [ + AssetPlansTestCase { + description: "any known upstream node", + upstream_asset_kinds_only: false, + expected_plans: vec![(0, vec![]), (1, vec![2, 0]), (3, vec![2, 0])], + }, + AssetPlansTestCase { + description: "upstream asset nodes only", + upstream_asset_kinds_only: true, + expected_plans: vec![(0, vec![]), (1, vec![0]), (3, vec![0])], + }, + ]; + for test_case in test_cases { + let policy: AssetProjectionPolicy = AssetProjectionPolicy { + asset_kinds: asset_kinds(), + upstream_asset_kinds_only: test_case.upstream_asset_kinds_only, + }; + let plans: Vec<(usize, Vec)> = dag_asset_plans(&projection_graph(), &policy) + .into_iter() + .map(|plan: AssetNodePlan| (plan.node, plan.upstream)) + .collect(); + assert_eq!(plans, test_case.expected_plans, "{}", test_case.description); + } +} + +#[test] +fn given_check_filter_when_planning_checks_then_marks_exposed_kinds() { + let scenario: BTreeSet = BTreeSet::from(["scenario".to_owned()]); + let test_cases = [ + CheckPlansTestCase { + description: "scenarios excluded", + filter: CheckKindFilter { + excluded: scenario.clone(), + required: None, + }, + expected_plans: vec![(0, false, vec![3]), (1, true, vec![0])], + }, + CheckPlansTestCase { + description: "scenarios only", + filter: CheckKindFilter { + excluded: BTreeSet::new(), + required: Some(scenario.clone()), + }, + expected_plans: vec![(0, true, vec![3]), (1, false, vec![0])], + }, + ]; + for test_case in test_cases { + let plans: Vec<(usize, bool, Vec)> = + dag_check_plans(&projection_graph(), &test_case.filter) + .into_iter() + .map(|plan: CheckPlan| (plan.check, plan.included, plan.nodes)) + .collect(); + assert_eq!(plans, test_case.expected_plans, "{}", test_case.description); + } +} + +#[test] +fn given_asset_node_when_choosing_spec_kinds_then_reports_materialization_or_kind() { + let test_cases = [ + SpecKindsTestCase { + description: "view model", + kind: "model", + materialization_type: "view", + expected_kinds: vec!["sqlbuild", "view"], + }, + SpecKindsTestCase { + description: "incremental model is a table", + kind: "model", + materialization_type: "incremental", + expected_kinds: vec!["sqlbuild", "table"], + }, + SpecKindsTestCase { + description: "direct kind", + kind: "seed", + materialization_type: "table", + expected_kinds: vec!["sqlbuild", "seed"], + }, + SpecKindsTestCase { + description: "other kind", + kind: "check", + materialization_type: "table", + expected_kinds: vec!["sqlbuild"], + }, + ]; + let policy: AssetSpecKindPolicy = AssetSpecKindPolicy { + model_kind: "model".to_owned(), + view_materialization: "view".to_owned(), + direct_kinds: BTreeSet::from(["seed".to_owned()]), + }; + for test_case in test_cases { + assert_eq!( + dag_asset_spec_kinds((test_case.kind, test_case.materialization_type), &policy), + test_case.expected_kinds, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs index 529ebbda57..5b05ed6281 100644 --- a/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs @@ -9,3 +9,22 @@ pub(crate) struct BuildDagJsonTestCase { pub(crate) upstream: Vec<(ObjectKey, Vec)>, pub(crate) expected_json: Result<&'static str, DagArtifactError>, } + +pub(crate) struct AssetPlansTestCase { + pub(crate) description: &'static str, + pub(crate) upstream_asset_kinds_only: bool, + pub(crate) expected_plans: Vec<(usize, Vec)>, +} + +pub(crate) struct CheckPlansTestCase { + pub(crate) description: &'static str, + pub(crate) filter: crate::dag_artifact::models::CheckKindFilter, + pub(crate) expected_plans: Vec<(usize, bool, Vec)>, +} + +pub(crate) struct SpecKindsTestCase { + pub(crate) description: &'static str, + pub(crate) kind: &'static str, + pub(crate) materialization_type: &'static str, + pub(crate) expected_kinds: Vec<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs index 17aad5e650..791dcc6822 100644 --- a/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs +++ b/crates/sqlbuild-analysis/src/fingerprint_storage/constants.rs @@ -31,3 +31,5 @@ pub const FINGERPRINT_COLUMNS: [&str; 12] = [ /// The node type that wins a node-name collision between latest rows. pub const NODE_TYPE_MODEL: &str = "model"; +/// The node type of seed fingerprints. +pub const NODE_TYPE_SEED: &str = "seed"; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 9a1087f740..6f0b096877 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -1,16 +1,22 @@ //! Janitor planning names, prefixes and skip reasons. +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; + /// SQLBuild state tables the janitor never acts on. pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 9] = [ - "_sqlbuild_fingerprints", + FINGERPRINT_TABLE_NAME, "_sqlbuild_microbatches", - "_sqlbuild_source_freshness", + SOURCE_FRESHNESS_TABLE_NAME, "_sqlbuild_node_results", "_sqlbuild_audit_results", "_sqlbuild_janitor_events", - "_sqlbuild_migrations", - "_sqlbuild_column_migrations", - "_sqlbuild_old_name_views", + MIGRATION_TABLE_NAME, + COLUMN_MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, ]; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs index 69c555cdcd..dd88d0ff6b 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/mod.rs @@ -6,6 +6,7 @@ pub mod matching_exclude_pattern; pub mod migration_origin_keys; pub mod old_name_request_matches; pub mod parse_archive_name; +pub mod query_diff_artifact_name; pub mod query_diff_artifact_parts; pub mod resolve_retention_days; pub mod unaddressable_relation_reason; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs b/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs new file mode 100644 index 0000000000..0bccd96cd2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/janitor_planning/main/query_diff_artifact_name.rs @@ -0,0 +1,9 @@ +//! The reserved relation name of one run's query-diff artifact. + +use crate::janitor_planning::constants::QUERY_DIFF_ARTIFACT_PREFIX; + +/// The artifact relation name for `run_id` and `side`, which `query_diff_artifact_parts` parses. +#[must_use] +pub fn query_diff_artifact_name(run_id: &str, side: &str) -> String { + format!("{QUERY_DIFF_ARTIFACT_PREFIX}{run_id}_{side}") +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs index a322336757..9273ff4854 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_archive_names.rs @@ -2,10 +2,12 @@ use crate::janitor_planning::main::build_archive_name::build_archive_name; use crate::janitor_planning::main::parse_archive_name::parse_archive_name; +use crate::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name; use crate::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; use crate::janitor_planning::models::ArchiveTimestamp; use crate::janitor_planning::tests::test_types::{ - ArtifactNameTestCase, BuildArchiveNameTestCase, ParseArchiveNameTestCase, + ArtifactNameBuildTestCase, ArtifactNameTestCase, BuildArchiveNameTestCase, + ParseArchiveNameTestCase, }; const ARCHIVED_AT: ArchiveTimestamp = ArchiveTimestamp { @@ -148,3 +150,31 @@ fn given_relation_name_when_checking_query_diff_artifact_then_returns_run_and_si ); } } + +#[test] +fn given_run_and_side_when_building_query_diff_artifact_name_then_parses_back() { + let test_cases = [ + ArtifactNameBuildTestCase { + description: "generated run id round trips", + run_id: "20260115T123005Z_0123456789ab", + side: "right", + expected_parts: Some(("20260115T123005Z_0123456789ab", "right")), + }, + ArtifactNameBuildTestCase { + description: "free-form run id is not reserved", + run_id: "manual", + side: "left", + expected_parts: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + query_diff_artifact_parts(&query_diff_artifact_name(test_case.run_id, test_case.side)), + test_case + .expected_parts + .map(|(run_id, side)| (run_id.to_owned(), side.to_owned())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs index 1dde854e76..1ad44509e9 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/tests/test_types.rs @@ -50,6 +50,13 @@ pub(super) struct ArtifactNameTestCase { pub expected_parts: Option<(&'static str, &'static str)>, } +pub(super) struct ArtifactNameBuildTestCase { + pub description: &'static str, + pub run_id: &'static str, + pub side: &'static str, + pub expected_parts: Option<(&'static str, &'static str)>, +} + pub(super) struct RetentionTestCase { pub description: &'static str, pub override_days: Option, diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 53e6d5545c..3f2e538f9d 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -21,6 +21,7 @@ pub mod lineage; pub mod lineage_command; pub mod lineage_graph; pub mod manifest; +pub mod nullability_rules; pub mod planner_changes; pub mod planner_fixtures; pub mod planner_identity; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs new file mode 100644 index 0000000000..67b44465be --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/conditional_result_nullability.rs @@ -0,0 +1,19 @@ +//! The nullability of an IF/IFF-style conditional from its two result arguments. + +use crate::assembly::analysis_session::constants::{ + NON_NULL_NULLABILITY, NULLABLE_NULLABILITY, UNKNOWN_NULLABILITY, +}; + +/// Non-null when both results are, nullable when either is, otherwise unknown. +pub fn conditional_result_nullability(arguments: &[&str]) -> &'static str { + let [_, when_true, when_false, ..] = arguments else { + return UNKNOWN_NULLABILITY; + }; + if *when_true == NON_NULL_NULLABILITY && *when_false == NON_NULL_NULLABILITY { + return NON_NULL_NULLABILITY; + } + if *when_true == NULLABLE_NULLABILITY || *when_false == NULLABLE_NULLABILITY { + return NULLABLE_NULLABILITY; + } + UNKNOWN_NULLABILITY +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs new file mode 100644 index 0000000000..f4d72700b4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/first_arg_nullability.rs @@ -0,0 +1,8 @@ +//! The nullability of a null-propagating unary function. + +use crate::assembly::analysis_session::constants::UNKNOWN_NULLABILITY; + +/// The first argument's nullability, or unknown without arguments. +pub fn first_arg_nullability<'a>(arguments: &[&'a str]) -> &'a str { + arguments.first().copied().unwrap_or(UNKNOWN_NULLABILITY) +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs new file mode 100644 index 0000000000..8efe47be41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/mod.rs @@ -0,0 +1,3 @@ +pub mod conditional_result_nullability; +pub mod first_arg_nullability; +pub mod safe_cast_nullability; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs b/crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs new file mode 100644 index 0000000000..90cf26c4d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/main/safe_cast_nullability.rs @@ -0,0 +1,12 @@ +//! The nullability of a SAFE_CAST/TRY_CAST-style conversion. + +use crate::assembly::analysis_session::constants::{NULLABLE_NULLABILITY, UNKNOWN_NULLABILITY}; + +/// Nullable when the converted value is nullable; otherwise unknown, since conversion may fail. +pub fn safe_cast_nullability(arguments: &[&str]) -> &'static str { + if arguments.first() == Some(&NULLABLE_NULLABILITY) { + NULLABLE_NULLABILITY + } else { + UNKNOWN_NULLABILITY + } +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/mod.rs b/crates/sqlbuild-analysis/src/nullability_rules/mod.rs new file mode 100644 index 0000000000..e29b572755 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/mod.rs @@ -0,0 +1,4 @@ +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/tests.rs b/crates/sqlbuild-analysis/src/nullability_rules/tests.rs new file mode 100644 index 0000000000..5c5151c2b0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_nullability_rules.rs"] +mod nullability_rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs new file mode 100644 index 0000000000..31b8270d7c --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_nullability_rules.rs @@ -0,0 +1,69 @@ +use crate::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; +use crate::nullability_rules::main::first_arg_nullability::first_arg_nullability; +use crate::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; +use crate::nullability_rules::tests::test_types::NullabilityRuleTestCase; + +#[test] +fn given_argument_nullabilities_when_inferred_then_each_rule_matches_its_contract() { + let test_cases = [ + NullabilityRuleTestCase { + description: "no arguments", + arguments: vec![], + expected_first_arg: "unknown", + expected_conditional: "unknown", + expected_safe_cast: "unknown", + }, + NullabilityRuleTestCase { + description: "a nullable value only", + arguments: vec!["nullable"], + expected_first_arg: "nullable", + expected_conditional: "unknown", + expected_safe_cast: "nullable", + }, + NullabilityRuleTestCase { + description: "two non-null results", + arguments: vec!["unknown", "non_null", "non_null"], + expected_first_arg: "unknown", + expected_conditional: "non_null", + expected_safe_cast: "unknown", + }, + NullabilityRuleTestCase { + description: "one nullable result", + arguments: vec!["non_null", "non_null", "nullable", "nullable"], + expected_first_arg: "non_null", + expected_conditional: "nullable", + expected_safe_cast: "unknown", + }, + NullabilityRuleTestCase { + description: "an unproven result", + arguments: vec!["nullable", "non_null", "unknown"], + expected_first_arg: "nullable", + expected_conditional: "unknown", + expected_safe_cast: "nullable", + }, + NullabilityRuleTestCase { + description: "an unrecognised value", + arguments: vec!["other", "other", "non_null"], + expected_first_arg: "other", + expected_conditional: "unknown", + expected_safe_cast: "unknown", + }, + ]; + for test_case in test_cases { + let actual = ( + first_arg_nullability(&test_case.arguments), + conditional_result_nullability(&test_case.arguments), + safe_cast_nullability(&test_case.arguments), + ); + assert_eq!( + actual, + ( + test_case.expected_first_arg, + test_case.expected_conditional, + test_case.expected_safe_cast + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs new file mode 100644 index 0000000000..11caf17093 --- /dev/null +++ b/crates/sqlbuild-analysis/src/nullability_rules/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct NullabilityRuleTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: Vec<&'static str>, + pub(crate) expected_first_arg: &'static str, + pub(crate) expected_conditional: &'static str, + pub(crate) expected_safe_cast: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs index d9eab548d5..c6f07fd0d5 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/main/stored_migration_fingerprint.rs @@ -1,5 +1,7 @@ //! Read persisted migration hashes without changing their representation. +use crate::planner_changes::constants::MIGRATION_FINGERPRINT_METADATA_KEY; + /// Read only string migration fingerprints from object metadata. pub fn stored_migration_fingerprint(metadata: &str) -> Option { let payload: serde_json::Value = match serde_json::from_str(metadata) { @@ -7,7 +9,7 @@ pub fn stored_migration_fingerprint(metadata: &str) -> Option { Err(_) => return None, }; payload - .get("migration_fingerprint")? + .get(MIGRATION_FINGERPRINT_METADATA_KEY)? .as_str() .map(str::to_owned) } diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index 40a7abb782..e3d4e92112 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -10,12 +10,12 @@ use crate::graph::main::parse_selector::parse_selector; use crate::graph::models::{EdgeDirection, ParsedSelector, ProjectGraph}; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, PATH_BETWEEN_CODE, - PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, PYTHON_NODE_PATH_ROOT, - SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNIT_TEST_CODE, - UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, - UNMAPPED_KIND_CODE, + PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PATTERN_CHARACTERS, SQL_MODEL_PATH_ROOT, TAG_KIND, + TASK_KIND, TEST_KIND, UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, + UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; +use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; pub(crate) type Names = BTreeSet; type Resolved = Result; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 6357ed80b9..8d91e471af 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -19,11 +19,12 @@ use crate::python_selection::_helpers::validation::{ }; use crate::python_selection::constants::{ ASSET_KIND, CHECK_KIND, EXPANSION_MARKER, INTERSECTION_SEPARATOR, LOADER_KIND, NAME_KIND, - PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, PYTHON_NODE_PATH_ROOT, - SEED_KIND, SOURCE_KIND, SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, - UNKNOWN_NAME_CODE, UNKNOWN_TAG_CODE, + PATH_KIND, PATH_ROOT_CODE, PATH_ROOT_ERROR, PLANNER_DEFAULT_CODE, SEED_KIND, SOURCE_KIND, + SOURCE_RESOURCE, SQL_MODEL_PATH_ROOT, TAG_KIND, TASK_KIND, TEST_KIND, UNKNOWN_NAME_CODE, + UNKNOWN_TAG_CODE, }; use crate::python_selection::models::{Atom, PythonSelectionNode, RunSelection, SelectionContext}; +use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; type Atoms = BTreeSet; type Resolved = Result; diff --git a/crates/sqlbuild-analysis/src/python_selection/constants.rs b/crates/sqlbuild-analysis/src/python_selection/constants.rs index 3bd1d1c28a..202a3b2169 100644 --- a/crates/sqlbuild-analysis/src/python_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/python_selection/constants.rs @@ -19,7 +19,6 @@ pub(crate) const EXPANSION_MARKER: char = '+'; pub(crate) const INTERSECTION_SEPARATOR: char = ','; pub(crate) const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; pub(crate) const SQL_MODEL_PATH_ROOT: &str = "models"; -pub(crate) const PYTHON_NODE_PATH_ROOT: &str = "python"; pub(crate) const CURRENT_DIRECTORY: &str = "."; pub(crate) const ROOT_DIRECTORY: &str = "/"; pub(crate) const PLANNER_DEFAULT_CODE: &str = "S000"; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs index f1eccc28f2..cc4e726024 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs @@ -6,10 +6,11 @@ use regex::Regex; use crate::semantic_checks::_helpers::explanation::messages::{compiled, pattern}; use crate::semantic_checks::constants::{ - ADDITIONAL_HELP_SEPARATOR, FINDING_KINDS, PROJECT_CONFIG_FILENAME, REQUIRE_SQL_ANALYSIS_KEY, - SETTING_SNIPPET_INDENT, SETTINGS_SECTION, + ADDITIONAL_HELP_SEPARATOR, FINDING_KINDS, REQUIRE_SQL_ANALYSIS_KEY, SETTING_SNIPPET_INDENT, }; use crate::semantic_checks::models::{OptOutDiagnostic, SemanticFailure}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; static FINDING_PATTERNS: LazyLock> = LazyLock::new(|| { let alternatives: Vec = FINDING_KINDS diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index 486b6f4bde..fc721ba5db 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -16,8 +16,6 @@ pub(crate) const TYPE_CODES: [&str; 10] = [ pub(crate) const MIN_ABBREVIATION_LENGTH: usize = 3; pub(crate) const MAX_EDIT_DISTANCE: usize = 2; pub(crate) const DISPLAY_LIMIT: usize = 10; -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; -pub(crate) const SETTINGS_SECTION: &str = "settings"; pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs index e941f13be5..67a32cb1f1 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/constants.rs @@ -5,6 +5,6 @@ pub(crate) const DROP_QUERY_TYPE: &str = "DROP"; pub(crate) const CREATE_QUERY_TYPE: &str = "CREATE"; pub(crate) const DYNAMIC_TABLE_MODIFIER: &str = "DYNAMIC"; pub(crate) const CREATE_VARIANT_PREFIX: &str = "create"; -pub(crate) const QUERY_TAG_APP: &str = "sqlbuild"; +pub const QUERY_TAG_APP: &str = "sqlbuild"; pub(crate) const QUERY_TAG_APP_KEY: &str = "app"; pub(crate) const QUERY_TAG_VERSION_KEY: &str = "v"; diff --git a/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs index 1e64f3d116..f01181f853 100644 --- a/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs +++ b/crates/sqlbuild-analysis/src/snowflake_cost/mod.rs @@ -1,5 +1,5 @@ pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs b/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs index c9df6e1bce..13750471b7 100644 --- a/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs +++ b/crates/sqlbuild-attachments/src/audits/_helpers/policies.rs @@ -5,7 +5,7 @@ use crate::audits::models::PolicySource; const SEVERITIES: [&str; 2] = ["warn", "error"]; /// Python lists run scopes sorted. const RUN_SCOPES: [&str; 2] = ["delta_and_final", "final"]; -const SETTINGS_FILE: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as SETTINGS_FILE; /// The severity text, or Python's error naming the unknown value. pub(crate) fn resolve_severity( diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs index dcb580caa4..b1b26f57a2 100644 --- a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs @@ -5,7 +5,6 @@ use std::fs::{File, OpenOptions}; use std::io::{ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; -use crate::artifacts::constants::NEW_FILE_MODE; use crate::artifacts::errors::ArtifactError; use crate::artifacts::models::{Publication, WrittenArtifacts}; @@ -261,7 +260,9 @@ fn copy_new_file(source: &Path, path: &Path) -> std::io::Result<()> { #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let _ = builder.permissions(std::fs::Permissions::from_mode(NEW_FILE_MODE)); + let _ = builder.permissions(std::fs::Permissions::from_mode( + crate::artifacts::constants::NEW_FILE_MODE, + )); } let mut temporary = builder.tempfile_in(directory)?; temporary.write_all(&std::fs::read(source)?)?; diff --git a/crates/sqlbuild-cache/src/artifacts/constants.rs b/crates/sqlbuild-cache/src/artifacts/constants.rs index 6625926d4d..3feacccd00 100644 --- a/crates/sqlbuild-cache/src/artifacts/constants.rs +++ b/crates/sqlbuild-cache/src/artifacts/constants.rs @@ -2,4 +2,5 @@ pub const NEW_FILE_MODE: u32 = 0o666; /// The extension every compiled SQL test artifact carries. pub(crate) const SQL_EXTENSION: &str = "sql"; +#[cfg(unix)] pub(crate) const NANOS_PER_SECOND: i64 = 1_000_000_000; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs new file mode 100644 index 0000000000..8f30546304 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -0,0 +1,1962 @@ +//! The Python shape names of the natively hosted compiler shapes. + +/// The extension module hosting the shapes. +pub const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The Python model type a natively hosted compiler shape is captured as. +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 491] = [ + ( + "sqlbuild._native:AlignedInterval", + "sqlbuild.cursor_algebra.models:AlignedInterval", + ), + ( + "sqlbuild._native:AnalysisCacheContext", + "sqlbuild.compiler.compile.models:AnalysisCacheContext", + ), + ( + "sqlbuild._native:AttachedAuditGateEdge", + "sqlbuild.compiler.graph.models:AttachedAuditGateEdge", + ), + ( + "sqlbuild._native:AuditGateIdentity", + "sqlbuild.compiler.auditing.models:AuditGateIdentity", + ), + ( + "sqlbuild._native:AuditIdentity", + "sqlbuild.compiler.auditing.models:AuditIdentity", + ), + ( + "sqlbuild._native:AuditPlanEntry", + "sqlbuild.compiler.planner.models:AuditPlanEntry", + ), + ( + "sqlbuild._native:AuthoredTimeTravelRetention", + "sqlbuild.spec.contracts.models:AuthoredTimeTravelRetention", + ), + ( + "sqlbuild._native:BackfillAction", + "sqlbuild.compiler.planner.types:BackfillAction", + ), + ( + "sqlbuild._native:ChainStep", + "sqlbuild.compiler.planner.models:ChainStep", + ), + ( + "sqlbuild._native:ClonePipelineConnection", + "sqlbuild.compiler.pipeline.models:ClonePipelineConnection", + ), + ( + "sqlbuild._native:ClonePipelineOptions", + "sqlbuild.compiler.pipeline.models:ClonePipelineOptions", + ), + ( + "sqlbuild._native:ClonePipelineResult", + "sqlbuild.compiler.pipeline.models:ClonePipelineResult", + ), + ( + "sqlbuild._native:ClonePolicy", + "sqlbuild.spec.contracts.models:ClonePolicy", + ), + ( + "sqlbuild._native:CloneSourcePlanEntry", + "sqlbuild.compiler.planner.models:CloneSourcePlanEntry", + ), + ( + "sqlbuild._native:CollectionRendering", + "sqlbuild.sql_values.types:CollectionRendering", + ), + ( + "sqlbuild._native:ColumnContractMode", + "sqlbuild.spec.contracts.types:ColumnContractMode", + ), + ( + "sqlbuild._native:ColumnLineage", + "sqlbuild.compiler.lineage.models:ColumnLineage", + ), + ( + "sqlbuild._native:ColumnLineageConfidence", + "sqlbuild.compiler.lineage.types:ColumnLineageConfidence", + ), + ( + "sqlbuild._native:ColumnLineageEdge", + "sqlbuild.compiler.lineage.models:ColumnLineageEdge", + ), + ( + "sqlbuild._native:ColumnLineageMode", + "sqlbuild.compiler.lineage.types:ColumnLineageMode", + ), + ( + "sqlbuild._native:ColumnLineageSource", + "sqlbuild.compiler.lineage.models:ColumnLineageSource", + ), + ( + "sqlbuild._native:ColumnMigrationDecision", + "sqlbuild.compiler.migrations.types:ColumnMigrationDecision", + ), + ( + "sqlbuild._native:ColumnMigrationEvent", + "sqlbuild.compiler.migrations.models:ColumnMigrationEvent", + ), + ( + "sqlbuild._native:ColumnMigrationPlanEntry", + "sqlbuild.compiler.planner.models:ColumnMigrationPlanEntry", + ), + ( + "sqlbuild._native:ColumnTransformKind", + "sqlbuild.compiler.lineage.types:ColumnTransformKind", + ), + ( + "sqlbuild._native:CompactLineageFacts", + "sqlbuild.compiler.compile.models:CompactLineageFacts", + ), + ( + "sqlbuild._native:CompileAdapterContext", + "sqlbuild.compiler.compile.models:CompileAdapterContext", + ), + ( + "sqlbuild._native:CompileAnalysisSelection", + "sqlbuild.compiler.compile.models:CompileAnalysisSelection", + ), + ( + "sqlbuild._native:CompileDirectLogicSqlTestInputPayload", + "sqlbuild.compiler.compile.models:CompileDirectLogicSqlTestInputPayload", + ), + ( + "sqlbuild._native:CompileFacts", + "sqlbuild.lint.models:CompileFacts", + ), + ( + "sqlbuild._native:CompileInputError", + "sqlbuild.compiler.compile.exceptions:CompileInputError", + ), + ( + "sqlbuild._native:CompileModelConfig", + "sqlbuild.compiler.compile.models:CompileModelConfig", + ), + ( + "sqlbuild._native:CompileModelInput", + "sqlbuild.compiler.compile.models:CompileModelInput", + ), + ( + "sqlbuild._native:CompileModelSqlTestInputPayload", + "sqlbuild.compiler.compile.models:CompileModelSqlTestInputPayload", + ), + ( + "sqlbuild._native:CompilePipelineOptions", + "sqlbuild.compiler.pipeline.models:CompilePipelineOptions", + ), + ( + "sqlbuild._native:CompilePipelineResult", + "sqlbuild.compiler.pipeline.models:CompilePipelineResult", + ), + ( + "sqlbuild._native:CompileProjectInputs", + "sqlbuild.compiler.compile.models:CompileProjectInputs", + ), + ( + "sqlbuild._native:CompileSqlReference", + "sqlbuild.compiler.compile.models:CompileSqlReference", + ), + ( + "sqlbuild._native:CompileSqlScenarioCte", + "sqlbuild.compiler.compile.models:CompileSqlScenarioCte", + ), + ( + "sqlbuild._native:CompileSqlScenarioInput", + "sqlbuild.compiler.compile.models:CompileSqlScenarioInput", + ), + ( + "sqlbuild._native:CompileSqlTestCte", + "sqlbuild.compiler.compile.models:CompileSqlTestCte", + ), + ( + "sqlbuild._native:CompileSqlTestInput", + "sqlbuild.compiler.compile.models:CompileSqlTestInput", + ), + ( + "sqlbuild._native:CompiledAudit", + "sqlbuild.compiler.compile.models:CompiledAudit", + ), + ( + "sqlbuild._native:CompiledDirectLogicSqlTestPayload", + "sqlbuild.compiler.compile.models:CompiledDirectLogicSqlTestPayload", + ), + ( + "sqlbuild._native:CompiledFunction", + "sqlbuild.compiler.compile.models:CompiledFunction", + ), + ( + "sqlbuild._native:CompiledLineageColumnFact", + "sqlbuild.compiler.compile.models:CompiledLineageColumnFact", + ), + ( + "sqlbuild._native:CompiledLineageSourceFact", + "sqlbuild.compiler.compile.models:CompiledLineageSourceFact", + ), + ( + "sqlbuild._native:CompiledModel", + "sqlbuild.compiler.compile.models:CompiledModel", + ), + ( + "sqlbuild._native:CompiledModelSqlTestPayload", + "sqlbuild.compiler.compile.models:CompiledModelSqlTestPayload", + ), + ( + "sqlbuild._native:CompiledObjectKey", + "sqlbuild.compiler.compile.models:CompiledObjectKey", + ), + ( + "sqlbuild._native:CompiledProject", + "sqlbuild.compiler.compile.models:CompiledProject", + ), + ( + "sqlbuild._native:CompiledProjectPhaseResult", + "sqlbuild.compiler.pipeline.models:CompiledProjectPhaseResult", + ), + ( + "sqlbuild._native:CompiledRelationLocation", + "sqlbuild.compiler.compile.models:CompiledRelationLocation", + ), + ( + "sqlbuild._native:CompiledSeed", + "sqlbuild.compiler.compile.models:CompiledSeed", + ), + ( + "sqlbuild._native:CompiledSource", + "sqlbuild.compiler.compile.models:CompiledSource", + ), + ( + "sqlbuild._native:CompiledSqlExpansion", + "sqlbuild.compiler.compile.models:CompiledSqlExpansion", + ), + ( + "sqlbuild._native:CompiledSqlScenario", + "sqlbuild.compiler.compile.models:CompiledSqlScenario", + ), + ( + "sqlbuild._native:CompiledSqlTest", + "sqlbuild.compiler.compile.models:CompiledSqlTest", + ), + ( + "sqlbuild._native:CompiledSqlTestResource", + "sqlbuild.compiler.compile.models:CompiledSqlTestResource", + ), + ( + "sqlbuild._native:CompilerDiagnostic", + "sqlbuild.compiler.compile.models:CompilerDiagnostic", + ), + ( + "sqlbuild._native:CompilerStage", + "sqlbuild.compiler.frontier.types:CompilerStage", + ), + ( + "sqlbuild._native:ConfigValueTypeError", + "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", + ), + ( + "sqlbuild._native:ConstantDeclaration", + "sqlbuild.compiler.discovery.models:ConstantDeclaration", + ), + ( + "sqlbuild._native:ConstantMetadata", + "sqlbuild.compiler.scopes.models:ConstantMetadata", + ), + ( + "sqlbuild._native:ConstantsConfig", + "sqlbuild.spec.contracts.models:ConstantsConfig", + ), + ( + "sqlbuild._native:ContractAdoptionError", + "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", + ), + ( + "sqlbuild._native:ContractAdoptionResult", + "sqlbuild.compiler.contract_adoption.models:ContractAdoptionResult", + ), + ( + "sqlbuild._native:ContractEvidence", + "sqlbuild.compiler.contract_adoption.models:ContractEvidence", + ), + ( + "sqlbuild._native:ContractFinding", + "sqlbuild.compiler.contract_adoption.models:ContractFinding", + ), + ( + "sqlbuild._native:ContractFindingKind", + "sqlbuild.compiler.contract_adoption.types:ContractFindingKind", + ), + ( + "sqlbuild._native:ContractPolicy", + "sqlbuild.compiler.planner.types:ContractPolicy", + ), + ( + "sqlbuild._native:ContractValidationResult", + "sqlbuild.compiler.contracts.models:ContractValidationResult", + ), + ( + "sqlbuild._native:CostConfig", + "sqlbuild.spec.contracts.models:CostConfig", + ), + ( + "sqlbuild._native:CursorBounds", + "sqlbuild.compiler.planner.models:CursorBounds", + ), + ( + "sqlbuild._native:CursorGrain", + "sqlbuild.compiler.planner.types:CursorGrain", + ), + ( + "sqlbuild._native:CursorInputEvidence", + "sqlbuild.compiler.planner.models:CursorInputEvidence", + ), + ( + "sqlbuild._native:CursorInputRelation", + "sqlbuild.compiler.planner.models:CursorInputRelation", + ), + ( + "sqlbuild._native:CursorInputRoles", + "sqlbuild.compiler.compile.models:CursorInputRoles", + ), + ( + "sqlbuild._native:CursorOverrides", + "sqlbuild.compiler.planner.models:CursorOverrides", + ), + ( + "sqlbuild._native:CursorType", + "sqlbuild.compiler.planner.types:CursorType", + ), + ( + "sqlbuild._native:CursorsConfig", + "sqlbuild.spec.contracts.models:CursorsConfig", + ), + ( + "sqlbuild._native:CustomHostPartition", + "sqlbuild.rule_engine.models:CustomHostPartition", + ), + ( + "sqlbuild._native:CustomRuleEvaluation", + "sqlbuild.rule_engine.models:CustomRuleEvaluation", + ), + ( + "sqlbuild._native:CustomRuleRun", + "sqlbuild.rule_engine.models:CustomRuleRun", + ), + ( + "sqlbuild._native:CustomRulesOutcome", + "sqlbuild.rule_engine.models:CustomRulesOutcome", + ), + ( + "sqlbuild._native:DateValue", + "sqlbuild.cursor_algebra.models:DateValue", + ), + ( + "sqlbuild._native:DbtCliOptions", + "sqlbuild.integrations.dbt.models:DbtCliOptions", + ), + ( + "sqlbuild._native:DbtCommandResult", + "sqlbuild.integrations.dbt.models:DbtCommandResult", + ), + ( + "sqlbuild._native:DbtConfig", + "sqlbuild.spec.contracts.models:DbtConfig", + ), + ( + "sqlbuild._native:DbtInitProgressCallbacks", + "sqlbuild.integrations.dbt.models:DbtInitProgressCallbacks", + ), + ( + "sqlbuild._native:DbtInitRequest", + "sqlbuild.integrations.dbt.models:DbtInitRequest", + ), + ( + "sqlbuild._native:DbtInitResult", + "sqlbuild.integrations.dbt.models:DbtInitResult", + ), + ( + "sqlbuild._native:DbtInteropArgumentError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropArgumentError", + ), + ( + "sqlbuild._native:DbtInteropConfigError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropConfigError", + ), + ( + "sqlbuild._native:DbtInteropError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropError", + ), + ( + "sqlbuild._native:DbtInteropRuntimeError", + "sqlbuild.integrations.dbt.exceptions:DbtInteropRuntimeError", + ), + ( + "sqlbuild._native:DbtLsResult", + "sqlbuild.integrations.dbt.models:DbtLsResult", + ), + ( + "sqlbuild._native:DbtNodeExecutionResult", + "sqlbuild.integrations.dbt.models:DbtNodeExecutionResult", + ), + ( + "sqlbuild._native:DeclarationExpansionContext", + "sqlbuild.compiler.compile.models:DeclarationExpansionContext", + ), + ( + "sqlbuild._native:DeclarationExplanation", + "sqlbuild.compiler.scopes.models:DeclarationExplanation", + ), + ( + "sqlbuild._native:DeclarationIdentity", + "sqlbuild.compiler.scopes.models:DeclarationIdentity", + ), + ( + "sqlbuild._native:DeclarationKind", + "sqlbuild.compiler.scopes.types:DeclarationKind", + ), + ( + "sqlbuild._native:DeclarationParseError", + "sqlbuild.compiler.discovery.exceptions:DeclarationParseError", + ), + ( + "sqlbuild._native:DeclarationRecord", + "sqlbuild.compiler.scopes.models:DeclarationRecord", + ), + ( + "sqlbuild._native:DeclarationReport", + "sqlbuild.compiler.scopes.models:DeclarationReport", + ), + ( + "sqlbuild._native:DeclarationResolutionContext", + "sqlbuild.compiler.compile.models:DeclarationResolutionContext", + ), + ( + "sqlbuild._native:DeclarationRuntimeProjection", + "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", + ), + ( + "sqlbuild._native:DeclarationScopeBuild", + "sqlbuild.compiler.compile.models:DeclarationScopeBuild", + ), + ( + "sqlbuild._native:DeclarationScopeResolver", + "sqlbuild.compiler.compile.models:DeclarationScopeResolver", + ), + ( + "sqlbuild._native:DeclarationVisibility", + "sqlbuild.compiler.scopes.models:DeclarationVisibility", + ), + ( + "sqlbuild._native:DeclarationVisibilityIndex", + "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", + ), + ( + "sqlbuild._native:DefaultsConfig", + "sqlbuild.spec.contracts.models:DefaultsConfig", + ), + ( + "sqlbuild._native:DeferralInputs", + "sqlbuild.compiler.planner.models:DeferralInputs", + ), + ( + "sqlbuild._native:DiagnosticPhase", + "sqlbuild.compiler.compile.types:DiagnosticPhase", + ), + ( + "sqlbuild._native:DiagnosticSeverity", + "sqlbuild.compiler.compile.types:DiagnosticSeverity", + ), + ( + "sqlbuild._native:DiffConfig", + "sqlbuild.spec.contracts.models:DiffConfig", + ), + ( + "sqlbuild._native:DirectSemanticColumnUse", + "sqlbuild.compiler.lineage.models:DirectSemanticColumnUse", + ), + ( + "sqlbuild._native:DirectSourceFreshnessPlanningResult", + "sqlbuild.compiler.source_freshness.models:DirectSourceFreshnessPlanningResult", + ), + ( + "sqlbuild._native:DiscoveredAdapterFile", + "sqlbuild.compiler.discovery.models:DiscoveredAdapterFile", + ), + ( + "sqlbuild._native:DiscoveredAssetFunction", + "sqlbuild.compiler.discovery.models:DiscoveredAssetFunction", + ), + ( + "sqlbuild._native:DiscoveredAuditBlock", + "sqlbuild.compiler.discovery.models:DiscoveredAuditBlock", + ), + ( + "sqlbuild._native:DiscoveredAuditFactory", + "sqlbuild.compiler.discovery.models:DiscoveredAuditFactory", + ), + ( + "sqlbuild._native:DiscoveredAuditFile", + "sqlbuild.compiler.discovery.models:DiscoveredAuditFile", + ), + ( + "sqlbuild._native:DiscoveredCheckFunction", + "sqlbuild.compiler.discovery.models:DiscoveredCheckFunction", + ), + ( + "sqlbuild._native:DiscoveredCommandOutputSink", + "sqlbuild.compiler.discovery.models:DiscoveredCommandOutputSink", + ), + ( + "sqlbuild._native:DiscoveredCommandOutputSinkDeclaration", + "sqlbuild.compiler.discovery.models:DiscoveredCommandOutputSinkDeclaration", + ), + ( + "sqlbuild._native:DiscoveredConstantFile", + "sqlbuild.compiler.discovery.models:DiscoveredConstantFile", + ), + ( + "sqlbuild._native:DiscoveredEnumFile", + "sqlbuild.compiler.discovery.models:DiscoveredEnumFile", + ), + ( + "sqlbuild._native:DiscoveredEventExporter", + "sqlbuild.compiler.discovery.models:DiscoveredEventExporter", + ), + ( + "sqlbuild._native:DiscoveredEventExporterDeclaration", + "sqlbuild.compiler.discovery.models:DiscoveredEventExporterDeclaration", + ), + ( + "sqlbuild._native:DiscoveredHookFunction", + "sqlbuild.compiler.discovery.models:DiscoveredHookFunction", + ), + ( + "sqlbuild._native:DiscoveredLoaderFunction", + "sqlbuild.compiler.discovery.models:DiscoveredLoaderFunction", + ), + ( + "sqlbuild._native:DiscoveredMacroFile", + "sqlbuild.compiler.discovery.models:DiscoveredMacroFile", + ), + ( + "sqlbuild._native:DiscoveredMaterializationFile", + "sqlbuild.compiler.discovery.models:DiscoveredMaterializationFile", + ), + ( + "sqlbuild._native:DiscoveredModelSchemaFile", + "sqlbuild.compiler.discovery.models:DiscoveredModelSchemaFile", + ), + ( + "sqlbuild._native:DiscoveredProjectInputs", + "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", + ), + ( + "sqlbuild._native:DiscoveredProvider", + "sqlbuild.compiler.discovery.models:DiscoveredProvider", + ), + ( + "sqlbuild._native:DiscoveredProviderUsage", + "sqlbuild.compiler.discovery.models:DiscoveredProviderUsage", + ), + ( + "sqlbuild._native:DiscoveredPythonAssetMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonAssetMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonCheckMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonCheckMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonFunctionFile", + "sqlbuild.compiler.discovery.models:DiscoveredPythonFunctionFile", + ), + ( + "sqlbuild._native:DiscoveredPythonLoaderMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonLoaderMetadata", + ), + ( + "sqlbuild._native:DiscoveredPythonNode", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonNode", + ), + ( + "sqlbuild._native:DiscoveredPythonNodeFunctions", + "sqlbuild.compiler.discovery.models:DiscoveredPythonNodeFunctions", + ), + ( + "sqlbuild._native:DiscoveredPythonTaskMetadata", + "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", + ), + ( + "sqlbuild._native:DiscoveredRuntimeExtensions", + "sqlbuild.compiler.discovery.models:DiscoveredRuntimeExtensions", + ), + ( + "sqlbuild._native:DiscoveredSchemaFile", + "sqlbuild.compiler.discovery.models:DiscoveredSchemaFile", + ), + ( + "sqlbuild._native:DiscoveredSeedFile", + "sqlbuild.compiler.discovery.models:DiscoveredSeedFile", + ), + ( + "sqlbuild._native:DiscoveredSourceFile", + "sqlbuild.compiler.discovery.models:DiscoveredSourceFile", + ), + ( + "sqlbuild._native:DiscoveredSqlFunctionFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlFunctionFile", + ), + ( + "sqlbuild._native:DiscoveredSqlHookFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlHookFile", + ), + ( + "sqlbuild._native:DiscoveredSqlScenarioFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlScenarioFile", + ), + ( + "sqlbuild._native:DiscoveredSqlTestBlock", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestBlock", + ), + ( + "sqlbuild._native:DiscoveredSqlTestCase", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestCase", + ), + ( + "sqlbuild._native:DiscoveredSqlTestFile", + "sqlbuild.compiler.discovery.models:DiscoveredSqlTestFile", + ), + ( + "sqlbuild._native:DiscoveredTaskFunction", + "sqlbuild.compiler.discovery.models:DiscoveredTaskFunction", + ), + ( + "sqlbuild._native:DiscoveryConflictError", + "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", + ), + ( + "sqlbuild._native:DiscoveryError", + "sqlbuild.compiler.discovery.exceptions:DiscoveryError", + ), + ( + "sqlbuild._native:DiscoveryFileFault", + "sqlbuild.compiler.discovery.models:DiscoveryFileFault", + ), + ( + "sqlbuild._native:DisplayOptions", + "sqlbuild.presentation.models:DisplayOptions", + ), + ( + "sqlbuild._native:Duration", + "sqlbuild.cursor_algebra.models:Duration", + ), + ( + "sqlbuild._native:DynamicColumnContractProof", + "sqlbuild.compiler.compile.models:DynamicColumnContractProof", + ), + ( + "sqlbuild._native:DynamicColumnFamilyProof", + "sqlbuild.compiler.compile.models:DynamicColumnFamilyProof", + ), + ( + "sqlbuild._native:EmptyCursorInputDecision", + "sqlbuild.compiler.planner.models:EmptyCursorInputDecision", + ), + ( + "sqlbuild._native:EnumDeclaration", + "sqlbuild.compiler.discovery.models:EnumDeclaration", + ), + ( + "sqlbuild._native:EnumMember", + "sqlbuild.compiler.discovery.models:EnumMember", + ), + ( + "sqlbuild._native:EnumMemberMetadata", + "sqlbuild.compiler.scopes.models:EnumMemberMetadata", + ), + ( + "sqlbuild._native:EnumMetadata", + "sqlbuild.compiler.scopes.models:EnumMetadata", + ), + ( + "sqlbuild._native:EventExportSeverity", + "sqlbuild.spec.contracts.types:EventExportSeverity", + ), + ( + "sqlbuild._native:EventExporterDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", + ), + ( + "sqlbuild._native:ExecutionLimitsConfig", + "sqlbuild.spec.contracts.models:ExecutionLimitsConfig", + ), + ( + "sqlbuild._native:ExpandedLintFile", + "sqlbuild.lint.models:ExpandedLintFile", + ), + ( + "sqlbuild._native:ExpansionSpan", + "sqlbuild.compiler.compile.models:ExpansionSpan", + ), + ( + "sqlbuild._native:FactDigestError", + "sqlbuild.rule_engine.exceptions:FactDigestError", + ), + ( + "sqlbuild._native:Finding", + "sqlbuild.rule_engine.models:Finding", + ), + ( + "sqlbuild._native:Fingerprint", + "sqlbuild.compiler.fingerprints.models:Fingerprint", + ), + ( + "sqlbuild._native:FingerprintInputError", + "sqlbuild.compiler.fingerprints.exceptions:FingerprintInputError", + ), + ( + "sqlbuild._native:FingerprintSet", + "sqlbuild.compiler.fingerprints.models:FingerprintSet", + ), + ( + "sqlbuild._native:FixVerdict", + "sqlbuild.lint.models:FixVerdict", + ), + ( + "sqlbuild._native:FixtureColumnInference", + "sqlbuild.compiler.compile.models:FixtureColumnInference", + ), + ( + "sqlbuild._native:FunctionArgument", + "sqlbuild.compiler.compile.models:FunctionArgument", + ), + ( + "sqlbuild._native:FunctionPlanEntry", + "sqlbuild.compiler.planner.models:FunctionPlanEntry", + ), + ( + "sqlbuild._native:FunctionReturnColumn", + "sqlbuild.compiler.compile.models:FunctionReturnColumn", + ), + ( + "sqlbuild._native:FutureCursorAction", + "sqlbuild.spec.contracts.types:FutureCursorAction", + ), + ( + "sqlbuild._native:FutureCursorSafetyError", + "sqlbuild.compiler.planner.exceptions:FutureCursorSafetyError", + ), + ( + "sqlbuild._native:FutureCursorSafetyEvidence", + "sqlbuild.compiler.planner.models:FutureCursorSafetyEvidence", + ), + ( + "sqlbuild._native:FutureCursorsConfig", + "sqlbuild.spec.contracts.models:FutureCursorsConfig", + ), + ( + "sqlbuild._native:GrantKind", + "sqlbuild.compiler.scopes.types:GrantKind", + ), + ( + "sqlbuild._native:GrantRecord", + "sqlbuild.compiler.scopes.models:GrantRecord", + ), + ( + "sqlbuild._native:GraphEdgeExclusion", + "sqlbuild.rule_engine.models:GraphEdgeExclusion", + ), + ( + "sqlbuild._native:HandBuiltAnalysis", + "sqlbuild.compiler.compile.models:HandBuiltAnalysis", + ), + ( + "sqlbuild._native:HardCodedRelationOwnerKind", + "sqlbuild.compiler.references.types:HardCodedRelationOwnerKind", + ), + ( + "sqlbuild._native:HeaderSpan", + "sqlbuild.lint.models:HeaderSpan", + ), + ( + "sqlbuild._native:HistoricalInput", + "sqlbuild.compiler.planner.types:HistoricalInput", + ), + ( + "sqlbuild._native:HookReadEdge", + "sqlbuild.compiler.graph.models:HookReadEdge", + ), + ( + "sqlbuild._native:HookReadType", + "sqlbuild.compiler.graph.types:HookReadType", + ), + ( + "sqlbuild._native:HostCancelledError", + "sqlbuild.rule_engine.exceptions:HostCancelledError", + ), + ( + "sqlbuild._native:InaccessibleReason", + "sqlbuild.compiler.scopes.types:InaccessibleReason", + ), + ( + "sqlbuild._native:InaccessibleRecord", + "sqlbuild.compiler.scopes.models:InaccessibleRecord", + ), + ( + "sqlbuild._native:IncrementalMode", + "sqlbuild.compiler.planner.types:IncrementalMode", + ), + ( + "sqlbuild._native:IncrementalStrategy", + "sqlbuild.compiler.planner.types:IncrementalStrategy", + ), + ( + "sqlbuild._native:InferredColumn", + "sqlbuild.compiler.compile.models:InferredColumn", + ), + ( + "sqlbuild._native:InitialValidFrom", + "sqlbuild.compiler.planner.types:InitialValidFrom", + ), + ( + "sqlbuild._native:IntegerValue", + "sqlbuild.cursor_algebra.models:IntegerValue", + ), + ( + "sqlbuild._native:IntegrationLoaderConfig", + "sqlbuild.spec.contracts.models:IntegrationLoaderConfig", + ), + ( + "sqlbuild._native:InterpolationSite", + "sqlbuild.lint.models:InterpolationSite", + ), + ( + "sqlbuild._native:InvalidQualifiedIdentityError", + "sqlbuild.compiler.scopes.exceptions:InvalidQualifiedIdentityError", + ), + ( + "sqlbuild._native:InvalidScopePathError", + "sqlbuild.compiler.scopes.exceptions:InvalidScopePathError", + ), + ( + "sqlbuild._native:InvalidSqlReferenceCall", + "sqlbuild.compiler.compile.models:InvalidSqlReferenceCall", + ), + ( + "sqlbuild._native:JanitorConfig", + "sqlbuild.spec.contracts.models:JanitorConfig", + ), + ( + "sqlbuild._native:LayoutConfig", + "sqlbuild.rule_engine.models:LayoutConfig", + ), + ( + "sqlbuild._native:LifecycleEventSinkFilterConfig", + "sqlbuild.spec.contracts.models:LifecycleEventSinkFilterConfig", + ), + ( + "sqlbuild._native:LifecycleEventSinksConfig", + "sqlbuild.spec.contracts.models:LifecycleEventSinksConfig", + ), + ( + "sqlbuild._native:LineageGraphViews", + "sqlbuild.compiler.graph.models:LineageGraphViews", + ), + ("sqlbuild._native:LintBody", "sqlbuild.lint.models:LintBody"), + ( + "sqlbuild._native:LintConfig", + "sqlbuild.lint.models:LintConfig", + ), + ("sqlbuild._native:LintEdit", "sqlbuild.lint.models:LintEdit"), + ( + "sqlbuild._native:LintError", + "sqlbuild.lint.exceptions:LintError", + ), + ( + "sqlbuild._native:LintFileRole", + "sqlbuild.lint.models:LintFileRole", + ), + ( + "sqlbuild._native:LintRunResult", + "sqlbuild.lint.models:LintRunResult", + ), + ( + "sqlbuild._native:LintSeverity", + "sqlbuild.lint.types:LintSeverity", + ), + ( + "sqlbuild._native:LintStoppedError", + "sqlbuild.lint.exceptions:LintStoppedError", + ), + ( + "sqlbuild._native:LintViolation", + "sqlbuild.lint.models:LintViolation", + ), + ( + "sqlbuild._native:LiteralSqlRelation", + "sqlbuild.compiler.references.models:LiteralSqlRelation", + ), + ( + "sqlbuild._native:LoadedMacro", + "sqlbuild.compiler.compile.models:LoadedMacro", + ), + ( + "sqlbuild._native:LoaderConnectionMode", + "sqlbuild.compiler.discovery.types:LoaderConnectionMode", + ), + ( + "sqlbuild._native:LoaderDestinationParts", + "sqlbuild.spec.contracts.models:LoaderDestinationParts", + ), + ( + "sqlbuild._native:LocalClonePolicy", + "sqlbuild.spec.contracts.models:LocalClonePolicy", + ), + ( + "sqlbuild._native:LocalConfig", + "sqlbuild.spec.contracts.models:LocalConfig", + ), + ( + "sqlbuild._native:LocalDbtConfig", + "sqlbuild.spec.contracts.models:LocalDbtConfig", + ), + ( + "sqlbuild._native:LocalTargetConfig", + "sqlbuild.spec.contracts.models:LocalTargetConfig", + ), + ( + "sqlbuild._native:MacroArgumentError", + "sqlbuild.compiler.compile.exceptions:MacroArgumentError", + ), + ( + "sqlbuild._native:MacroContext", + "sqlbuild.compiler.compile.models:MacroContext", + ), + ( + "sqlbuild._native:MacroExpansionResult", + "sqlbuild.compiler.compile.models:MacroExpansionResult", + ), + ( + "sqlbuild._native:MacroMetadata", + "sqlbuild.compiler.scopes.models:MacroMetadata", + ), + ( + "sqlbuild._native:ManualLocation", + "sqlbuild.compiler.refactoring.models:ManualLocation", + ), + ( + "sqlbuild._native:MappedOffset", + "sqlbuild.compiler.compile.models:MappedOffset", + ), + ( + "sqlbuild._native:MaterializationDefaultsConfig", + "sqlbuild.spec.contracts.models:MaterializationDefaultsConfig", + ), + ( + "sqlbuild._native:MaterializationRetentionDefaults", + "sqlbuild.spec.contracts.models:MaterializationRetentionDefaults", + ), + ( + "sqlbuild._native:MaterializationType", + "sqlbuild.compiler.planner.types:MaterializationType", + ), + ( + "sqlbuild._native:MaximumAutomaticStartError", + "sqlbuild.compiler.planner.exceptions:MaximumAutomaticStartError", + ), + ( + "sqlbuild._native:MaximumStartPolicyInputs", + "sqlbuild.compiler.planner.models:MaximumStartPolicyInputs", + ), + ( + "sqlbuild._native:MaximumStartSafetyEvidence", + "sqlbuild.compiler.planner.models:MaximumStartSafetyEvidence", + ), + ( + "sqlbuild._native:MeasurementAuditError", + "sqlbuild.compiler.auditing.exceptions:MeasurementAuditError", + ), + ( + "sqlbuild._native:MeasurementContract", + "sqlbuild.compiler.auditing.models:MeasurementContract", + ), + ( + "sqlbuild._native:MeasurementThresholdBound", + "sqlbuild.compiler.auditing.models:MeasurementThresholdBound", + ), + ( + "sqlbuild._native:MeasurementThresholds", + "sqlbuild.compiler.auditing.models:MeasurementThresholds", + ), + ( + "sqlbuild._native:MicrobatchLimitAction", + "sqlbuild.spec.contracts.types:MicrobatchLimitAction", + ), + ( + "sqlbuild._native:MicrobatchLimitsConfig", + "sqlbuild.spec.contracts.models:MicrobatchLimitsConfig", + ), + ( + "sqlbuild._native:MicrobatchStrategy", + "sqlbuild.compiler.planner.types:MicrobatchStrategy", + ), + ( + "sqlbuild._native:MicrobatchesConfig", + "sqlbuild.spec.contracts.models:MicrobatchesConfig", + ), + ( + "sqlbuild._native:MigrationCompatibility", + "sqlbuild.compiler.migrations.types:MigrationCompatibility", + ), + ( + "sqlbuild._native:MigrationDecision", + "sqlbuild.compiler.migrations.types:MigrationDecision", + ), + ( + "sqlbuild._native:MigrationDiscovery", + "sqlbuild.compiler.migrations.types:MigrationDiscovery", + ), + ( + "sqlbuild._native:MigrationEvent", + "sqlbuild.compiler.migrations.models:MigrationEvent", + ), + ( + "sqlbuild._native:MigrationPromotion", + "sqlbuild.compiler.migrations.types:MigrationPromotion", + ), + ( + "sqlbuild._native:MigrationRelation", + "sqlbuild.compiler.migrations.models:MigrationRelation", + ), + ( + "sqlbuild._native:MigrationsConfig", + "sqlbuild.spec.contracts.models:MigrationsConfig", + ), + ( + "sqlbuild._native:MissingMigrationOriginPolicy", + "sqlbuild.spec.contracts.types:MissingMigrationOriginPolicy", + ), + ( + "sqlbuild._native:Model", + "sqlbuild.rule_engine.models:Model", + ), + ( + "sqlbuild._native:ModelColumnLineage", + "sqlbuild.compiler.lineage.models:ModelColumnLineage", + ), + ( + "sqlbuild._native:ModelCursorSnapshot", + "sqlbuild.compiler.planner.models:ModelCursorSnapshot", + ), + ( + "sqlbuild._native:ModelInputBuildContext", + "sqlbuild.compiler.compile.models:ModelInputBuildContext", + ), + ( + "sqlbuild._native:ModelMigrationPlanEntry", + "sqlbuild.compiler.planner.models:ModelMigrationPlanEntry", + ), + ( + "sqlbuild._native:ModelPlanEntry", + "sqlbuild.compiler.planner.models:ModelPlanEntry", + ), + ( + "sqlbuild._native:ModelSchemaDeclaration", + "sqlbuild.compiler.discovery.models:ModelSchemaDeclaration", + ), + ( + "sqlbuild._native:ModelSql", + "sqlbuild.rule_engine.models:ModelSql", + ), + ( + "sqlbuild._native:ModelSqlAnalysis", + "sqlbuild.compiler.compile.models:ModelSqlAnalysis", + ), + ( + "sqlbuild._native:ModelSqlParseError", + "sqlbuild.compiler.discovery.exceptions:ModelSqlParseError", + ), + ( + "sqlbuild._native:ModelValidationRequest", + "sqlbuild.compiler.compile.models:ModelValidationRequest", + ), + ( + "sqlbuild._native:ModuleStateFingerprint", + "sqlbuild.rule_engine.models:ModuleStateFingerprint", + ), + ( + "sqlbuild._native:ModuleStateTrace", + "sqlbuild.rule_engine.models:ModuleStateTrace", + ), + ( + "sqlbuild._native:MovePreview", + "sqlbuild.compiler.scopes.models:MovePreview", + ), + ( + "sqlbuild._native:NamedSqlHookEntry", + "sqlbuild.compiler.discovery.models:NamedSqlHookEntry", + ), + ( + "sqlbuild._native:NativeColumnQuery", + "sqlbuild.compiler.analysis_session.models:NativeColumnQuery", + ), + ( + "sqlbuild._native:NativeLintError", + "sqlbuild.lint.exceptions:NativeLintError", + ), + ( + "sqlbuild._native:NativeModelAnalyses", + "sqlbuild.compiler.analysis_session.models:NativeModelAnalyses", + ), + ( + "sqlbuild._native:NativeModelAnalysisRequest", + "sqlbuild.compiler.analysis_session.models:NativeModelAnalysisRequest", + ), + ( + "sqlbuild._native:NativeModelConfigInputs", + "sqlbuild.compiler.compile.models:NativeModelConfigInputs", + ), + ( + "sqlbuild._native:NativeModelConfigSession", + "sqlbuild.compiler.compile.models:NativeModelConfigSession", + ), + ( + "sqlbuild._native:NativePivotTables", + "sqlbuild.compiler.analysis_session.models:NativePivotTables", + ), + ( + "sqlbuild._native:NativeQueryColumns", + "sqlbuild.compiler.analysis_session.models:NativeQueryColumns", + ), + ( + "sqlbuild._native:NativeRulesEvaluation", + "sqlbuild.rule_engine.models:NativeRulesEvaluation", + ), + ( + "sqlbuild._native:NativeSqlTestAssembly", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestAssembly", + ), + ( + "sqlbuild._native:NativeSqlTestChainRequest", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestChainRequest", + ), + ( + "sqlbuild._native:NativeSqlTestFailure", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestFailure", + ), + ( + "sqlbuild._native:NativeSqlTestPlanningError", + "sqlbuild.compiler.planner.exceptions:NativeSqlTestPlanningError", + ), + ( + "sqlbuild._native:NativeSqlTestPlanningRequest", + "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestPlanningRequest", + ), + ( + "sqlbuild._native:NativeStageMismatchError", + "sqlbuild.compiler.frontier.exceptions:NativeStageMismatchError", + ), + ( + "sqlbuild._native:NonHermeticRuleError", + "sqlbuild.rule_engine.exceptions:NonHermeticRuleError", + ), + ( + "sqlbuild._native:NormalizedDbtProfileConnection", + "sqlbuild.integrations.dbt.models:NormalizedDbtProfileConnection", + ), + ( + "sqlbuild._native:OldNameView", + "sqlbuild.compiler.planner.models:OldNameView", + ), + ( + "sqlbuild._native:OldNameViewAction", + "sqlbuild.compiler.migrations.types:OldNameViewAction", + ), + ( + "sqlbuild._native:OldNameViewDropReason", + "sqlbuild.compiler.migrations.types:OldNameViewDropReason", + ), + ( + "sqlbuild._native:OldNameViewEvent", + "sqlbuild.compiler.migrations.models:OldNameViewEvent", + ), + ( + "sqlbuild._native:OldNameViewEventType", + "sqlbuild.compiler.migrations.types:OldNameViewEventType", + ), + ( + "sqlbuild._native:OldNameViewHistory", + "sqlbuild.compiler.migrations.models:OldNameViewHistory", + ), + ( + "sqlbuild._native:OldNameViewPlanEntry", + "sqlbuild.compiler.planner.models:OldNameViewPlanEntry", + ), + ( + "sqlbuild._native:OldNameViewStatus", + "sqlbuild.compiler.migrations.types:OldNameViewStatus", + ), + ( + "sqlbuild._native:OnSchemaChange", + "sqlbuild.compiler.planner.types:OnSchemaChange", + ), + ( + "sqlbuild._native:OpaqueModuleStateError", + "sqlbuild.rule_engine.exceptions:OpaqueModuleStateError", + ), + ( + "sqlbuild._native:OwnershipRoot", + "sqlbuild.compiler.scopes.models:OwnershipRoot", + ), + ( + "sqlbuild._native:OwnershipRootKind", + "sqlbuild.compiler.scopes.types:OwnershipRootKind", + ), + ( + "sqlbuild._native:ParsedSelector", + "sqlbuild.compiler.planner.models:ParsedSelector", + ), + ( + "sqlbuild._native:PathSelector", + "sqlbuild.compiler.planner.models:PathSelector", + ), + ( + "sqlbuild._native:PendingLintBody", + "sqlbuild.lint.models:PendingLintBody", + ), + ( + "sqlbuild._native:PlanAction", + "sqlbuild.compiler.planner.types:PlanAction", + ), + ( + "sqlbuild._native:PlanOutput", + "sqlbuild.compiler.planner.models:PlanOutput", + ), + ( + "sqlbuild._native:PlanReason", + "sqlbuild.compiler.planner.types:PlanReason", + ), + ( + "sqlbuild._native:PlannerInputError", + "sqlbuild.compiler.planner.exceptions:PlannerInputError", + ), + ( + "sqlbuild._native:PlannerOverrides", + "sqlbuild.compiler.planner.models:PlannerOverrides", + ), + ( + "sqlbuild._native:PlannerPolicies", + "sqlbuild.compiler.planner.models:PlannerPolicies", + ), + ( + "sqlbuild._native:PlannerRelationsContext", + "sqlbuild.compiler.planner.models:PlannerRelationsContext", + ), + ( + "sqlbuild._native:PlannerScope", + "sqlbuild.compiler.planner.models:PlannerScope", + ), + ( + "sqlbuild._native:PlannerSelection", + "sqlbuild.compiler.planner.models:PlannerSelection", + ), + ( + "sqlbuild._native:PolyglotAnalysisDialect", + "sqlbuild.compiler.lineage.types:PolyglotAnalysisDialect", + ), + ( + "sqlbuild._native:PolyglotAnalysisResult", + "sqlbuild.compiler.compile.models:PolyglotAnalysisResult", + ), + ( + "sqlbuild._native:PreparedLintFiles", + "sqlbuild.lint.models:PreparedLintFiles", + ), + ( + "sqlbuild._native:PreparedSqlLint", + "sqlbuild.rule_engine.models:PreparedSqlLint", + ), + ( + "sqlbuild._native:PreparedSqlLintResult", + "sqlbuild.rule_engine.models:PreparedSqlLintResult", + ), + ( + "sqlbuild._native:Project", + "sqlbuild.rule_engine.models:Project", + ), + ( + "sqlbuild._native:ProjectCompileError", + "sqlbuild.lint.exceptions:ProjectCompileError", + ), + ( + "sqlbuild._native:ProjectConfig", + "sqlbuild.spec.contracts.models:ProjectConfig", + ), + ( + "sqlbuild._native:ProjectConfigError", + "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", + ), + ( + "sqlbuild._native:ProjectGraph", + "sqlbuild.compiler.pipeline.models:ProjectGraph", + ), + ( + "sqlbuild._native:ProjectPath", + "sqlbuild.rule_engine.models:ProjectPath", + ), + ( + "sqlbuild._native:ProjectPathError", + "sqlbuild.compiler.discovery.exceptions:ProjectPathError", + ), + ( + "sqlbuild._native:ProjectPythonPathError", + "sqlbuild.compiler.discovery.exceptions:ProjectPythonPathError", + ), + ( + "sqlbuild._native:ProviderDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", + ), + ( + "sqlbuild._native:PythonHookEntry", + "sqlbuild.compiler.discovery.models:PythonHookEntry", + ), + ( + "sqlbuild._native:PythonIdentityDependency", + "sqlbuild.compiler.python_nodes.models:PythonIdentityDependency", + ), + ( + "sqlbuild._native:PythonIdentityStatus", + "sqlbuild.compiler.python_nodes.types:PythonIdentityStatus", + ), + ( + "sqlbuild._native:PythonNodeDependencyEdge", + "sqlbuild.compiler.python_nodes.models:PythonNodeDependencyEdge", + ), + ( + "sqlbuild._native:PythonNodeDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:PythonNodeDiscoveryError", + ), + ( + "sqlbuild._native:PythonNodeGraph", + "sqlbuild.compiler.python_nodes.models:PythonNodeGraph", + ), + ( + "sqlbuild._native:PythonNodeIdentity", + "sqlbuild.compiler.python_nodes.models:PythonNodeIdentity", + ), + ( + "sqlbuild._native:PythonPlanEntry", + "sqlbuild.compiler.pipeline.models:PythonPlanEntry", + ), + ( + "sqlbuild._native:PythonRunPhase", + "sqlbuild.compiler.python_nodes.types:PythonRunPhase", + ), + ( + "sqlbuild._native:PythonRunPlanOutputs", + "sqlbuild.compiler.pipeline.models:PythonRunPlanOutputs", + ), + ( + "sqlbuild._native:PythonSqlReferenceReport", + "sqlbuild.compiler.compile.models:PythonSqlReferenceReport", + ), + ( + "sqlbuild._native:PythonSqlRunLifecyclePlan", + "sqlbuild.compiler.python_nodes.models:PythonSqlRunLifecyclePlan", + ), + ( + "sqlbuild._native:PythonSqlRunSelection", + "sqlbuild.compiler.python_nodes.models:PythonSqlRunSelection", + ), + ( + "sqlbuild._native:QualifiedLineageColumn", + "sqlbuild.compiler.lineage.models:QualifiedLineageColumn", + ), + ( + "sqlbuild._native:QueryFingerprintError", + "sqlbuild.compiler.fingerprints.exceptions:QueryFingerprintError", + ), + ( + "sqlbuild._native:RefactorIOError", + "sqlbuild.compiler.refactoring.exceptions:RefactorIOError", + ), + ( + "sqlbuild._native:RefactorProject", + "sqlbuild.compiler.refactoring.models:RefactorProject", + ), + ( + "sqlbuild._native:RefactorValueError", + "sqlbuild.compiler.refactoring.exceptions:RefactorValueError", + ), + ( + "sqlbuild._native:ReferencesConfig", + "sqlbuild.spec.contracts.models:ReferencesConfig", + ), + ( + "sqlbuild._native:RelatedLocation", + "sqlbuild.compiler.compile.models:RelatedLocation", + ), + ( + "sqlbuild._native:ResolvedDbtProfileOutput", + "sqlbuild.integrations.dbt.models:ResolvedDbtProfileOutput", + ), + ( + "sqlbuild._native:ResolvedTableType", + "sqlbuild.spec.contracts.models:ResolvedTableType", + ), + ( + "sqlbuild._native:ResolvedTimeTravelRetention", + "sqlbuild.spec.contracts.models:ResolvedTimeTravelRetention", + ), + ( + "sqlbuild._native:ResourceIdentity", + "sqlbuild.compiler.scopes.models:ResourceIdentity", + ), + ( + "sqlbuild._native:ResourceIdentityError", + "sqlbuild.compiler.resource_names.exceptions:ResourceIdentityError", + ), + ( + "sqlbuild._native:ResourceKind", + "sqlbuild.compiler.scopes.types:ResourceKind", + ), + ( + "sqlbuild._native:ResourceRecord", + "sqlbuild.compiler.scopes.models:ResourceRecord", + ), + ( + "sqlbuild._native:RetentionDecreasePolicy", + "sqlbuild.spec.contracts.types:RetentionDecreasePolicy", + ), + ( + "sqlbuild._native:RetentionDirection", + "sqlbuild.compiler.planner.types:RetentionDirection", + ), + ( + "sqlbuild._native:RetentionPlanEntry", + "sqlbuild.compiler.planner.models:RetentionPlanEntry", + ), + ( + "sqlbuild._native:RetentionPlanPhase", + "sqlbuild.compiler.planner.types:RetentionPlanPhase", + ), + ("sqlbuild._native:Rule", "sqlbuild.rule_engine.models:Rule"), + ( + "sqlbuild._native:RuleAssertionError", + "sqlbuild.rule_engine.exceptions:RuleAssertionError", + ), + ( + "sqlbuild._native:RuleCase", + "sqlbuild.rule_engine.models:RuleCase", + ), + ( + "sqlbuild._native:RuleExemption", + "sqlbuild.rule_engine.models:RuleExemption", + ), + ( + "sqlbuild._native:RuleFactViews", + "sqlbuild.rule_engine.models:RuleFactViews", + ), + ( + "sqlbuild._native:RuleFile", + "sqlbuild.rule_engine.models:RuleFile", + ), + ( + "sqlbuild._native:RuleFixResult", + "sqlbuild.lint.models:RuleFixResult", + ), + ( + "sqlbuild._native:RuleGuidance", + "sqlbuild.rule_engine.models:RuleGuidance", + ), + ( + "sqlbuild._native:RuleIgnore", + "sqlbuild.rule_engine.models:RuleIgnore", + ), + ( + "sqlbuild._native:RuleOption", + "sqlbuild.rule_engine.models:RuleOption", + ), + ( + "sqlbuild._native:RuleResult", + "sqlbuild.rule_engine.models:RuleResult", + ), + ( + "sqlbuild._native:RuleRunPlan", + "sqlbuild.rule_engine.models:RuleRunPlan", + ), + ( + "sqlbuild._native:RuleSubject", + "sqlbuild.rule_engine.types:RuleSubject", + ), + ( + "sqlbuild._native:RuleUsageError", + "sqlbuild.rule_engine.exceptions:RuleUsageError", + ), + ( + "sqlbuild._native:RulesCacheConfig", + "sqlbuild.rule_engine.models:RulesCacheConfig", + ), + ( + "sqlbuild._native:RulesConfig", + "sqlbuild.rule_engine.models:RulesConfig", + ), + ( + "sqlbuild._native:RulesConfigError", + "sqlbuild.rule_engine.exceptions:RulesConfigError", + ), + ( + "sqlbuild._native:RulesError", + "sqlbuild.rule_engine.exceptions:RulesError", + ), + ( + "sqlbuild._native:RulesResult", + "sqlbuild.rule_engine.models:RulesResult", + ), + ( + "sqlbuild._native:RulesRunResult", + "sqlbuild.rule_engine.models:RulesRunResult", + ), + ( + "sqlbuild._native:ScenarioArtifactKind", + "sqlbuild.compiler.planner.types:ScenarioArtifactKind", + ), + ( + "sqlbuild._native:ScenarioAssertionExpectationPlan", + "sqlbuild.compiler.planner.models:ScenarioAssertionExpectationPlan", + ), + ( + "sqlbuild._native:ScenarioConfig", + "sqlbuild.spec.contracts.models:ScenarioConfig", + ), + ( + "sqlbuild._native:ScenarioExecutionPlan", + "sqlbuild.compiler.planner.models:ScenarioExecutionPlan", + ), + ( + "sqlbuild._native:ScenarioExpectedExpectationPlan", + "sqlbuild.compiler.planner.models:ScenarioExpectedExpectationPlan", + ), + ( + "sqlbuild._native:ScenarioFixturePlan", + "sqlbuild.compiler.planner.models:ScenarioFixturePlan", + ), + ( + "sqlbuild._native:ScenarioRelationMap", + "sqlbuild.compiler.planner.models:ScenarioRelationMap", + ), + ( + "sqlbuild._native:ScenarioRelationPlan", + "sqlbuild.compiler.planner.models:ScenarioRelationPlan", + ), + ( + "sqlbuild._native:ScenarioSnapshotLimitsConfig", + "sqlbuild.spec.contracts.models:ScenarioSnapshotLimitsConfig", + ), + ( + "sqlbuild._native:SchemaAuditInstance", + "sqlbuild.spec.contracts.models:SchemaAuditInstance", + ), + ( + "sqlbuild._native:SchemaColumn", + "sqlbuild.spec.contracts.models:SchemaColumn", + ), + ( + "sqlbuild._native:SchemaDynamicColumnFamily", + "sqlbuild.spec.contracts.models:SchemaDynamicColumnFamily", + ), + ( + "sqlbuild._native:SchemaFinding", + "sqlbuild.compiler.planner.models:SchemaFinding", + ), + ( + "sqlbuild._native:SchemaModelEntry", + "sqlbuild.spec.contracts.models:SchemaModelEntry", + ), + ( + "sqlbuild._native:SchemaParseError", + "sqlbuild.compiler.discovery.exceptions:SchemaParseError", + ), + ( + "sqlbuild._native:SchemaSeedEntry", + "sqlbuild.spec.contracts.models:SchemaSeedEntry", + ), + ( + "sqlbuild._native:ScopeBrowseResult", + "sqlbuild.compiler.scopes.models:ScopeBrowseResult", + ), + ( + "sqlbuild._native:ScopeChainEntry", + "sqlbuild.compiler.scopes.models:ScopeChainEntry", + ), + ( + "sqlbuild._native:ScopeCompleteness", + "sqlbuild.compiler.scopes.models:ScopeCompleteness", + ), + ( + "sqlbuild._native:ScopeDiagnostic", + "sqlbuild.compiler.scopes.models:ScopeDiagnostic", + ), + ( + "sqlbuild._native:ScopeDiagnosticCode", + "sqlbuild.compiler.scopes.types:ScopeDiagnosticCode", + ), + ( + "sqlbuild._native:ScopeDiagnosticSeverity", + "sqlbuild.compiler.scopes.types:DiagnosticSeverity", + ), + ( + "sqlbuild._native:ScopeError", + "sqlbuild.compiler.scopes.exceptions:ScopeError", + ), + ( + "sqlbuild._native:ScopeFolder", + "sqlbuild.compiler.scopes.models:ScopeFolder", + ), + ( + "sqlbuild._native:ScopeIndex", + "sqlbuild.compiler.scopes.models:ScopeIndex", + ), + ( + "sqlbuild._native:ScopeKind", + "sqlbuild.compiler.scopes.types:ScopeKind", + ), + ( + "sqlbuild._native:ScopeListResult", + "sqlbuild.compiler.scopes.models:ScopeListResult", + ), + ( + "sqlbuild._native:ScopeLookup", + "sqlbuild.compiler.scopes.models:ScopeLookup", + ), + ( + "sqlbuild._native:ScopeReport", + "sqlbuild.compiler.scopes.models:ScopeReport", + ), + ( + "sqlbuild._native:ScopeReportFilters", + "sqlbuild.compiler.scopes.models:ScopeReportFilters", + ), + ( + "sqlbuild._native:ScopeResourceReport", + "sqlbuild.compiler.scopes.models:ScopeResourceReport", + ), + ( + "sqlbuild._native:ScopeSection", + "sqlbuild.compiler.scopes.models:ScopeSection", + ), + ( + "sqlbuild._native:ScopeSourceLocation", + "sqlbuild.compiler.scopes.models:SourceLocation", + ), + ( + "sqlbuild._native:ScopeTargetQuery", + "sqlbuild.compiler.scopes.models:ScopeTargetQuery", + ), + ( + "sqlbuild._native:ScopeValidationError", + "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", + ), + ( + "sqlbuild._native:ScopesConfig", + "sqlbuild.spec.contracts.models:ScopesConfig", + ), + ( + "sqlbuild._native:SeedCsvSettings", + "sqlbuild.spec.contracts.models:SeedCsvSettings", + ), + ( + "sqlbuild._native:SeedDiscoveryError", + "sqlbuild.compiler.discovery.exceptions:SeedDiscoveryError", + ), + ( + "sqlbuild._native:SeedPlanEntry", + "sqlbuild.compiler.planner.models:SeedPlanEntry", + ), + ( + "sqlbuild._native:SelectStarAllow", + "sqlbuild.rule_engine.models:SelectStarAllow", + ), + ( + "sqlbuild._native:SelectedDbtProfileOutput", + "sqlbuild.integrations.dbt.models:SelectedDbtProfileOutput", + ), + ( + "sqlbuild._native:SelectorKind", + "sqlbuild.compiler.planner.types:SelectorKind", + ), + ( + "sqlbuild._native:SettingsConfig", + "sqlbuild.spec.contracts.models:SettingsConfig", + ), + ( + "sqlbuild._native:SinksConfig", + "sqlbuild.spec.contracts.models:SinksConfig", + ), + ( + "sqlbuild._native:SnapshotSchemaChangePolicy", + "sqlbuild.compiler.planner.types:SnapshotSchemaChangePolicy", + ), + ( + "sqlbuild._native:SnapshotStrategy", + "sqlbuild.compiler.planner.types:SnapshotStrategy", + ), + ( + "sqlbuild._native:SnapshotsConfig", + "sqlbuild.spec.contracts.models:SnapshotsConfig", + ), + ( + "sqlbuild._native:SourceColumnEntry", + "sqlbuild.spec.contracts.models:SourceColumnEntry", + ), + ( + "sqlbuild._native:SourceEntry", + "sqlbuild.spec.contracts.models:SourceEntry", + ), + ( + "sqlbuild._native:SourceFreshnessAgePolicy", + "sqlbuild.spec.contracts.models:SourceFreshnessAgePolicy", + ), + ( + "sqlbuild._native:SourceFreshnessConfig", + "sqlbuild.spec.contracts.models:SourceFreshnessConfig", + ), + ( + "sqlbuild._native:SourceFreshnessRecord", + "sqlbuild.compiler.source_freshness.models:SourceFreshnessRecord", + ), + ( + "sqlbuild._native:SourceFreshnessStrategy", + "sqlbuild.spec.contracts.types:SourceFreshnessStrategy", + ), + ( + "sqlbuild._native:SourceFreshnessValueKind", + "sqlbuild.spec.contracts.types:SourceFreshnessValueKind", + ), + ( + "sqlbuild._native:SourceLoadPlanEntry", + "sqlbuild.compiler.planner.models:SourceLoadPlanEntry", + ), + ( + "sqlbuild._native:SourceLocation", + "sqlbuild.spec.contracts.models:SourceLocation", + ), + ( + "sqlbuild._native:SourceParseError", + "sqlbuild.compiler.discovery.exceptions:SourceParseError", + ), + ( + "sqlbuild._native:SourceWriteStrategy", + "sqlbuild.spec.contracts.types:SourceWriteStrategy", + ), + ( + "sqlbuild._native:SpecConfigError", + "sqlbuild.spec.contracts.exceptions:SpecConfigError", + ), + ( + "sqlbuild._native:SqlAuditParseError", + "sqlbuild.compiler.discovery.exceptions:SqlAuditParseError", + ), + ( + "sqlbuild._native:SqlBindingDiagnostic", + "sqlbuild.compiler.sql_analysis.models:SqlBindingDiagnostic", + ), + ( + "sqlbuild._native:SqlBindingResult", + "sqlbuild.compiler.sql_analysis.models:SqlBindingResult", + ), + ( + "sqlbuild._native:SqlExpansionContext", + "sqlbuild.compiler.compile.models:SqlExpansionContext", + ), + ( + "sqlbuild._native:SqlExpansionReuse", + "sqlbuild.rule_engine.models:SqlExpansionReuse", + ), + ( + "sqlbuild._native:SqlHookEntry", + "sqlbuild.compiler.discovery.models:SqlHookEntry", + ), + ( + "sqlbuild._native:SqlHookParseError", + "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", + ), + ( + "sqlbuild._native:SqlLexicalSyntax", + "sqlbuild.compiler.sql_analysis.models:SqlLexicalSyntax", + ), + ( + "sqlbuild._native:SqlLogicalType", + "sqlbuild.sql_values.models:SqlLogicalType", + ), + ( + "sqlbuild._native:SqlNode", + "sqlbuild.rule_engine.models:SqlNode", + ), + ( + "sqlbuild._native:SqlReferenceKind", + "sqlbuild.compiler.references.types:SqlReferenceKind", + ), + ( + "sqlbuild._native:SqlReferenceScan", + "sqlbuild.compiler.compile.models:SqlReferenceScan", + ), + ( + "sqlbuild._native:SqlRulesEvaluation", + "sqlbuild.rule_engine.models:SqlRulesEvaluation", + ), + ( + "sqlbuild._native:SqlScenarioParseError", + "sqlbuild.compiler.discovery.exceptions:SqlScenarioParseError", + ), + ( + "sqlbuild._native:SqlSchemaValidationRequest", + "sqlbuild.compiler.sql_analysis.models:SqlSchemaValidationRequest", + ), + ( + "sqlbuild._native:SqlTestAssertionStep", + "sqlbuild.compiler.planner.models:SqlTestAssertionStep", + ), + ( + "sqlbuild._native:SqlTestParameterDeclaration", + "sqlbuild.compiler.discovery.models:SqlTestParameterDeclaration", + ), + ( + "sqlbuild._native:SqlTestParseError", + "sqlbuild.compiler.discovery.exceptions:SqlTestParseError", + ), + ( + "sqlbuild._native:SqlTestPlanEntry", + "sqlbuild.compiler.planner.models:SqlTestPlanEntry", + ), + ( + "sqlbuild._native:SqlTestReferenceError", + "sqlbuild.compiler.compile.exceptions:SqlTestReferenceError", + ), + ( + "sqlbuild._native:SqlTestRulesConfig", + "sqlbuild.rule_engine.models:SqlTestRulesConfig", + ), + ( + "sqlbuild._native:SqlTestSelection", + "sqlbuild.compiler.planner.models:SqlTestSelection", + ), + ( + "sqlbuild._native:SqlValue", + "sqlbuild.sql_values.models:SqlValue", + ), + ( + "sqlbuild._native:SqlValueKind", + "sqlbuild.sql_values.types:SqlValueKind", + ), + ( + "sqlbuild._native:StartCursorsConfig", + "sqlbuild.spec.contracts.models:StartCursorsConfig", + ), + ( + "sqlbuild._native:StateSqlValueType", + "sqlbuild.sql_values.types:StateSqlValueType", + ), + ( + "sqlbuild._native:StaticCommandContext", + "sqlbuild.compiler.pipeline.models:StaticCommandContext", + ), + ( + "sqlbuild._native:StaticMacroExport", + "sqlbuild.compiler.compile.models:StaticMacroExport", + ), + ( + "sqlbuild._native:StaticMacroFault", + "sqlbuild.compiler.compile.models:StaticMacroFault", + ), + ( + "sqlbuild._native:StaticMacroInventory", + "sqlbuild.compiler.compile.models:StaticMacroInventory", + ), + ( + "sqlbuild._native:TableType", + "sqlbuild.spec.contracts.types:TableType", + ), + ( + "sqlbuild._native:TableTypeDowngradePolicy", + "sqlbuild.spec.contracts.types:TableTypeDowngradePolicy", + ), + ( + "sqlbuild._native:TableTypePlanEntry", + "sqlbuild.compiler.planner.models:TableTypePlanEntry", + ), + ( + "sqlbuild._native:TableTypeSource", + "sqlbuild.spec.contracts.types:TableTypeSource", + ), + ( + "sqlbuild._native:TargetConfig", + "sqlbuild.spec.contracts.models:TargetConfig", + ), + ( + "sqlbuild._native:TargetDiffConfig", + "sqlbuild.spec.contracts.models:TargetDiffConfig", + ), + ( + "sqlbuild._native:TargetWarehousesConfig", + "sqlbuild.spec.contracts.models:TargetWarehousesConfig", + ), + ( + "sqlbuild._native:ThresholdOverride", + "sqlbuild.rule_engine.models:ThresholdOverride", + ), + ( + "sqlbuild._native:TimeTravelRetentionSource", + "sqlbuild.spec.contracts.types:TimeTravelRetentionSource", + ), + ( + "sqlbuild._native:TimestampValue", + "sqlbuild.cursor_algebra.models:TimestampValue", + ), + ( + "sqlbuild._native:TolerantScopeDiscovery", + "sqlbuild.compiler.discovery.models:TolerantScopeDiscovery", + ), + ( + "sqlbuild._native:UnindexableScopeError", + "sqlbuild.compiler.scopes.exceptions:UnindexableScopeError", + ), + ( + "sqlbuild._native:UnknownLineageDialectError", + "sqlbuild.compiler.lineage.exceptions:UnknownLineageDialectError", + ), + ( + "sqlbuild._native:UnsupportedLexicalSyntaxError", + "sqlbuild.compiler.sql_analysis.exceptions:UnsupportedLexicalSyntaxError", + ), + ( + "sqlbuild._native:UsageKind", + "sqlbuild.compiler.scopes.types:UsageKind", + ), + ( + "sqlbuild._native:UsageRecord", + "sqlbuild.compiler.scopes.models:UsageRecord", + ), + ( + "sqlbuild._native:VisibilityProvenance", + "sqlbuild.compiler.scopes.models:VisibilityProvenance", + ), + ( + "sqlbuild._native:VisibilityReason", + "sqlbuild.compiler.scopes.types:VisibilityReason", + ), + ( + "sqlbuild._native:VisibilityRecord", + "sqlbuild.compiler.scopes.models:VisibilityRecord", + ), + ( + "sqlbuild._native:VisibilityResolution", + "sqlbuild.compiler.scopes.models:VisibilityResolution", + ), + ( + "sqlbuild._native:VisibleModelDeclarations", + "sqlbuild.compiler.compile.models:VisibleModelDeclarations", + ), + ( + "sqlbuild._native:WarehouseGroup", + "sqlbuild.spec.contracts.types:WarehouseGroup", + ), +]; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/errors.rs b/crates/sqlbuild-cache/src/hosted_shapes/errors.rs new file mode 100644 index 0000000000..0f1f6ce65b --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/errors.rs @@ -0,0 +1,5 @@ +//! Why a Python shape name could not be resolved to one native class. + +/// More than one natively hosted class claims the same Python shape name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AmbiguousShapeName; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs b/crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs new file mode 100644 index 0000000000..62473e0934 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/main/hosted_native_name.rs @@ -0,0 +1,28 @@ +//! The native class hosting one Python shape, found by its module and class name. + +use crate::hosted_shapes::constants::{HOSTED_SHAPE_TYPES, NATIVE_MODULE}; +use crate::hosted_shapes::errors::AmbiguousShapeName; + +/// The native class name hosting `module:class`, or `None` when the shape is still Python. +pub fn hosted_native_name( + module: &str, + class: &str, +) -> Result, AmbiguousShapeName> { + let mut found: Option<&'static str> = None; + for (native, python) in HOSTED_SHAPE_TYPES { + if python.split_once(':') != Some((module, class)) { + continue; + } + let Some(native_name) = native + .strip_prefix(NATIVE_MODULE) + .and_then(|rest| rest.strip_prefix(':')) + else { + continue; + }; + if found.is_some_and(|existing| existing != native_name) { + return Err(AmbiguousShapeName); + } + found = Some(native_name); + } + Ok(found) +} diff --git a/crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs b/crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs new file mode 100644 index 0000000000..99b510965b --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/main/mod.rs @@ -0,0 +1 @@ +pub mod hosted_native_name; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/mod.rs b/crates/sqlbuild-cache/src/hosted_shapes/mod.rs new file mode 100644 index 0000000000..45be7c1b09 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/mod.rs @@ -0,0 +1,8 @@ +//! Natively hosted compiler shapes and the Python names they are captured and resolved as. + +pub mod constants; +pub mod errors; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests.rs new file mode 100644 index 0000000000..94b69ddd59 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_hosted_shape_types.rs"] +mod hosted_shape_types; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs new file mode 100644 index 0000000000..a6bf6a941c --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests/helpers.rs @@ -0,0 +1,10 @@ +use std::collections::HashSet; + +/// The keys of `keys` seen more than once, in first-repeat order. +pub(super) fn duplicates(keys: &[&'static str]) -> Vec<&'static str> { + let mut seen: HashSet<&str> = HashSet::new(); + keys.iter() + .copied() + .filter(|key| !seen.insert(key)) + .collect() +} diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs new file mode 100644 index 0000000000..8070b1c832 --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_hosted_shape_types.rs @@ -0,0 +1,63 @@ +use crate::hosted_shapes::constants::HOSTED_SHAPE_TYPES; +use crate::hosted_shapes::main::hosted_native_name::hosted_native_name; +use crate::hosted_shapes::tests::helpers::duplicates; +use crate::hosted_shapes::tests::test_types::{HostedKeyTestCase, NativeNameTestCase}; + +#[test] +fn given_hosted_shape_table_when_keyed_then_every_native_and_python_name_is_unique() { + let test_cases = [ + HostedKeyTestCase { + description: "native class names", + keys: HOSTED_SHAPE_TYPES + .iter() + .map(|(native, _)| *native) + .collect(), + expected_duplicates: Vec::new(), + }, + HostedKeyTestCase { + description: "qualified Python shape names", + keys: HOSTED_SHAPE_TYPES + .iter() + .map(|(_, python)| *python) + .collect(), + expected_duplicates: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + duplicates(&test_case.keys), + test_case.expected_duplicates, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_python_shape_when_resolving_then_only_its_own_module_selects_the_native_class() { + let test_cases = [ + NativeNameTestCase { + description: "a scope severity resolves to the scope enum", + shape: ("sqlbuild.compiler.scopes.types", "DiagnosticSeverity"), + expected_native: Some("ScopeDiagnosticSeverity"), + }, + NativeNameTestCase { + description: "a shape name from another module is not hosted", + shape: ("sqlbuild.compiler.compile.models", "DiagnosticSeverity"), + expected_native: None, + }, + NativeNameTestCase { + description: "a hosted planner record resolves by its own name", + shape: ("sqlbuild.compiler.planner.models", "PlanOutput"), + expected_native: Some("PlanOutput"), + }, + ]; + for test_case in test_cases { + assert_eq!( + hosted_native_name(test_case.shape.0, test_case.shape.1), + Ok(test_case.expected_native), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs new file mode 100644 index 0000000000..68ad02324d --- /dev/null +++ b/crates/sqlbuild-cache/src/hosted_shapes/tests/test_types.rs @@ -0,0 +1,11 @@ +pub(crate) struct HostedKeyTestCase { + pub(crate) description: &'static str, + pub(crate) keys: Vec<&'static str>, + pub(crate) expected_duplicates: Vec<&'static str>, +} + +pub(crate) struct NativeNameTestCase { + pub(crate) description: &'static str, + pub(crate) shape: (&'static str, &'static str), + pub(crate) expected_native: Option<&'static str>, +} diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs index 679ef09814..6899e432d6 100644 --- a/crates/sqlbuild-cache/src/lib.rs +++ b/crates/sqlbuild-cache/src/lib.rs @@ -4,9 +4,11 @@ pub mod artifacts; pub mod digest; +pub mod hosted_shapes; pub mod macro_store_keys; pub mod module_digests; pub mod project_reuse; pub mod project_snapshot; pub mod stage_capture; +pub mod startup_inputs; pub mod store; diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index f9d5b74f87..a3877a527a 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -382,852 +382,3 @@ pub const CANONICAL_TYPES: [(&str, &str); 11] = [ ]; /// Sequence types captured as their items. pub const DECODED_SEQUENCES: [&str; 1] = ["sqlbuild.compiler.compile.models:CompactLineageFacts"]; -/// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 213] = [ - ( - "sqlbuild._native:AnalysisCacheContext", - "sqlbuild.compiler.compile.models:AnalysisCacheContext", - ), - ( - "sqlbuild._native:AttachedAuditGateEdge", - "sqlbuild.compiler.graph.models:AttachedAuditGateEdge", - ), - ( - "sqlbuild._native:ClonePipelineConnection", - "sqlbuild.compiler.pipeline.models:ClonePipelineConnection", - ), - ( - "sqlbuild._native:ClonePipelineOptions", - "sqlbuild.compiler.pipeline.models:ClonePipelineOptions", - ), - ( - "sqlbuild._native:ClonePipelineResult", - "sqlbuild.compiler.pipeline.models:ClonePipelineResult", - ), - ( - "sqlbuild._native:CollectionRendering", - "sqlbuild.sql_values.types:CollectionRendering", - ), - ( - "sqlbuild._native:ColumnLineage", - "sqlbuild.compiler.lineage.models:ColumnLineage", - ), - ( - "sqlbuild._native:ColumnLineageConfidence", - "sqlbuild.compiler.lineage.types:ColumnLineageConfidence", - ), - ( - "sqlbuild._native:ColumnLineageEdge", - "sqlbuild.compiler.lineage.models:ColumnLineageEdge", - ), - ( - "sqlbuild._native:ColumnLineageMode", - "sqlbuild.compiler.lineage.types:ColumnLineageMode", - ), - ( - "sqlbuild._native:ColumnLineageSource", - "sqlbuild.compiler.lineage.models:ColumnLineageSource", - ), - ( - "sqlbuild._native:ColumnTransformKind", - "sqlbuild.compiler.lineage.types:ColumnTransformKind", - ), - ( - "sqlbuild._native:CompileAdapterContext", - "sqlbuild.compiler.compile.models:CompileAdapterContext", - ), - ( - "sqlbuild._native:CompileAnalysisSelection", - "sqlbuild.compiler.compile.models:CompileAnalysisSelection", - ), - ( - "sqlbuild._native:CompileDirectLogicSqlTestInputPayload", - "sqlbuild.compiler.compile.models:CompileDirectLogicSqlTestInputPayload", - ), - ( - "sqlbuild._native:CompileFacts", - "sqlbuild.lint.models:CompileFacts", - ), - ( - "sqlbuild._native:CompileModelConfig", - "sqlbuild.compiler.compile.models:CompileModelConfig", - ), - ( - "sqlbuild._native:CompileModelSqlTestInputPayload", - "sqlbuild.compiler.compile.models:CompileModelSqlTestInputPayload", - ), - ( - "sqlbuild._native:CompilePipelineOptions", - "sqlbuild.compiler.pipeline.models:CompilePipelineOptions", - ), - ( - "sqlbuild._native:CompileProjectInputs", - "sqlbuild.compiler.compile.models:CompileProjectInputs", - ), - ( - "sqlbuild._native:CompileSqlScenarioCte", - "sqlbuild.compiler.compile.models:CompileSqlScenarioCte", - ), - ( - "sqlbuild._native:CompileSqlScenarioInput", - "sqlbuild.compiler.compile.models:CompileSqlScenarioInput", - ), - ( - "sqlbuild._native:CompileSqlTestCte", - "sqlbuild.compiler.compile.models:CompileSqlTestCte", - ), - ( - "sqlbuild._native:CompileSqlTestInput", - "sqlbuild.compiler.compile.models:CompileSqlTestInput", - ), - ( - "sqlbuild._native:CompiledDirectLogicSqlTestPayload", - "sqlbuild.compiler.compile.models:CompiledDirectLogicSqlTestPayload", - ), - ( - "sqlbuild._native:CompiledFunction", - "sqlbuild.compiler.compile.models:CompiledFunction", - ), - ( - "sqlbuild._native:CompiledLineageColumnFact", - "sqlbuild.compiler.compile.models:CompiledLineageColumnFact", - ), - ( - "sqlbuild._native:CompiledModelSqlTestPayload", - "sqlbuild.compiler.compile.models:CompiledModelSqlTestPayload", - ), - ( - "sqlbuild._native:CompiledProjectPhaseResult", - "sqlbuild.compiler.pipeline.models:CompiledProjectPhaseResult", - ), - ( - "sqlbuild._native:CompiledSeed", - "sqlbuild.compiler.compile.models:CompiledSeed", - ), - ( - "sqlbuild._native:CompiledSource", - "sqlbuild.compiler.compile.models:CompiledSource", - ), - ( - "sqlbuild._native:CompiledSqlExpansion", - "sqlbuild.compiler.compile.models:CompiledSqlExpansion", - ), - ( - "sqlbuild._native:CompiledSqlTestResource", - "sqlbuild.compiler.compile.models:CompiledSqlTestResource", - ), - ( - "sqlbuild._native:CompilerStage", - "sqlbuild.compiler.frontier.types:CompilerStage", - ), - ( - "sqlbuild._native:ConfigValueTypeError", - "sqlbuild.spec.contracts.exceptions:ConfigValueTypeError", - ), - ( - "sqlbuild._native:ContractAdoptionError", - "sqlbuild.compiler.contract_adoption.exceptions:ContractAdoptionError", - ), - ( - "sqlbuild._native:ContractAdoptionResult", - "sqlbuild.compiler.contract_adoption.models:ContractAdoptionResult", - ), - ( - "sqlbuild._native:ContractEvidence", - "sqlbuild.compiler.contract_adoption.models:ContractEvidence", - ), - ( - "sqlbuild._native:ContractFinding", - "sqlbuild.compiler.contract_adoption.models:ContractFinding", - ), - ( - "sqlbuild._native:ContractFindingKind", - "sqlbuild.compiler.contract_adoption.types:ContractFindingKind", - ), - ( - "sqlbuild._native:ContractValidationResult", - "sqlbuild.compiler.contracts.models:ContractValidationResult", - ), - ( - "sqlbuild._native:CursorInputRoles", - "sqlbuild.compiler.compile.models:CursorInputRoles", - ), - ( - "sqlbuild._native:CustomRulesOutcome", - "sqlbuild.rule_engine.models:CustomRulesOutcome", - ), - ( - "sqlbuild._native:DbtCliOptions", - "sqlbuild.integrations.dbt.models:DbtCliOptions", - ), - ( - "sqlbuild._native:DbtCommandResult", - "sqlbuild.integrations.dbt.models:DbtCommandResult", - ), - ( - "sqlbuild._native:DbtInitProgressCallbacks", - "sqlbuild.integrations.dbt.models:DbtInitProgressCallbacks", - ), - ( - "sqlbuild._native:DbtInitRequest", - "sqlbuild.integrations.dbt.models:DbtInitRequest", - ), - ( - "sqlbuild._native:DbtInitResult", - "sqlbuild.integrations.dbt.models:DbtInitResult", - ), - ( - "sqlbuild._native:DbtInteropArgumentError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropArgumentError", - ), - ( - "sqlbuild._native:DbtInteropConfigError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropConfigError", - ), - ( - "sqlbuild._native:DbtInteropError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropError", - ), - ( - "sqlbuild._native:DbtInteropRuntimeError", - "sqlbuild.integrations.dbt.exceptions:DbtInteropRuntimeError", - ), - ( - "sqlbuild._native:DbtLsResult", - "sqlbuild.integrations.dbt.models:DbtLsResult", - ), - ( - "sqlbuild._native:DbtNodeExecutionResult", - "sqlbuild.integrations.dbt.models:DbtNodeExecutionResult", - ), - ( - "sqlbuild._native:DeclarationExpansionContext", - "sqlbuild.compiler.compile.models:DeclarationExpansionContext", - ), - ( - "sqlbuild._native:DeclarationParseError", - "sqlbuild.compiler.discovery.exceptions:DeclarationParseError", - ), - ( - "sqlbuild._native:DeclarationRuntimeProjection", - "sqlbuild.compiler.compile.models:DeclarationRuntimeProjection", - ), - ( - "sqlbuild._native:DeclarationScopeBuild", - "sqlbuild.compiler.compile.models:DeclarationScopeBuild", - ), - ( - "sqlbuild._native:DeclarationScopeResolver", - "sqlbuild.compiler.compile.models:DeclarationScopeResolver", - ), - ( - "sqlbuild._native:DeferralInputs", - "sqlbuild.compiler.planner.models:DeferralInputs", - ), - ( - "sqlbuild._native:DiagnosticPhase", - "sqlbuild.compiler.compile.types:DiagnosticPhase", - ), - ( - "sqlbuild._native:DiagnosticSeverity", - "sqlbuild.compiler.compile.types:DiagnosticSeverity", - ), - ( - "sqlbuild._native:DirectSemanticColumnUse", - "sqlbuild.compiler.lineage.models:DirectSemanticColumnUse", - ), - ( - "sqlbuild._native:DirectSourceFreshnessPlanningResult", - "sqlbuild.compiler.source_freshness.models:DirectSourceFreshnessPlanningResult", - ), - ( - "sqlbuild._native:DiscoveredAdapterFile", - "sqlbuild.compiler.discovery.models:DiscoveredAdapterFile", - ), - ( - "sqlbuild._native:DiscoveredAuditBlock", - "sqlbuild.compiler.discovery.models:DiscoveredAuditBlock", - ), - ( - "sqlbuild._native:DiscoveredAuditFile", - "sqlbuild.compiler.discovery.models:DiscoveredAuditFile", - ), - ( - "sqlbuild._native:DiscoveredConstantFile", - "sqlbuild.compiler.discovery.models:DiscoveredConstantFile", - ), - ( - "sqlbuild._native:DiscoveredEnumFile", - "sqlbuild.compiler.discovery.models:DiscoveredEnumFile", - ), - ( - "sqlbuild._native:DiscoveredModelSchemaFile", - "sqlbuild.compiler.discovery.models:DiscoveredModelSchemaFile", - ), - ( - "sqlbuild._native:DiscoveredPythonAssetMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonAssetMetadata", - ), - ( - "sqlbuild._native:DiscoveredPythonCheckMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonCheckMetadata", - ), - ( - "sqlbuild._native:DiscoveredPythonFunctionFile", - "sqlbuild.compiler.discovery.models:DiscoveredPythonFunctionFile", - ), - ( - "sqlbuild._native:DiscoveredPythonLoaderMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonLoaderMetadata", - ), - ( - "sqlbuild._native:DiscoveredPythonTaskMetadata", - "sqlbuild.compiler.python_nodes.models:DiscoveredPythonTaskMetadata", - ), - ( - "sqlbuild._native:DiscoveredSchemaFile", - "sqlbuild.compiler.discovery.models:DiscoveredSchemaFile", - ), - ( - "sqlbuild._native:DiscoveredSeedFile", - "sqlbuild.compiler.discovery.models:DiscoveredSeedFile", - ), - ( - "sqlbuild._native:DiscoveredSqlFunctionFile", - "sqlbuild.compiler.discovery.models:DiscoveredSqlFunctionFile", - ), - ( - "sqlbuild._native:DiscoveredSqlHookFile", - "sqlbuild.compiler.discovery.models:DiscoveredSqlHookFile", - ), - ( - "sqlbuild._native:DiscoveredSqlTestBlock", - "sqlbuild.compiler.discovery.models:DiscoveredSqlTestBlock", - ), - ( - "sqlbuild._native:DiscoveredSqlTestCase", - "sqlbuild.compiler.discovery.models:DiscoveredSqlTestCase", - ), - ( - "sqlbuild._native:DiscoveryConflictError", - "sqlbuild.compiler.discovery.exceptions:DiscoveryConflictError", - ), - ( - "sqlbuild._native:DiscoveryError", - "sqlbuild.compiler.discovery.exceptions:DiscoveryError", - ), - ( - "sqlbuild._native:DiscoveryFileFault", - "sqlbuild.compiler.discovery.models:DiscoveryFileFault", - ), - ( - "sqlbuild._native:DisplayOptions", - "sqlbuild.presentation.models:DisplayOptions", - ), - ( - "sqlbuild._native:DynamicColumnContractProof", - "sqlbuild.compiler.compile.models:DynamicColumnContractProof", - ), - ( - "sqlbuild._native:DynamicColumnFamilyProof", - "sqlbuild.compiler.compile.models:DynamicColumnFamilyProof", - ), - ( - "sqlbuild._native:EnumMember", - "sqlbuild.compiler.discovery.models:EnumMember", - ), - ( - "sqlbuild._native:EventExporterDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:EventExporterDiscoveryError", - ), - ( - "sqlbuild._native:ExpandedLintFile", - "sqlbuild.lint.models:ExpandedLintFile", - ), - ( - "sqlbuild._native:ExpansionSpan", - "sqlbuild.compiler.compile.models:ExpansionSpan", - ), - ( - "sqlbuild._native:FingerprintInputError", - "sqlbuild.compiler.fingerprints.exceptions:FingerprintInputError", - ), - ( - "sqlbuild._native:FixVerdict", - "sqlbuild.lint.models:FixVerdict", - ), - ( - "sqlbuild._native:FixtureColumnInference", - "sqlbuild.compiler.compile.models:FixtureColumnInference", - ), - ( - "sqlbuild._native:FunctionArgument", - "sqlbuild.compiler.compile.models:FunctionArgument", - ), - ( - "sqlbuild._native:FunctionReturnColumn", - "sqlbuild.compiler.compile.models:FunctionReturnColumn", - ), - ( - "sqlbuild._native:GraphEdgeExclusion", - "sqlbuild.rule_engine.models:GraphEdgeExclusion", - ), - ( - "sqlbuild._native:HandBuiltAnalysis", - "sqlbuild.compiler.compile.models:HandBuiltAnalysis", - ), - ( - "sqlbuild._native:HeaderSpan", - "sqlbuild.lint.models:HeaderSpan", - ), - ( - "sqlbuild._native:HookReadEdge", - "sqlbuild.compiler.graph.models:HookReadEdge", - ), - ( - "sqlbuild._native:HookReadType", - "sqlbuild.compiler.graph.types:HookReadType", - ), - ( - "sqlbuild._native:InterpolationSite", - "sqlbuild.lint.models:InterpolationSite", - ), - ( - "sqlbuild._native:InvalidQualifiedIdentityError", - "sqlbuild.compiler.scopes.exceptions:InvalidQualifiedIdentityError", - ), - ( - "sqlbuild._native:InvalidScopePathError", - "sqlbuild.compiler.scopes.exceptions:InvalidScopePathError", - ), - ( - "sqlbuild._native:InvalidSqlReferenceCall", - "sqlbuild.compiler.compile.models:InvalidSqlReferenceCall", - ), - ( - "sqlbuild._native:LineageGraphViews", - "sqlbuild.compiler.graph.models:LineageGraphViews", - ), - ("sqlbuild._native:LintBody", "sqlbuild.lint.models:LintBody"), - ( - "sqlbuild._native:LintConfig", - "sqlbuild.lint.models:LintConfig", - ), - ("sqlbuild._native:LintEdit", "sqlbuild.lint.models:LintEdit"), - ( - "sqlbuild._native:LintError", - "sqlbuild.lint.exceptions:LintError", - ), - ( - "sqlbuild._native:LintFileRole", - "sqlbuild.lint.models:LintFileRole", - ), - ( - "sqlbuild._native:LintRunResult", - "sqlbuild.lint.models:LintRunResult", - ), - ( - "sqlbuild._native:LintSeverity", - "sqlbuild.lint.types:LintSeverity", - ), - ( - "sqlbuild._native:LintStoppedError", - "sqlbuild.lint.exceptions:LintStoppedError", - ), - ( - "sqlbuild._native:LintViolation", - "sqlbuild.lint.models:LintViolation", - ), - ( - "sqlbuild._native:LiteralSqlRelation", - "sqlbuild.compiler.references.models:LiteralSqlRelation", - ), - ( - "sqlbuild._native:MacroExpansionResult", - "sqlbuild.compiler.compile.models:MacroExpansionResult", - ), - ( - "sqlbuild._native:ManualLocation", - "sqlbuild.compiler.refactoring.models:ManualLocation", - ), - ( - "sqlbuild._native:MappedOffset", - "sqlbuild.compiler.compile.models:MappedOffset", - ), - ( - "sqlbuild._native:MeasurementAuditError", - "sqlbuild.compiler.auditing.exceptions:MeasurementAuditError", - ), - ( - "sqlbuild._native:MigrationCompatibility", - "sqlbuild.compiler.migrations.types:MigrationCompatibility", - ), - ( - "sqlbuild._native:MigrationDiscovery", - "sqlbuild.compiler.migrations.types:MigrationDiscovery", - ), - ( - "sqlbuild._native:ModelColumnLineage", - "sqlbuild.compiler.lineage.models:ModelColumnLineage", - ), - ( - "sqlbuild._native:ModelSchemaDeclaration", - "sqlbuild.compiler.discovery.models:ModelSchemaDeclaration", - ), - ( - "sqlbuild._native:ModelSqlAnalysis", - "sqlbuild.compiler.compile.models:ModelSqlAnalysis", - ), - ( - "sqlbuild._native:ModelSqlParseError", - "sqlbuild.compiler.discovery.exceptions:ModelSqlParseError", - ), - ( - "sqlbuild._native:ModelValidationRequest", - "sqlbuild.compiler.compile.models:ModelValidationRequest", - ), - ( - "sqlbuild._native:NamedSqlHookEntry", - "sqlbuild.compiler.discovery.models:NamedSqlHookEntry", - ), - ( - "sqlbuild._native:NativeColumnQuery", - "sqlbuild.compiler.analysis_session.models:NativeColumnQuery", - ), - ( - "sqlbuild._native:NativeLintError", - "sqlbuild.lint.exceptions:NativeLintError", - ), - ( - "sqlbuild._native:NativeModelAnalyses", - "sqlbuild.compiler.analysis_session.models:NativeModelAnalyses", - ), - ( - "sqlbuild._native:NativeModelAnalysisRequest", - "sqlbuild.compiler.analysis_session.models:NativeModelAnalysisRequest", - ), - ( - "sqlbuild._native:NativeModelConfigInputs", - "sqlbuild.compiler.compile.models:NativeModelConfigInputs", - ), - ( - "sqlbuild._native:NativeModelConfigSession", - "sqlbuild.compiler.compile.models:NativeModelConfigSession", - ), - ( - "sqlbuild._native:NativePivotTables", - "sqlbuild.compiler.analysis_session.models:NativePivotTables", - ), - ( - "sqlbuild._native:NativeQueryColumns", - "sqlbuild.compiler.analysis_session.models:NativeQueryColumns", - ), - ( - "sqlbuild._native:NativeRulesEvaluation", - "sqlbuild.rule_engine.models:NativeRulesEvaluation", - ), - ( - "sqlbuild._native:NativeSqlTestAssembly", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestAssembly", - ), - ( - "sqlbuild._native:NativeSqlTestChainRequest", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestChainRequest", - ), - ( - "sqlbuild._native:NativeSqlTestFailure", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestFailure", - ), - ( - "sqlbuild._native:NativeSqlTestPlanningRequest", - "sqlbuild.compiler.sql_test_glue.models:NativeSqlTestPlanningRequest", - ), - ( - "sqlbuild._native:NativeStageMismatchError", - "sqlbuild.compiler.frontier.exceptions:NativeStageMismatchError", - ), - ( - "sqlbuild._native:NormalizedDbtProfileConnection", - "sqlbuild.integrations.dbt.models:NormalizedDbtProfileConnection", - ), - ( - "sqlbuild._native:OldNameViewAction", - "sqlbuild.compiler.migrations.types:OldNameViewAction", - ), - ( - "sqlbuild._native:ParsedSelector", - "sqlbuild.compiler.planner.models:ParsedSelector", - ), - ( - "sqlbuild._native:PathSelector", - "sqlbuild.compiler.planner.models:PathSelector", - ), - ( - "sqlbuild._native:PendingLintBody", - "sqlbuild.lint.models:PendingLintBody", - ), - ( - "sqlbuild._native:PlannerOverrides", - "sqlbuild.compiler.planner.models:PlannerOverrides", - ), - ( - "sqlbuild._native:PlannerPolicies", - "sqlbuild.compiler.planner.models:PlannerPolicies", - ), - ( - "sqlbuild._native:PlannerRelationsContext", - "sqlbuild.compiler.planner.models:PlannerRelationsContext", - ), - ( - "sqlbuild._native:PlannerScope", - "sqlbuild.compiler.planner.models:PlannerScope", - ), - ( - "sqlbuild._native:PlannerSelection", - "sqlbuild.compiler.planner.models:PlannerSelection", - ), - ( - "sqlbuild._native:PolyglotAnalysisDialect", - "sqlbuild.compiler.lineage.types:PolyglotAnalysisDialect", - ), - ( - "sqlbuild._native:PolyglotAnalysisResult", - "sqlbuild.compiler.compile.models:PolyglotAnalysisResult", - ), - ( - "sqlbuild._native:PreparedLintFiles", - "sqlbuild.lint.models:PreparedLintFiles", - ), - ( - "sqlbuild._native:PreparedSqlLint", - "sqlbuild.rule_engine.models:PreparedSqlLint", - ), - ( - "sqlbuild._native:PreparedSqlLintResult", - "sqlbuild.rule_engine.models:PreparedSqlLintResult", - ), - ( - "sqlbuild._native:ProjectCompileError", - "sqlbuild.lint.exceptions:ProjectCompileError", - ), - ( - "sqlbuild._native:ProjectConfigError", - "sqlbuild.compiler.discovery.exceptions:ProjectConfigError", - ), - ( - "sqlbuild._native:ProjectPathError", - "sqlbuild.compiler.discovery.exceptions:ProjectPathError", - ), - ( - "sqlbuild._native:ProjectPythonPathError", - "sqlbuild.compiler.discovery.exceptions:ProjectPythonPathError", - ), - ( - "sqlbuild._native:ProviderDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:ProviderDiscoveryError", - ), - ( - "sqlbuild._native:PythonIdentityStatus", - "sqlbuild.compiler.python_nodes.types:PythonIdentityStatus", - ), - ( - "sqlbuild._native:PythonNodeDependencyEdge", - "sqlbuild.compiler.python_nodes.models:PythonNodeDependencyEdge", - ), - ( - "sqlbuild._native:PythonNodeDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:PythonNodeDiscoveryError", - ), - ( - "sqlbuild._native:PythonPlanEntry", - "sqlbuild.compiler.pipeline.models:PythonPlanEntry", - ), - ( - "sqlbuild._native:PythonRunPhase", - "sqlbuild.compiler.python_nodes.types:PythonRunPhase", - ), - ( - "sqlbuild._native:PythonRunPlanOutputs", - "sqlbuild.compiler.pipeline.models:PythonRunPlanOutputs", - ), - ( - "sqlbuild._native:PythonSqlReferenceReport", - "sqlbuild.compiler.compile.models:PythonSqlReferenceReport", - ), - ( - "sqlbuild._native:PythonSqlRunSelection", - "sqlbuild.compiler.python_nodes.models:PythonSqlRunSelection", - ), - ( - "sqlbuild._native:QualifiedLineageColumn", - "sqlbuild.compiler.lineage.models:QualifiedLineageColumn", - ), - ( - "sqlbuild._native:QueryFingerprintError", - "sqlbuild.compiler.fingerprints.exceptions:QueryFingerprintError", - ), - ( - "sqlbuild._native:RefactorIOError", - "sqlbuild.compiler.refactoring.exceptions:RefactorIOError", - ), - ( - "sqlbuild._native:RefactorProject", - "sqlbuild.compiler.refactoring.models:RefactorProject", - ), - ( - "sqlbuild._native:RefactorValueError", - "sqlbuild.compiler.refactoring.exceptions:RefactorValueError", - ), - ( - "sqlbuild._native:RelatedLocation", - "sqlbuild.compiler.compile.models:RelatedLocation", - ), - ( - "sqlbuild._native:ResourceIdentityError", - "sqlbuild.compiler.resource_names.exceptions:ResourceIdentityError", - ), - ( - "sqlbuild._native:RuleExemption", - "sqlbuild.rule_engine.models:RuleExemption", - ), - ( - "sqlbuild._native:RuleFixResult", - "sqlbuild.lint.models:RuleFixResult", - ), - ( - "sqlbuild._native:RuleGuidance", - "sqlbuild.rule_engine.models:RuleGuidance", - ), - ( - "sqlbuild._native:RuleIgnore", - "sqlbuild.rule_engine.models:RuleIgnore", - ), - ( - "sqlbuild._native:RuleRunPlan", - "sqlbuild.rule_engine.models:RuleRunPlan", - ), - ( - "sqlbuild._native:RulesCacheConfig", - "sqlbuild.rule_engine.models:RulesCacheConfig", - ), - ( - "sqlbuild._native:RulesRunResult", - "sqlbuild.rule_engine.models:RulesRunResult", - ), - ( - "sqlbuild._native:SchemaParseError", - "sqlbuild.compiler.discovery.exceptions:SchemaParseError", - ), - ( - "sqlbuild._native:ScopeError", - "sqlbuild.compiler.scopes.exceptions:ScopeError", - ), - ( - "sqlbuild._native:ScopeValidationError", - "sqlbuild.compiler.scopes.exceptions:ScopeValidationError", - ), - ( - "sqlbuild._native:SeedDiscoveryError", - "sqlbuild.compiler.discovery.exceptions:SeedDiscoveryError", - ), - ( - "sqlbuild._native:SelectStarAllow", - "sqlbuild.rule_engine.models:SelectStarAllow", - ), - ( - "sqlbuild._native:SelectorKind", - "sqlbuild.compiler.planner.types:SelectorKind", - ), - ( - "sqlbuild._native:SourceParseError", - "sqlbuild.compiler.discovery.exceptions:SourceParseError", - ), - ( - "sqlbuild._native:SpecConfigError", - "sqlbuild.spec.contracts.exceptions:SpecConfigError", - ), - ( - "sqlbuild._native:SqlAuditParseError", - "sqlbuild.compiler.discovery.exceptions:SqlAuditParseError", - ), - ( - "sqlbuild._native:SqlBindingDiagnostic", - "sqlbuild.compiler.sql_analysis.models:SqlBindingDiagnostic", - ), - ( - "sqlbuild._native:SqlBindingResult", - "sqlbuild.compiler.sql_analysis.models:SqlBindingResult", - ), - ( - "sqlbuild._native:SqlExpansionContext", - "sqlbuild.compiler.compile.models:SqlExpansionContext", - ), - ( - "sqlbuild._native:SqlExpansionReuse", - "sqlbuild.rule_engine.models:SqlExpansionReuse", - ), - ( - "sqlbuild._native:SqlHookParseError", - "sqlbuild.compiler.discovery.exceptions:SqlHookParseError", - ), - ( - "sqlbuild._native:SqlReferenceScan", - "sqlbuild.compiler.compile.models:SqlReferenceScan", - ), - ( - "sqlbuild._native:SqlRulesEvaluation", - "sqlbuild.rule_engine.models:SqlRulesEvaluation", - ), - ( - "sqlbuild._native:SqlScenarioParseError", - "sqlbuild.compiler.discovery.exceptions:SqlScenarioParseError", - ), - ( - "sqlbuild._native:SqlSchemaValidationRequest", - "sqlbuild.compiler.sql_analysis.models:SqlSchemaValidationRequest", - ), - ( - "sqlbuild._native:SqlTestAssertionStep", - "sqlbuild.compiler.planner.models:SqlTestAssertionStep", - ), - ( - "sqlbuild._native:SqlTestParseError", - "sqlbuild.compiler.discovery.exceptions:SqlTestParseError", - ), - ( - "sqlbuild._native:SqlTestRulesConfig", - "sqlbuild.rule_engine.models:SqlTestRulesConfig", - ), - ( - "sqlbuild._native:SqlTestSelection", - "sqlbuild.compiler.planner.models:SqlTestSelection", - ), - ( - "sqlbuild._native:StaticCommandContext", - "sqlbuild.compiler.pipeline.models:StaticCommandContext", - ), - ( - "sqlbuild._native:ThresholdOverride", - "sqlbuild.rule_engine.models:ThresholdOverride", - ), - ( - "sqlbuild._native:TolerantScopeDiscovery", - "sqlbuild.compiler.discovery.models:TolerantScopeDiscovery", - ), - ( - "sqlbuild._native:UnindexableScopeError", - "sqlbuild.compiler.scopes.exceptions:UnindexableScopeError", - ), - ( - "sqlbuild._native:UnknownLineageDialectError", - "sqlbuild.compiler.lineage.exceptions:UnknownLineageDialectError", - ), - ( - "sqlbuild._native:UnsupportedLexicalSyntaxError", - "sqlbuild.compiler.sql_analysis.exceptions:UnsupportedLexicalSyntaxError", - ), - ( - "sqlbuild._native:VisibleModelDeclarations", - "sqlbuild.compiler.compile.models:VisibleModelDeclarations", - ), -]; diff --git a/crates/sqlbuild-cache/src/stage_capture/main/mod.rs b/crates/sqlbuild-cache/src/stage_capture/main/mod.rs index 96a12d57ab..3af58fac93 100644 --- a/crates/sqlbuild-cache/src/stage_capture/main/mod.rs +++ b/crates/sqlbuild-cache/src/stage_capture/main/mod.rs @@ -1,4 +1,5 @@ pub mod capture_sort_key; pub mod compact_capture_json; pub mod mask_capture_noise; +pub mod next_capture_number; pub mod shared_node_digest; diff --git a/crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs b/crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs new file mode 100644 index 0000000000..5de392957e --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/main/next_capture_number.rs @@ -0,0 +1,21 @@ +//! The sequence number of the next capture written into a capture directory. + +/// Whether `name` is a numbered capture such as `001-discovery.json`. +fn numbered_capture(name: &str) -> bool { + let Some((number, _)) = name.split_once('-') else { + return false; + }; + name.ends_with(".json") + && !number.is_empty() + && number.bytes().all(|byte| byte.is_ascii_digit()) +} + +/// One more than the numbered captures (`NNN-name.json`) among `file_names`. +#[must_use] +pub fn next_capture_number<'a>(file_names: impl IntoIterator) -> u64 { + let written: usize = file_names + .into_iter() + .filter(|name| numbered_capture(name)) + .count(); + u64::try_from(written).map_or(u64::MAX, |count| count.saturating_add(1)) +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests.rs b/crates/sqlbuild-cache/src/stage_capture/tests.rs index 80c6c333f9..b69b5e1b53 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests.rs @@ -4,5 +4,7 @@ mod capture_document; mod helpers; #[path = "tests/test_mask_capture_noise.rs"] mod mask_capture_noise; +#[path = "tests/test_next_capture_number.rs"] +mod next_capture_number; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs new file mode 100644 index 0000000000..c4ed4d04ca --- /dev/null +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_next_capture_number.rs @@ -0,0 +1,36 @@ +use crate::stage_capture::main::next_capture_number::next_capture_number; +use crate::stage_capture::tests::test_types::CaptureNumberTestCase; + +#[test] +fn given_capture_directory_files_when_numbering_then_counts_numbered_captures() { + let test_cases = [ + CaptureNumberTestCase { + description: "an empty directory starts at one", + file_names: &[], + expected_number: 1, + }, + CaptureNumberTestCase { + description: "numbered captures advance the sequence", + file_names: &["001-discovery.json", "002-plan_output.json"], + expected_number: 3, + }, + CaptureNumberTestCase { + description: "other files are ignored", + file_names: &[ + "001-discovery.json", + "notes.txt", + "x-1.json", + ".001-a.json.tmp", + ], + expected_number: 2, + }, + ]; + for test_case in test_cases { + assert_eq!( + next_capture_number(test_case.file_names.iter().copied()), + test_case.expected_number, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs index 9962247a4d..7a1bacf6ba 100644 --- a/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/stage_capture/tests/test_types.rs @@ -18,3 +18,9 @@ pub(crate) struct SortKeyTestCase { pub(crate) item: JsonValue, pub(crate) expected_key: &'static str, } + +pub(crate) struct CaptureNumberTestCase { + pub(crate) description: &'static str, + pub(crate) file_names: &'static [&'static str], + pub(crate) expected_number: u64, +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs b/crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs new file mode 100644 index 0000000000..eb1eb4d0ff --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod pure_paths; diff --git a/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs b/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs new file mode 100644 index 0000000000..9f219cbdab --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/_helpers/pure_paths.rs @@ -0,0 +1,121 @@ +//! `pathlib` spelling, ordering and stat reads for startup stamps on each platform. + +use std::ffi::OsStr; +use std::fs::Metadata; +use std::path::{Path, PathBuf}; + +use crate::startup_inputs::constants::PATH_FILE_SUFFIX; +#[cfg(unix)] +use crate::startup_inputs::constants::{CURRENT, SEPARATOR}; + +/// The root a POSIX path keeps: two leading slashes stay two, more collapse to one. +#[cfg(unix)] +fn root(bytes: &[u8]) -> &[u8] { + match bytes { + [SEPARATOR, SEPARATOR, rest @ ..] if rest.first() != Some(&SEPARATOR) => &bytes[..2], + [SEPARATOR, ..] => &bytes[..1], + _ => &[], + } +} + +/// `str(PurePosixPath(path, *children))`. +#[cfg(unix)] +pub(crate) fn pure(path: &Path, children: &[&OsStr]) -> PathBuf { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let bytes: &[u8] = path.as_os_str().as_bytes(); + let mut text: Vec = root(bytes).to_vec(); + let parts = std::iter::once(bytes) + .chain(children.iter().map(|child| child.as_bytes())) + .flat_map(|segment| segment.split(|byte| *byte == SEPARATOR)) + .filter(|part| !part.is_empty() && *part != CURRENT); + let mut first: bool = true; + for part in parts { + if !first { + text.push(SEPARATOR); + } + text.extend_from_slice(part); + first = false; + } + if text.is_empty() { + text.extend_from_slice(CURRENT); + } + PathBuf::from(std::ffi::OsString::from_vec(text)) +} + +/// `str(PureWindowsPath(path, *children))` through the platform path rules. +#[cfg(not(unix))] +pub(crate) fn pure(path: &Path, children: &[&OsStr]) -> PathBuf { + let mut joined: PathBuf = path.components().collect(); + for child in children { + joined.push(child); + } + joined +} + +/// `str(Path(path).absolute())` against the working directory `cwd`. +pub(crate) fn absolute(path: &Path, cwd: &Path) -> PathBuf { + if path.has_root() && (cfg!(unix) || path.is_absolute()) { + pure(path, &[]) + } else { + pure(cwd, &[path.as_os_str()]) + } +} + +/// The `/`-separated parts `PurePosixPath` orders paths by. +#[cfg(unix)] +pub(crate) fn order_key(path: &Path) -> Vec> { + use std::os::unix::ffi::OsStrExt; + + path.as_os_str() + .as_bytes() + .split(|byte| *byte == SEPARATOR) + .map(<[u8]>::to_vec) + .collect() +} + +/// The case-folded parts `PureWindowsPath` orders paths by. +#[cfg(not(unix))] +pub(crate) fn order_key(path: &Path) -> Vec> { + path.components() + .map(|part| { + part.as_os_str() + .to_string_lossy() + .to_lowercase() + .into_bytes() + }) + .collect() +} + +/// Whether `glob("*.pth")` matches a directory entry name on this platform. +pub(crate) fn is_path_file(name: &OsStr) -> bool { + if cfg!(unix) { + name.as_encoded_bytes().ends_with(PATH_FILE_SUFFIX) + } else { + name.to_string_lossy() + .to_lowercase() + .as_bytes() + .ends_with(PATH_FILE_SUFFIX) + } +} + +/// `os.stat(path).st_mtime_ns`. +#[cfg(unix)] +pub(crate) fn modified_ns(metadata: &Metadata) -> std::io::Result { + use std::os::unix::fs::MetadataExt; + + Ok(metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec()) +} + +/// `os.stat(path).st_mtime_ns` from the platform modification time, negative before 1970. +#[cfg(not(unix))] +pub(crate) fn modified_ns(metadata: &Metadata) -> std::io::Result { + let modified: std::time::SystemTime = metadata.modified()?; + let (nanos, sign): (u128, i64) = match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(elapsed) => (elapsed.as_nanos(), 1), + Err(before) => (before.duration().as_nanos(), -1), + }; + i64::try_from(nanos) + .map(|value| sign * value) + .map_err(std::io::Error::other) +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/constants.rs b/crates/sqlbuild-cache/src/startup_inputs/constants.rs new file mode 100644 index 0000000000..f9f35121d3 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/constants.rs @@ -0,0 +1,10 @@ +//! Names the startup stamps read. + +/// The virtual-environment configuration file of an interpreter prefix. +pub(crate) const VENV_CONFIG: &str = "pyvenv.cfg"; +/// The suffix of the path configuration files `site` processes. +pub(crate) const PATH_FILE_SUFFIX: &[u8] = b".pth"; +#[cfg(unix)] +pub(crate) const SEPARATOR: u8 = b'/'; +#[cfg(unix)] +pub(crate) const CURRENT: &[u8] = b"."; diff --git a/crates/sqlbuild-cache/src/startup_inputs/main/mod.rs b/crates/sqlbuild-cache/src/startup_inputs/main/mod.rs new file mode 100644 index 0000000000..44325627ba --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/main/mod.rs @@ -0,0 +1 @@ +pub mod startup_stamps; diff --git a/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs b/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs new file mode 100644 index 0000000000..8f502c2077 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/main/startup_stamps.rs @@ -0,0 +1,70 @@ +//! Stamp every startup directory, its `.pth` files, the interpreter and its venv configuration. + +use std::collections::BTreeSet; +use std::ffi::{OsStr, OsString}; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::startup_inputs::_helpers::pure_paths::{ + absolute, is_path_file, modified_ns, order_key, pure, +}; +use crate::startup_inputs::constants::VENV_CONFIG; +use crate::startup_inputs::models::{StartupLocations, StartupStamp}; + +/// Sorted stamps of the startup paths; missing paths are unstamped and other stat failures fail. +pub fn startup_stamps( + locations: &StartupLocations, + cwd: &Path, +) -> Result, (PathBuf, io::Error)> { + let directories: BTreeSet = locations + .directories + .iter() + .map(|directory| absolute(directory, cwd).into_os_string()) + .collect(); + let mut paths: BTreeSet = directories.clone(); + paths.extend( + locations + .files + .iter() + .map(|file| pure(file, &[]).into_os_string()), + ); + paths.extend( + locations + .prefixes + .iter() + .map(|prefix| pure(prefix, &[OsStr::new(VENV_CONFIG)]).into_os_string()), + ); + for directory in &directories { + let directory: &Path = Path::new(directory); + let Ok(listing) = std::fs::read_dir(directory) else { + continue; + }; + for entry in listing.flatten() { + let name: OsString = entry.file_name(); + if is_path_file(&name) { + paths.insert(pure(directory, &[name.as_os_str()]).into_os_string()); + } + } + } + let mut ordered: Vec = paths.into_iter().map(PathBuf::from).collect(); + ordered.sort_by_key(|path| order_key(path)); + ordered + .into_iter() + .map(|path| match std::fs::metadata(&path) { + Ok(metadata) => match modified_ns(&metadata) { + Ok(modified) => Ok(StartupStamp { + modified_ns: Some(modified), + size: Some(metadata.len()), + path, + }), + Err(error) => Err((path, error)), + }, + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(StartupStamp { + path, + modified_ns: None, + size: None, + }), + Err(error) => Err((path, error)), + }) + .collect() +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/mod.rs b/crates/sqlbuild-cache/src/startup_inputs/mod.rs new file mode 100644 index 0000000000..ab8bbc79c9 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/mod.rs @@ -0,0 +1,9 @@ +//! Interpreter and site-path stamps that must be unchanged before Python startup is skipped. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/startup_inputs/models.rs b/crates/sqlbuild-cache/src/startup_inputs/models.rs new file mode 100644 index 0000000000..e14607c8d4 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/models.rs @@ -0,0 +1,22 @@ +//! One startup input and its file stamp. + +use std::path::PathBuf; + +/// The interpreter locations the stamps cover. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StartupLocations { + /// Import-path and site-packages directories, made absolute. + pub directories: Vec, + /// Interpreter files, as given. + pub files: Vec, + /// Interpreter prefixes whose `pyvenv.cfg` is stamped. + pub prefixes: Vec, +} + +/// A path with its modification time in nanoseconds and size, both absent when it is missing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StartupStamp { + pub path: PathBuf, + pub modified_ns: Option, + pub size: Option, +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/tests.rs b/crates/sqlbuild-cache/src/startup_inputs/tests.rs new file mode 100644 index 0000000000..7d8de14dac --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_startup_stamps.rs"] +mod startup_stamps; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs b/crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs new file mode 100644 index 0000000000..690d1ce3b5 --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/tests/test_startup_stamps.rs @@ -0,0 +1,66 @@ +use std::path::{Path, PathBuf}; + +use crate::startup_inputs::main::startup_stamps::startup_stamps; +use crate::startup_inputs::models::{StartupLocations, StartupStamp}; +use crate::startup_inputs::tests::test_types::StartupStampsTestCase; + +#[test] +fn given_startup_locations_when_stamped_then_paths_are_absolute_listed_and_sorted_by_parts() { + let test_cases = [StartupStampsTestCase { + description: "relative entries, path files, missing entries and the venv configuration", + directories: vec!["site", "./a//", "a-b", "a/b", "missing"], + expected_stamps: vec![ + ("a", true), + ("a/b", true), + ("a/q.pth", true), + ("a-b", true), + ("a-b/x.pth", true), + ("missing", false), + ("pyvenv.cfg", false), + ("site", true), + ("site/.pth", true), + ("site/one.pth", true), + ], + }]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary directory"); + let root: PathBuf = std::fs::canonicalize(directory.path()).expect("resolved"); + for folder in ["a", "a-b", "a/b", "site"] { + std::fs::create_dir_all(root.join(folder)).expect("folder"); + } + for file in [ + "site/one.pth", + "site/.pth", + "site/two.pth.bak", + "a/q.pth", + "a-b/x.pth", + ] { + std::fs::write(root.join(file), "x\n").expect("file"); + } + let locations = StartupLocations { + directories: test_case.directories.iter().map(PathBuf::from).collect(), + files: vec![root.join("site/one.pth")], + prefixes: vec![root.clone()], + }; + let stamps: Vec = startup_stamps(&locations, &root).expect("stamps"); + let listed: Vec<(String, bool)> = stamps + .iter() + .map(|stamp| { + let relative: &Path = stamp.path.strip_prefix(&root).unwrap_or(&stamp.path); + ( + relative.to_string_lossy().into_owned(), + stamp.size.is_some(), + ) + }) + .collect(); + let listed: Vec<(&str, bool)> = listed + .iter() + .map(|(path, present)| (path.as_str(), *present)) + .collect(); + assert_eq!( + listed, test_case.expected_stamps, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs b/crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs new file mode 100644 index 0000000000..57ad24a56c --- /dev/null +++ b/crates/sqlbuild-cache/src/startup_inputs/tests/test_types.rs @@ -0,0 +1,7 @@ +pub(crate) struct StartupStampsTestCase { + pub(crate) description: &'static str, + /// Import-path entries, relative to the temporary root unless absolute-looking. + pub(crate) directories: Vec<&'static str>, + /// Stamped paths under the root and whether each exists, in order. + pub(crate) expected_stamps: Vec<(&'static str, bool)>, +} diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs index 54650af368..ba6664c9b8 100644 --- a/crates/sqlbuild-cli/src/compute_logs/constants.rs +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -1,4 +1,4 @@ -//! Python's `sqlbuild.runtime.compute_logs.constants` and storage file names. +//! Compute log format version, retention defaults and storage file names. pub const COMPUTE_LOG_FORMAT_VERSION: i64 = 1; pub const DEFAULT_RETENTION_COUNT: usize = 20; diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index d61be461e5..e375fd4a63 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -13,7 +13,7 @@ pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; const NATIVE_RECORDS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); const SELECT: Field = field("select", Source::Select); diff --git a/crates/sqlbuild-cli/src/replay/main/mod.rs b/crates/sqlbuild-cli/src/replay/main/mod.rs index f377c949dc..dc3b58e79e 100644 --- a/crates/sqlbuild-cli/src/replay/main/mod.rs +++ b/crates/sqlbuild-cli/src/replay/main/mod.rs @@ -1,2 +1,3 @@ +pub mod output_file_identities; pub mod serve_reused_compile; pub mod write_launch_record; diff --git a/crates/sqlbuild-cli/src/replay/main/output_file_identities.rs b/crates/sqlbuild-cli/src/replay/main/output_file_identities.rs new file mode 100644 index 0000000000..a607253b98 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/main/output_file_identities.rs @@ -0,0 +1,9 @@ +//! Identify the regular files standard output and error write to. + +use crate::replay::_helpers::console::redirected_output_files; + +/// Sorted, distinct device and inode pairs of stdout and stderr when they are regular files. +#[must_use] +pub fn output_file_identities() -> Vec<(u64, u64)> { + redirected_output_files() +} diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 969eeb4ed6..d8474dd099 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -9,7 +9,6 @@ pub const CONNECTION_SOURCE_KEY: &str = "source"; pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile"; pub const DUCKDB_ADAPTER: &str = "duckdb"; pub const DUCKDB_MEMORY_DATABASE: &str = ":memory:"; -pub const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Connection keys a dbt profile connection routes on rather than connects with. pub const DBT_PROFILE_ROUTING_KEYS: [&str; 5] = diff --git a/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs index 450078ba96..2e88d58a4f 100644 --- a/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs +++ b/crates/sqlbuild-config/src/command_connection/main/authored_warehouse.rs @@ -1,7 +1,8 @@ //! The authored warehouse a target sets for a command group. -use crate::command_connection::constants::{LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME}; +use crate::command_connection::constants::LOCAL_CONFIG_FILENAME; use crate::command_connection::models::AuthoredWarehouse; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// The local target's group warehouse, else the project target's, with the file that sets it. #[must_use] diff --git a/crates/sqlbuild-config/src/dbt_profile/constants.rs b/crates/sqlbuild-config/src/dbt_profile/constants.rs index cbfd030d3a..e705f661d8 100644 --- a/crates/sqlbuild-config/src/dbt_profile/constants.rs +++ b/crates/sqlbuild-config/src/dbt_profile/constants.rs @@ -51,3 +51,9 @@ pub(crate) const BIGQUERY_METHOD_ERROR_SUFFIX: &str = "is not supported by SQLBu /// The dbt target path used when `dbt_project.yml` sets none. pub(crate) const DEFAULT_DBT_TARGET_PATH: &str = "target"; +/// Text dbt's `as_bool` reads as true, after stripping and lowercasing. +pub(crate) const DBT_TRUE_VALUES: [&str; 5] = ["true", "1", "yes", "y", "on"]; +/// Text dbt's `as_bool` reads as false, after stripping and lowercasing. +pub(crate) const DBT_FALSE_VALUES: [&str; 5] = ["false", "0", "no", "n", "off"]; +/// The separator that makes dbt's `as_number` read text as a float. +pub(crate) const DBT_NUMBER_DECIMAL_SEPARATOR: char = '.'; diff --git a/crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs b/crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs new file mode 100644 index 0000000000..4f2d6b8b35 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/dbt_bool_text.rs @@ -0,0 +1,17 @@ +//! `dbt_bool_text`: the boolean dbt's `as_bool` reads from text. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dbt_profile::constants::{DBT_FALSE_VALUES, DBT_TRUE_VALUES}; + +/// The boolean dbt's `as_bool` reads from stripped, lowercased text, or `None`. +#[must_use] +pub fn dbt_bool_text(text: &str) -> Option { + let normalized: String = python_strip(text).to_lowercase(); + if DBT_TRUE_VALUES.contains(&normalized.as_str()) { + return Some(true); + } + DBT_FALSE_VALUES + .contains(&normalized.as_str()) + .then_some(false) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs b/crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs new file mode 100644 index 0000000000..c01d1d66c6 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/main/dbt_number_text.rs @@ -0,0 +1,18 @@ +//! `dbt_number_text`: how dbt's `as_number` parses text. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dbt_profile::constants::DBT_NUMBER_DECIMAL_SEPARATOR; +use crate::dbt_profile::models::DbtNumberParse; + +/// The stripped text dbt's `as_number` parses and whether it parses it as a float. +#[must_use] +pub fn dbt_number_text(text: &str) -> (&str, DbtNumberParse) { + let stripped: &str = python_strip(text); + let parse: DbtNumberParse = if stripped.contains(DBT_NUMBER_DECIMAL_SEPARATOR) { + DbtNumberParse::Float + } else { + DbtNumberParse::Integer + }; + (stripped, parse) +} diff --git a/crates/sqlbuild-config/src/dbt_profile/main/mod.rs b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs index 640790dd39..480aa81680 100644 --- a/crates/sqlbuild-config/src/dbt_profile/main/mod.rs +++ b/crates/sqlbuild-config/src/dbt_profile/main/mod.rs @@ -1,4 +1,6 @@ +pub mod dbt_bool_text; pub mod dbt_init_project_toml; +pub mod dbt_number_text; pub mod dbt_profile_outputs; pub mod dbt_project_metadata; pub mod dbt_yaml_mapping; diff --git a/crates/sqlbuild-config/src/dbt_profile/models.rs b/crates/sqlbuild-config/src/dbt_profile/models.rs index a4e5264201..21c1b86b0e 100644 --- a/crates/sqlbuild-config/src/dbt_profile/models.rs +++ b/crates/sqlbuild-config/src/dbt_profile/models.rs @@ -100,3 +100,10 @@ pub struct DbtProfileOutputs { /// Stripped output names in first-seen order; a repeated name keeps its last entry. pub outputs: Vec<(String, usize)>, } + +/// How dbt's `as_number` parses one stripped text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DbtNumberParse { + Integer, + Float, +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests.rs b/crates/sqlbuild-config/src/dbt_profile/tests.rs index 003e0bb7d0..9ae346d0e2 100644 --- a/crates/sqlbuild-config/src/dbt_profile/tests.rs +++ b/crates/sqlbuild-config/src/dbt_profile/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_dbt_coercion.rs"] +mod dbt_coercion; #[path = "tests/test_dbt_profile.rs"] mod dbt_profile; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs new file mode 100644 index 0000000000..c915b76167 --- /dev/null +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_dbt_coercion.rs @@ -0,0 +1,79 @@ +use crate::dbt_profile::main::dbt_bool_text::dbt_bool_text; +use crate::dbt_profile::main::dbt_number_text::dbt_number_text; +use crate::dbt_profile::models::DbtNumberParse; +use crate::dbt_profile::tests::test_types::{DbtBoolTestCase, DbtNumberTestCase}; + +#[test] +fn given_profile_text_when_coercing_to_bool_then_dbt_spellings_are_read() { + let test_cases = [ + DbtBoolTestCase { + description: "padded mixed-case yes", + text: " Yes ", + expected_bool: Some(true), + }, + DbtBoolTestCase { + description: "uppercase on", + text: "ON", + expected_bool: Some(true), + }, + DbtBoolTestCase { + description: "whitespace-padded no", + text: "\tno\n", + expected_bool: Some(false), + }, + DbtBoolTestCase { + description: "zero", + text: "0", + expected_bool: Some(false), + }, + DbtBoolTestCase { + description: "other digit", + text: "2", + expected_bool: None, + }, + DbtBoolTestCase { + description: "empty", + text: "", + expected_bool: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_bool_text(test_case.text), + test_case.expected_bool, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_profile_text_when_coercing_to_number_then_decimal_separator_selects_float() { + let test_cases = [ + DbtNumberTestCase { + description: "padded negative integer", + text: " -3 ", + expected_parse: ("-3", DbtNumberParse::Integer), + }, + DbtNumberTestCase { + description: "decimal", + text: "1.5", + expected_parse: ("1.5", DbtNumberParse::Float), + }, + DbtNumberTestCase { + description: "exponent without separator parses as integer", + text: "1e3", + expected_parse: ("1e3", DbtNumberParse::Integer), + }, + ]; + + for test_case in test_cases { + assert_eq!( + dbt_number_text(test_case.text), + test_case.expected_parse, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs index 752c57b53d..e4e7b2742a 100644 --- a/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs +++ b/crates/sqlbuild-config/src/dbt_profile/tests/test_types.rs @@ -1,7 +1,7 @@ use crate::dbt_profile::errors::DbtProfileError; use crate::dbt_profile::models::{ - DbtInitConfig, DbtProfileOutputs, DbtProjectMetadata, DbtYamlValue, LoadedDbtProfile, - NormalizedProfile, ProfileEntry, + DbtInitConfig, DbtNumberParse, DbtProfileOutputs, DbtProjectMetadata, DbtYamlValue, + LoadedDbtProfile, NormalizedProfile, ProfileEntry, }; pub(super) struct NormalizeTestCase { @@ -34,3 +34,15 @@ pub(super) struct SelectTargetTestCase { pub(super) requested: Option<&'static str>, pub(super) expected_result: Result<(String, usize), DbtProfileError>, } + +pub(super) struct DbtBoolTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_bool: Option, +} + +pub(super) struct DbtNumberTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_parse: (&'static str, DbtNumberParse), +} diff --git a/crates/sqlbuild-config/src/project/_helpers/defaults.rs b/crates/sqlbuild-config/src/project/_helpers/defaults.rs index fc20814c61..e6d118dc9c 100644 --- a/crates/sqlbuild-config/src/project/_helpers/defaults.rs +++ b/crates/sqlbuild-config/src/project/_helpers/defaults.rs @@ -13,12 +13,14 @@ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validat use crate::project::constants::{ BATCH_CONCURRENCY_KEY, CHANGE_POLICY_KEYS, CONTRACT_POLICIES, DEFAULTS_KEYS, DISABLED_VALUE, GLOB_SEGMENTS, INHERIT_VALUE, MATERIALIZATION_DEFAULT_KEYS, MAX_MICROBATCHES_KEY, - MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, PROJECT_CONFIG_FILENAME, RETENTION_MATERIALIZATIONS, - TABLE_TYPES, TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, ZERO_DAY_DURATION, + MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, RETENTION_MATERIALIZATIONS, TABLE_TYPES, + TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::ZERO_DAY_DURATION; /// A retention or table-type field: where it lives and whether `'inherit'` is accepted. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 827dfb0d08..80ff474882 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -24,13 +24,15 @@ use crate::project::_helpers::targets::{ }; use crate::project::_helpers::values::is_truthy; use crate::project::constants::{ - LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, - QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SETTINGS_SECTION, SQL_ANALYSIS_KEY, + LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{Built, ConfigFile, ConfigPaths}; use crate::project::types::{Entries, Fields, Loaded}; use crate::toml::main::load_toml::load_toml; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; use std::path::Path; const PROJECT_FILE: ConfigFile = ConfigFile::Project; diff --git a/crates/sqlbuild-config/src/project/_helpers/help.rs b/crates/sqlbuild-config/src/project/_helpers/help.rs index 9cdb500837..027276dd83 100644 --- a/crates/sqlbuild-config/src/project/_helpers/help.rs +++ b/crates/sqlbuild-config/src/project/_helpers/help.rs @@ -8,6 +8,8 @@ use crate::project::constants::{ REPLAY_ON_CHANGE_VALUES, SETTING_SNIPPET_INDENT, }; use crate::project::errors::ConfigFailure; +use crate::toml::main::toml_literal::toml_literal; +use crate::toml::models::TomlLiteral; /// A setting value as TOML shows it. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -28,14 +30,11 @@ pub(crate) struct Setting<'a> { } fn toml_value(value: SettingValue<'_>) -> String { - match value { - SettingValue::Bool(true) => "true".to_owned(), - SettingValue::Bool(false) => "false".to_owned(), - SettingValue::Integer(digits) => digits.to_owned(), - SettingValue::Text(text) => { - format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) - } - } + toml_literal(&match value { + SettingValue::Bool(flag) => TomlLiteral::Bool(flag), + SettingValue::Integer(digits) => TomlLiteral::Integer(digits.to_owned()), + SettingValue::Text(text) => TomlLiteral::Text(text.to_owned()), + }) } /// `setting_help`: `, set this in :` and the TOML section and key. diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 54de2a9029..e6d2fb5489 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -17,11 +17,12 @@ use crate::project::constants::{ REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_DAY_DURATION, ZERO_FIXED_DURATION, ZERO_FLOAT, + USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::constants::ZERO_DAY_DURATION; fn place(file: ConfigFile, label: &str) -> Place<'_> { Place { file, label } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index c932d0a782..373d44b4c7 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -1,8 +1,6 @@ //! Project configuration file names, keys and the vocabularies their values come from. -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -pub(crate) const SETTINGS_SECTION: &str = "settings"; pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; pub(crate) const LEGACY_SQL_VALIDATION_KEY: &str = "sql_validation"; pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; @@ -27,7 +25,6 @@ pub(crate) const CHANGE_POLICY_KEYS: [&str; 2] = [ON_SCHEMA_CHANGE_KEY, REPLAY_O pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; pub(crate) const MODELS_DIRECTORY_NAME: &str = "models"; pub(crate) const UNLIMITED_DIFF_FULL_ROWS: &str = "unlimited"; -pub(crate) const ZERO_DAY_DURATION: &str = "0d"; pub(crate) const ZERO_FIXED_DURATION: &str = "0s"; pub(crate) const DEFAULT_QUERY_ARTIFACT_TTL: &str = "24h"; pub(crate) const BOUNDED_REPLAY_PREFIX: &str = "bounded-"; diff --git a/crates/sqlbuild-config/src/project/main/explicit_references_help.rs b/crates/sqlbuild-config/src/project/main/explicit_references_help.rs index 43a8d02407..d87e6a3613 100644 --- a/crates/sqlbuild-config/src/project/main/explicit_references_help.rs +++ b/crates/sqlbuild-config/src/project/main/explicit_references_help.rs @@ -1,9 +1,8 @@ //! The help shown when explicit references reject an implicit reference. use crate::project::_helpers::help::{Setting, SettingValue, current_value_note, setting_help}; -use crate::project::constants::{ - ENFORCE_EXPLICIT_KEY, PROJECT_CONFIG_FILENAME, REFERENCES_SECTION, -}; +use crate::project::constants::{ENFORCE_EXPLICIT_KEY, REFERENCES_SECTION}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// The current-value note, then help that relaxes the policy to allow `allowed`. #[must_use] diff --git a/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs b/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs index 56451a16b6..7125bb4c8f 100644 --- a/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs +++ b/crates/sqlbuild-config/src/project/main/microbatch_guidance.rs @@ -1,10 +1,9 @@ //! The note and help shown when microbatch concurrency is off. use crate::project::_helpers::help::{Setting, SettingValue, current_value_note, setting_help}; -use crate::project::constants::{ - MICROBATCH_CONCURRENCY_KEY, MICROBATCH_CONCURRENCY_PURPOSE, PROJECT_CONFIG_FILENAME, - SETTINGS_SECTION, -}; +use crate::project::constants::{MICROBATCH_CONCURRENCY_KEY, MICROBATCH_CONCURRENCY_PURPOSE}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; /// The current-value note and the help that enables microbatch concurrency. #[must_use] diff --git a/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs b/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs index b5169f0cd1..c369a30091 100644 --- a/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs +++ b/crates/sqlbuild-config/src/project/main/sql_analysis_off_guidance.rs @@ -1,9 +1,9 @@ //! The note and help shown when SQL analysis is turned off. use crate::project::_helpers::help::{Setting, SettingValue, setting_help, setting_note}; -use crate::project::constants::{ - LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, SQL_ANALYSIS_KEY, -}; +use crate::project::constants::{LOCAL_CONFIG_FILENAME, SQL_ANALYSIS_KEY}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::SETTINGS_SECTION; /// The note naming the file that turns SQL analysis off and the help that turns it on for `purpose`. #[must_use] diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index ff2a018aa6..d583900e86 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -25,7 +25,7 @@ pub struct ConfigPaths { pub enum Built { /// A plain value, including `None`, booleans, integers and raw authored mappings. Value(ConfigValue), - /// A dataclass from `sqlbuild.spec.contracts.models`; omitted fields keep their defaults. + /// A native project configuration record; omitted fields keep their defaults. Object(&'static str, Vec<(&'static str, Built)>), /// A member of a named `StrEnum`, by value. Enum(&'static str, String), diff --git a/crates/sqlbuild-config/src/toml/main/mod.rs b/crates/sqlbuild-config/src/toml/main/mod.rs index 04683b5110..f50eb8ba0e 100644 --- a/crates/sqlbuild-config/src/toml/main/mod.rs +++ b/crates/sqlbuild-config/src/toml/main/mod.rs @@ -1 +1,2 @@ pub mod load_toml; +pub mod toml_literal; diff --git a/crates/sqlbuild-config/src/toml/main/toml_literal.rs b/crates/sqlbuild-config/src/toml/main/toml_literal.rs new file mode 100644 index 0000000000..e04b5835c1 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/main/toml_literal.rs @@ -0,0 +1,20 @@ +//! A TOML literal as SQLBuild writes it into configuration text and help. + +use crate::toml::models::TomlLiteral; + +/// `true`/`false`, the integer digits, a `"`-quoted string escaping `\` and `"`, or `[a, b]`. +#[must_use] +pub fn toml_literal(value: &TomlLiteral) -> String { + match value { + TomlLiteral::Bool(true) => "true".to_owned(), + TomlLiteral::Bool(false) => "false".to_owned(), + TomlLiteral::Integer(digits) => digits.clone(), + TomlLiteral::Text(text) => { + format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) + } + TomlLiteral::Array(items) => { + let items: Vec = items.iter().map(toml_literal).collect(); + format!("[{}]", items.join(", ")) + } + } +} diff --git a/crates/sqlbuild-config/src/toml/mod.rs b/crates/sqlbuild-config/src/toml/mod.rs index 37f9081f4f..1e64f3d116 100644 --- a/crates/sqlbuild-config/src/toml/mod.rs +++ b/crates/sqlbuild-config/src/toml/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod _helpers; pub(crate) mod constants; pub mod main; +pub mod models; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-config/src/toml/models.rs b/crates/sqlbuild-config/src/toml/models.rs new file mode 100644 index 0000000000..8273c88fe3 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/models.rs @@ -0,0 +1,11 @@ +//! TOML values as SQLBuild writes them into configuration text. + +/// A boolean, integer, string or array literal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TomlLiteral { + Bool(bool), + /// An integer, as its decimal text. + Integer(String), + Text(String), + Array(Vec), +} diff --git a/crates/sqlbuild-config/src/toml/tests.rs b/crates/sqlbuild-config/src/toml/tests.rs index 55b6a54c6d..3722e7e455 100644 --- a/crates/sqlbuild-config/src/toml/tests.rs +++ b/crates/sqlbuild-config/src/toml/tests.rs @@ -4,3 +4,5 @@ mod helpers; mod load_toml; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_toml_literal.rs"] +mod toml_literal; diff --git a/crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs b/crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs new file mode 100644 index 0000000000..b4cdc468e5 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/tests/test_toml_literal.rs @@ -0,0 +1,50 @@ +use crate::toml::main::toml_literal::toml_literal; +use crate::toml::models::TomlLiteral; +use crate::toml::tests::test_types::TomlLiteralTestCase; + +#[test] +fn given_literals_when_written_then_toml_text_matches_the_rule_harness_encoding() { + let test_cases = [ + TomlLiteralTestCase { + description: "a boolean", + value: TomlLiteral::Bool(true), + expected_text: "true", + }, + TomlLiteralTestCase { + description: "an integer keeps its digits", + value: TomlLiteral::Integer("-5".to_owned()), + expected_text: "-5", + }, + TomlLiteralTestCase { + description: "a plain string", + value: TomlLiteral::Text("plain".to_owned()), + expected_text: "\"plain\"", + }, + TomlLiteralTestCase { + description: "quotes and backslashes are escaped", + value: TomlLiteral::Text("a\"b\\c".to_owned()), + expected_text: "\"a\\\"b\\\\c\"", + }, + TomlLiteralTestCase { + description: "an empty array", + value: TomlLiteral::Array(Vec::new()), + expected_text: "[]", + }, + TomlLiteralTestCase { + description: "nested arrays", + value: TomlLiteral::Array(vec![ + TomlLiteral::Integer("1".to_owned()), + TomlLiteral::Array(vec![TomlLiteral::Bool(false)]), + ]), + expected_text: "[1, [false]]", + }, + ]; + for test_case in test_cases { + assert_eq!( + toml_literal(&test_case.value), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_types.rs b/crates/sqlbuild-config/src/toml/tests/test_types.rs index c33becd42d..fab0f70cdb 100644 --- a/crates/sqlbuild-config/src/toml/tests/test_types.rs +++ b/crates/sqlbuild-config/src/toml/tests/test_types.rs @@ -24,3 +24,9 @@ pub(super) struct DeepTomlTestCase { pub(super) text: String, pub(super) expected_value: ConfigValue, } + +pub(super) struct TomlLiteralTestCase { + pub(super) description: &'static str, + pub(super) value: crate::toml::models::TomlLiteral, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs b/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs index 4c0961fa12..ebe7413f0b 100644 --- a/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs +++ b/crates/sqlbuild-config/src/workspace/_helpers/skill_settings.rs @@ -6,10 +6,11 @@ use crate::models::ConfigValue; use crate::toml::main::load_toml::load_toml; use crate::workspace::_helpers::files::read_text; use crate::workspace::constants::{ - CLAUDE_TARGET, DEFAULT_SKILL_TARGETS, GIT_MARKER, OPENCODE_TARGET, PROJECT_CONFIG_FILENAME, - SKILL_ENTRY_FILE, SKILL_NAME, UTF8_BYTE_ORDER_MARK, VALID_SKILL_TARGETS, + CLAUDE_TARGET, DEFAULT_SKILL_TARGETS, GIT_MARKER, OPENCODE_TARGET, SKILL_ENTRY_FILE, + SKILL_NAME, UTF8_BYTE_ORDER_MARK, VALID_SKILL_TARGETS, }; use crate::workspace::errors::WorkspaceError; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// Project skill installation preferences (`SkillSettings`). pub(crate) struct SkillSettings { diff --git a/crates/sqlbuild-config/src/workspace/constants.rs b/crates/sqlbuild-config/src/workspace/constants.rs index 811834e0bf..3d328a8ef6 100644 --- a/crates/sqlbuild-config/src/workspace/constants.rs +++ b/crates/sqlbuild-config/src/workspace/constants.rs @@ -28,7 +28,6 @@ pub(crate) const VALID_SKILL_TARGETS: [&str; 3] = ["opencode", "claude", "agents pub(crate) const DEFAULT_SKILL_TARGETS: [&str; 2] = ["agents", "claude"]; pub(crate) const OPENCODE_TARGET: &str = "opencode"; pub(crate) const CLAUDE_TARGET: &str = "claude"; -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const UTF8_BYTE_ORDER_MARK: char = '\u{feff}'; pub(crate) const GIT_MARKER: &str = ".git"; pub(crate) const PLAYGROUND_EXAMPLES: [(&str, &str); 4] = [ diff --git a/crates/sqlbuild-core/Cargo.toml b/crates/sqlbuild-core/Cargo.toml index 2d235db809..e70eb050a7 100644 --- a/crates/sqlbuild-core/Cargo.toml +++ b/crates/sqlbuild-core/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] regex.workspace = true +sha2.workspace = true [lints] workspace = true diff --git a/crates/sqlbuild-core/src/command_output/constants.rs b/crates/sqlbuild-core/src/command_output/constants.rs index 627d6f538e..57c7e05962 100644 --- a/crates/sqlbuild-core/src/command_output/constants.rs +++ b/crates/sqlbuild-core/src/command_output/constants.rs @@ -1,4 +1,4 @@ -//! Python's `sqlbuild.runtime.output_capture.constants` delivery limits and text bytes. +//! Command output delivery limits, record types and text bytes. pub const MIN_OUTPUT_RECORD_BYTES: i64 = 4; pub const FLUSH_INTERVAL_SECONDS: f64 = 0.25; diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index cfb324eb8c..75a4f1cdc8 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -6,3 +6,29 @@ pub const CLOSE_PAREN: &str = ")"; pub const PANIC_MESSAGE: &str = "NativeCompilerError: native SQL compilation panicked"; /// The prefix every native internal failure carries, so the boundary raises `NativeCompilerError`. pub const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; +/// File name of the project configuration at the project root. +pub const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +/// Project configuration section holding settings. +pub const SETTINGS_SECTION: &str = "settings"; +/// Settings key selecting how table builds are promoted. +pub const TABLE_PROMOTION_MODE_SETTING_KEY: &str = "table_promotion_mode"; +/// Table promotion mode that builds into a staging relation before swapping. +pub const STAGED_PROMOTION_MODE: &str = "staged"; +/// Project directory holding Python node modules. +pub const PYTHON_NODE_ROOT: &str = "python"; +/// Stem of a Python package's initializer module. +pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; +/// State table of model, seed and hook fingerprints. +pub const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; +/// State table of observed source freshness. +pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +/// State table of model migration events. +pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +/// State table of column migration events. +pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; +/// State table of old-name view events. +pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; +/// A zero-length cursor duration. +pub const ZERO_DAY_DURATION: &str = "0d"; +/// Node key of a common table expression in the polyglot SQL AST. +pub const CTE_AST_KIND: &str = "cte"; diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs new file mode 100644 index 0000000000..34809b51a5 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/markdown.rs @@ -0,0 +1,22 @@ +//! A Markdown code span that shows any text verbatim. + +use crate::dagster_invocation::constants::{CODE_SPAN_MARK, CODE_SPAN_PADDED_EDGES}; + +/// `value` fenced by one more backtick than its longest run, padded at a backtick or space edge. +pub(crate) fn markdown_code_span(value: &str) -> String { + let mut longest: usize = 0; + let mut current: usize = 0; + for character in value.chars() { + current = if character == CODE_SPAN_MARK { + current + 1 + } else { + 0 + }; + longest = longest.max(current); + } + let delimiter: String = CODE_SPAN_MARK.to_string().repeat(longest + 1); + let padded: bool = + value.starts_with(CODE_SPAN_PADDED_EDGES) || value.ends_with(CODE_SPAN_PADDED_EDGES); + let padding: &str = if padded { " " } else { "" }; + format!("{delimiter}{padding}{value}{padding}{delimiter}") +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs index 07ebd31772..c23e22da7f 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/mod.rs @@ -1 +1,3 @@ pub(crate) mod graph; +pub(crate) mod markdown; +pub(crate) mod names; diff --git a/crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs b/crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs new file mode 100644 index 0000000000..a4c719165c --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/_helpers/names.rs @@ -0,0 +1,19 @@ +//! Runs of rejected characters collapsed to one separator. + +use crate::dagster_invocation::constants::NAME_SEPARATOR; + +/// `value` with every maximal run of characters `kept` rejects replaced by one `_`. +pub(crate) fn replace_runs(value: &str, kept: impl Fn(char) -> bool) -> String { + let mut replaced: String = String::with_capacity(value.len()); + let mut in_run: bool = false; + for character in value.chars() { + if kept(character) { + replaced.push(character); + in_run = false; + } else if !in_run { + replaced.push(NAME_SEPARATOR); + in_run = true; + } + } + replaced +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/constants.rs b/crates/sqlbuild-core/src/dagster_invocation/constants.rs index ede32be2a3..ba8a89311c 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/constants.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/constants.rs @@ -21,3 +21,15 @@ pub const DEFAULT_CHECK_NAME: &str = "check"; pub const MAX_DIAGNOSTIC_CHARS: usize = 4_000; /// Indentation of SQL in failure text. pub const SQL_INDENT: &str = " "; +/// Separator replacing characters a Dagster tag key or name segment cannot hold. +pub const NAME_SEPARATOR: char = '_'; +/// ASCII punctuation a Dagster tag key keeps besides letters, digits and `_`. +pub const TAG_KEY_PUNCTUATION: [char; 3] = ['.', '-', '/']; +/// The Markdown code-span delimiter. +pub const CODE_SPAN_MARK: char = '`'; +/// Edge characters that need padding inside a Markdown code span. +pub(crate) const CODE_SPAN_PADDED_EDGES: [char; 2] = [CODE_SPAN_MARK, ' ']; +/// The default translator's description kind of a node without one. +pub const DEFAULT_ASSET_KIND: &str = "asset"; +/// The default translator's description name of a node without one. +pub const DEFAULT_ASSET_NAME: &str = "unknown"; diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs b/crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs new file mode 100644 index 0000000000..ab85c819be --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/asset_description.rs @@ -0,0 +1,19 @@ +//! The default Dagster description: the node's text followed by its SQL or source file. + +use crate::dagster_invocation::_helpers::markdown::markdown_code_span; +use crate::text::main::python_strip::python_strip; + +/// `description` followed by its stripped SQL block, or else by its source file code span. +#[must_use] +pub fn asset_description(description: &str, sql: Option<&str>, path: Option<&str>) -> String { + if let Some(sql) = sql.map(python_strip).filter(|sql| !sql.is_empty()) { + return format!("{description}\n\n**SQLBuild SQL:**\n```sql\n{sql}\n```"); + } + if let Some(path) = path.filter(|path| !path.is_empty()) { + return format!( + "{description}\n\n**Source file:** {}", + markdown_code_span(path) + ); + } + description.to_owned() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs new file mode 100644 index 0000000000..737e54cf01 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_name_segment.rs @@ -0,0 +1,18 @@ +//! The default Dagster translator's check-name segment. + +use crate::dagster_invocation::_helpers::names::replace_runs; +use crate::dagster_invocation::constants::{DEFAULT_CHECK_NAME, NAME_SEPARATOR}; + +/// `re.sub(r"[^A-Za-z0-9_]+", "_", value).strip("_") or "check"`. +#[must_use] +pub fn dagster_name_segment(value: &str) -> String { + let replaced: String = replace_runs(value, |character| { + character.is_ascii_alphanumeric() || character == NAME_SEPARATOR + }); + let trimmed: &str = replaced.trim_matches(NAME_SEPARATOR); + if trimmed.is_empty() { + DEFAULT_CHECK_NAME.to_owned() + } else { + trimmed.to_owned() + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs new file mode 100644 index 0000000000..2b54e34160 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/dagster_tag_key.rs @@ -0,0 +1,16 @@ +//! The default Dagster translator's tag key for one SQLBuild tag. + +use crate::dagster_invocation::_helpers::names::replace_runs; +use crate::dagster_invocation::constants::{NAME_SEPARATOR, TAG_KEY_PUNCTUATION}; + +/// `re.sub(r"[^A-Za-z0-9_.\-/]+", "_", value).strip("_")`. +#[must_use] +pub fn dagster_tag_key(value: &str) -> String { + replace_runs(value, |character| { + character.is_ascii_alphanumeric() + || character == NAME_SEPARATOR + || TAG_KEY_PUNCTUATION.contains(&character) + }) + .trim_matches(NAME_SEPARATOR) + .to_owned() +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs b/crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs new file mode 100644 index 0000000000..a0ea7f4cc4 --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/main/fallback_asset_description.rs @@ -0,0 +1,23 @@ +//! The default Dagster description of a node that has none. + +/// `SQLBuild ``[ materialized as ``][ at ``].`, kind `_` as spaces. +#[must_use] +pub fn fallback_asset_description( + kind: &str, + name: &str, + qualified_name: &str, + materialization: Option<&str>, +) -> String { + let kind: String = kind.replace('_', " "); + let relation: String = if qualified_name.is_empty() { + String::new() + } else { + format!(" at `{qualified_name}`") + }; + match materialization { + Some(materialization) => { + format!("SQLBuild {kind} `{name}` materialized as `{materialization}`{relation}.") + } + None => format!("SQLBuild {kind} `{name}`{relation}."), + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs b/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs index 38e4f2ce5e..4c3765b5a1 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/main/mod.rs @@ -1,6 +1,10 @@ +pub mod asset_description; pub mod asset_failure_text; pub mod caller_json_output_path; +pub mod dagster_name_segment; +pub mod dagster_tag_key; pub mod diagnostic_tail; +pub mod fallback_asset_description; pub mod has_explicit_scenario_selector; pub mod json_output_requested; pub mod normalize_check_name; diff --git a/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs b/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs index 2ebe69f43b..41b13b95f8 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/main/normalize_check_name.rs @@ -1,14 +1,15 @@ //! Dagster-safe check name segments. use crate::dagster_invocation::constants::{CHECK_NAME_SEPARATOR, DEFAULT_CHECK_NAME}; +use crate::text::main::python_isalnum::python_isalnum; -/// `value` with every non-alphanumeric character replaced and outer separators trimmed. +/// `value` with every character Python's `str.isalnum` rejects replaced and outer separators trimmed. #[must_use] pub fn normalize_check_name(value: &str) -> String { let normalized: String = value .chars() .map(|character| { - if character.is_alphanumeric() || character == CHECK_NAME_SEPARATOR { + if python_isalnum(character) || character == CHECK_NAME_SEPARATOR { character } else { CHECK_NAME_SEPARATOR diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests.rs b/crates/sqlbuild-core/src/dagster_invocation/tests.rs index 969cb00f66..d2b5a423e7 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/tests.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/tests.rs @@ -4,5 +4,7 @@ mod helpers; mod test_arguments; #[path = "tests/test_texts.rs"] mod test_texts; +#[path = "tests/test_translator.rs"] +mod test_translator; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs index 8ee21180a9..573826ed85 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_texts.rs @@ -21,6 +21,36 @@ fn given_raw_check_name_when_normalizing_then_replaces_punctuation() { value: "--", expected_name: "check", }, + CheckNameTestCase { + description: "combining vowel sign is not alphanumeric", + value: "\u{915}\u{93f}1", + expected_name: "\u{915}_1", + }, + CheckNameTestCase { + description: "trailing combining mark is trimmed as a separator", + value: "\u{915}\u{93f}", + expected_name: "\u{915}", + }, + CheckNameTestCase { + description: "combining acute accent becomes a separator", + value: "e\u{301}tat", + expected_name: "e_tat", + }, + CheckNameTestCase { + description: "precomposed letters and Arabic-Indic digits are kept", + value: "\u{dc}n\u{ef}c\u{f6}d\u{e9}_\u{663}\u{664}", + expected_name: "\u{dc}n\u{ef}c\u{f6}d\u{e9}_\u{663}\u{664}", + }, + CheckNameTestCase { + description: "superscript, fraction and roman numeral are numeric", + value: "x\u{b2}\u{bd}\u{216b}", + expected_name: "x\u{b2}\u{bd}\u{216b}", + }, + CheckNameTestCase { + description: "ideographs and kana are kept around a separator", + value: "\u{65e5}\u{672c}\u{8a9e}-\u{30c1}\u{30a7}\u{30c3}\u{30af}", + expected_name: "\u{65e5}\u{672c}\u{8a9e}_\u{30c1}\u{30a7}\u{30c3}\u{30af}", + }, ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs new file mode 100644 index 0000000000..4f5e6c003b --- /dev/null +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_translator.rs @@ -0,0 +1,143 @@ +//! The default Dagster translator's names and descriptions match its Python spelling. + +use crate::dagster_invocation::main::asset_description::asset_description; +use crate::dagster_invocation::main::dagster_name_segment::dagster_name_segment; +use crate::dagster_invocation::main::dagster_tag_key::dagster_tag_key; +use crate::dagster_invocation::main::fallback_asset_description::fallback_asset_description; +use crate::dagster_invocation::tests::test_types::{ + DescriptionTestCase, FallbackDescriptionTestCase, NameNormalizationTestCase, +}; + +#[test] +fn given_tags_and_names_when_normalized_then_rejected_runs_become_one_separator() { + let test_cases = [ + NameNormalizationTestCase { + description: "a plain name", + value: "pii", + expected_tag_key: "pii", + expected_segment: "pii", + }, + NameNormalizationTestCase { + description: "punctuation runs collapse", + value: "a b!c", + expected_tag_key: "a_b_c", + expected_segment: "a_b_c", + }, + NameNormalizationTestCase { + description: "outer separators are trimmed", + value: "__x__", + expected_tag_key: "x", + expected_segment: "x", + }, + NameNormalizationTestCase { + description: "non-ASCII letters are rejected", + value: "\u{65e5}\u{672c}", + expected_tag_key: "", + expected_segment: "check", + }, + NameNormalizationTestCase { + description: "tag keys keep dots, dashes and slashes", + value: "ok/x.y-z", + expected_tag_key: "ok/x.y-z", + expected_segment: "ok_x_y_z", + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + dagster_tag_key(test_case.value), + dagster_name_segment(test_case.value) + ), + ( + test_case.expected_tag_key.to_owned(), + test_case.expected_segment.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_node_text_when_described_then_sql_wins_over_the_source_file() { + let test_cases = [ + DescriptionTestCase { + description: "stripped SQL is fenced", + text: "D", + sql: Some(" select 1 "), + path: Some("m.sql"), + expected_description: "D\n\n**SQLBuild SQL:**\n```sql\nselect 1\n```", + }, + DescriptionTestCase { + description: "blank SQL falls back to the source file", + text: "D", + sql: Some(" "), + path: Some("m.sql"), + expected_description: "D\n\n**Source file:** `m.sql`", + }, + DescriptionTestCase { + description: "backticks in the path widen the fence", + text: "D", + sql: None, + path: Some("a`b`` c.py"), + expected_description: "D\n\n**Source file:** ```a`b`` c.py```", + }, + DescriptionTestCase { + description: "an edge backtick pads the span", + text: "D", + sql: None, + path: Some("`start"), + expected_description: "D\n\n**Source file:** `` `start ``", + }, + DescriptionTestCase { + description: "no SQL or path keeps the text", + text: "D", + sql: None, + path: Some(""), + expected_description: "D", + }, + ]; + for test_case in test_cases { + assert_eq!( + asset_description(test_case.text, test_case.sql, test_case.path), + test_case.expected_description, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_undocumented_node_when_described_then_kind_name_and_relation_are_named() { + let test_cases = [ + FallbackDescriptionTestCase { + description: "a materialized relation", + kind: "python_asset", + name: "o", + qualified_name: "db.o", + materialization: Some("table"), + expected_description: "SQLBuild python asset `o` materialized as `table` at `db.o`.", + }, + FallbackDescriptionTestCase { + description: "no relation or materialization", + kind: "seed", + name: "s", + qualified_name: "", + materialization: None, + expected_description: "SQLBuild seed `s`.", + }, + ]; + for test_case in test_cases { + assert_eq!( + fallback_asset_description( + test_case.kind, + test_case.name, + test_case.qualified_name, + test_case.materialization + ), + test_case.expected_description, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs b/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs index 1f7b24722c..e63e23e7d4 100644 --- a/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs +++ b/crates/sqlbuild-core/src/dagster_invocation/tests/test_types.rs @@ -35,3 +35,27 @@ pub(super) struct FailureTextTestCase { pub failure: AssetFailure, pub expected_text: &'static str, } + +pub(super) struct NameNormalizationTestCase { + pub description: &'static str, + pub value: &'static str, + pub expected_tag_key: &'static str, + pub expected_segment: &'static str, +} + +pub(super) struct DescriptionTestCase { + pub description: &'static str, + pub text: &'static str, + pub sql: Option<&'static str>, + pub path: Option<&'static str>, + pub expected_description: &'static str, +} + +pub(super) struct FallbackDescriptionTestCase { + pub description: &'static str, + pub kind: &'static str, + pub name: &'static str, + pub qualified_name: &'static str, + pub materialization: Option<&'static str>, + pub expected_description: &'static str, +} diff --git a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs index 26d06a16a5..276a44ce2b 100644 --- a/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs +++ b/crates/sqlbuild-core/src/dbt_command/main/render_dbt_init_summary.rs @@ -1,6 +1,7 @@ //! The summary written after `sqb dbt init` creates a twin project. use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::dbt_command::constants::FIELD_LABEL_WIDTH; use crate::dbt_command::models::DbtInitSummary; @@ -27,7 +28,7 @@ pub fn render_dbt_init_summary(summary: &DbtInitSummary, style: CliStyle) -> Str document.lines.push(format!( " 1. {} ({})", style.value("SQLBuild twin config"), - style.object_name("sqlbuild_project.toml") + style.object_name(PROJECT_CONFIG_FILENAME) )); document.lines.push( " This points SQLBuild at your dbt project and profile. Edit it if dbt paths" diff --git a/crates/sqlbuild-core/src/debug_report/constants.rs b/crates/sqlbuild-core/src/debug_report/constants.rs index 320f8a6ef8..b7afd7f981 100644 --- a/crates/sqlbuild-core/src/debug_report/constants.rs +++ b/crates/sqlbuild-core/src/debug_report/constants.rs @@ -1,6 +1,5 @@ //! `sqb debug` file names, masking and wording. -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs b/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs index 6ab96b510a..72a9c8e63c 100644 --- a/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs +++ b/crates/sqlbuild-core/src/debug_report/main/debug_config_files.rs @@ -2,9 +2,9 @@ use std::path::{Path, PathBuf}; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::debug_report::constants::{ LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_PROJECT_CONFIG_FILENAME, LOCAL_CONFIG_FILENAME, - PROJECT_CONFIG_FILENAME, }; use crate::debug_report::models::DebugConfigFiles; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/constants.rs b/crates/sqlbuild-core/src/diagnostic_routing/constants.rs new file mode 100644 index 0000000000..a1b74ec18e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/constants.rs @@ -0,0 +1,64 @@ +//! Diagnostic logging record names, fields and messages. + +/// The root SQLBuild logger name. +pub const LOGGER_ROOT_NAME: &str = "sqlbuild"; +/// The prefix of every SQLBuild child logger name. +pub const LOGGER_PREFIX: &str = "sqlbuild."; +/// The prefix of every SQLBuild structured record field. +pub const FIELD_PREFIX: &str = "sqlbuild_"; +/// The record field holding a SQL statement's digest. +pub const SQL_DIGEST_FIELD: &str = "sqlbuild_sql_digest"; +/// The record field holding SQL text. +pub const SQL_TEXT_FIELD: &str = "sqlbuild_sql"; +/// The record field marking a trusted private SQL record. +pub const SQL_PRIVATE_RECORD_FIELD: &str = "sqlbuild_private_sql_record"; +/// The message replacing every SQL record's text. +pub const REDACTED_SQL_LOG_MESSAGE: &str = "SQL diagnostic omitted"; +/// The message of a record whose text cannot be formatted. +pub const UNFORMATTABLE_MESSAGE: &str = ""; +/// The logger name of every retained SQL diagnostic. +pub const SQL_LOGGER_NAME: &str = "sqlbuild.sql"; +/// The producer of retained diagnostics. +pub const DIAGNOSTIC_PRODUCER: &str = "sqlbuild"; +/// The source of diagnostics retained from Python logging. +pub const DIAGNOSTIC_SOURCE: &str = "python_logging"; +/// The retained field holding private SQL text. +pub const SQL_TEXT_DIAGNOSTIC_FIELD: &str = "sql_text"; +/// Formatter-populated record fields never treated as structured extras. +pub const FORMATTER_RECORD_FIELDS: [&str; 2] = ["message", "asctime"]; +/// The SQL digest field value prefix. +pub const SQL_DIGEST_PREFIX: &str = "sha256:"; +/// The SQL action, intent and query ID record fields, in record order. +pub const SQL_ACTION_FIELD: &str = "sqlbuild_sql_action"; +pub const SQL_INTENT_FIELD: &str = "sqlbuild_sql_intent"; +pub const SQL_QUERY_ID_FIELD: &str = "sqlbuild_sql_query_id"; +/// Execution identity fields, in diagnostic order. +pub const IDENTITY_FIELDS: [&str; 7] = [ + "invocation_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", + "log_stream_id", +]; +/// The identity field defaulted to the handler's invocation. +pub const INVOCATION_ID_FIELD: &str = "invocation_id"; +/// Structured fields a retained diagnostic may carry, in retention order. +pub const SAFE_FIELDS: [&str; 7] = [ + "sql_action", + "sql_digest", + "sql_intent", + "sql_query_id", + "channel", + "subscriber", + "error_type", +]; +/// The safe fields a retained SQL diagnostic keeps. +pub const SQL_SAFE_FIELDS: [&str; 4] = ["sql_action", "sql_digest", "sql_intent", "sql_query_id"]; +/// Python logging level numbers. +pub const LEVEL_DEBUG: i64 = 10; +pub const LEVEL_INFO: i64 = 20; +pub const LEVEL_WARNING: i64 = 30; +pub const LEVEL_ERROR: i64 = 40; +pub const LEVEL_CRITICAL: i64 = 50; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs new file mode 100644 index 0000000000..152e70bd7e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/compute_log_admits.rs @@ -0,0 +1,14 @@ +//! `compute_log_admits`: whether the compute log retains one record. + +use crate::diagnostic_routing::constants::LEVEL_INFO; +use crate::diagnostic_routing::models::RecordFacts; + +/// Whether compute logs retain the record; SQL text retention skips SQL records without text. +#[must_use] +pub fn compute_log_admits(record: RecordFacts, include_sql_text: bool) -> bool { + if !record.internal && record.level < LEVEL_INFO { + return false; + } + let contains_sql: bool = record.has_sql_digest || record.has_sql_text; + !(include_sql_text && contains_sql && !record.has_sql_text) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs new file mode 100644 index 0000000000..75a891a81e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/diagnostic_severity.rs @@ -0,0 +1,21 @@ +//! `diagnostic_severity`: the diagnostic severity of a Python logging level. + +use crate::diagnostic_routing::constants::{ + LEVEL_CRITICAL, LEVEL_ERROR, LEVEL_INFO, LEVEL_WARNING, +}; + +/// The severity name of logging level `level`. +#[must_use] +pub fn diagnostic_severity(level: i64) -> &'static str { + if level >= LEVEL_CRITICAL { + "critical" + } else if level >= LEVEL_ERROR { + "error" + } else if level >= LEVEL_WARNING { + "warning" + } else if level >= LEVEL_INFO { + "info" + } else { + "debug" + } +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs new file mode 100644 index 0000000000..dbd7e9a6a5 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/internal_logger.rs @@ -0,0 +1,9 @@ +//! `internal_logger`: whether a logger name belongs to SQLBuild. + +use crate::diagnostic_routing::constants::{LOGGER_PREFIX, LOGGER_ROOT_NAME}; + +/// Whether `name` is the SQLBuild root logger or one of its children. +#[must_use] +pub fn internal_logger(name: &str) -> bool { + name == LOGGER_ROOT_NAME || name.starts_with(LOGGER_PREFIX) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs new file mode 100644 index 0000000000..cb6240ba1c --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/mod.rs @@ -0,0 +1,5 @@ +pub mod compute_log_admits; +pub mod diagnostic_severity; +pub mod internal_logger; +pub mod route_admits; +pub mod sql_digest; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs new file mode 100644 index 0000000000..01c8701128 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/route_admits.rs @@ -0,0 +1,19 @@ +//! `route_admits`: whether a diagnostic destination receives one record. + +use crate::diagnostic_routing::constants::LEVEL_INFO; +use crate::diagnostic_routing::models::{RecordFacts, RouteOptions}; + +/// Whether a destination receives the record; SQL text routes skip digest-only SQL records. +#[must_use] +pub fn route_admits(options: RouteOptions, record: RecordFacts) -> bool { + if options.internal_only && !record.internal { + return false; + } + if record.internal && !options.allow_internal { + return false; + } + if !record.internal && record.level < LEVEL_INFO { + return false; + } + !(options.include_sql_text && record.has_sql_digest && !record.has_sql_text) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs b/crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs new file mode 100644 index 0000000000..faac04ead9 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/main/sql_digest.rs @@ -0,0 +1,11 @@ +//! `sql_digest`: the logged digest of one SQL statement. + +use sha2::{Digest, Sha256}; + +use crate::diagnostic_routing::constants::SQL_DIGEST_PREFIX; + +/// `sha256:` followed by the hex SHA-256 of the statement's UTF-8 bytes. +#[must_use] +pub fn sql_digest(sql: &str) -> String { + format!("{SQL_DIGEST_PREFIX}{:x}", Sha256::digest(sql.as_bytes())) +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/mod.rs b/crates/sqlbuild-core/src/diagnostic_routing/mod.rs new file mode 100644 index 0000000000..042268f15e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/mod.rs @@ -0,0 +1,6 @@ +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/models.rs b/crates/sqlbuild-core/src/diagnostic_routing/models.rs new file mode 100644 index 0000000000..fcd0a626ce --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/models.rs @@ -0,0 +1,24 @@ +//! Facts about one logging record that routing decisions read. + +/// One record's logger family, level and SQL fields. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordFacts { + /// The record comes from the SQLBuild logger or one of its children. + pub internal: bool, + pub level: i64, + /// The record carries a SQL digest field. + pub has_sql_digest: bool, + /// The record carries a SQL text field. + pub has_sql_text: bool, +} + +/// Which records one diagnostic destination receives. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RouteOptions { + /// Only SQLBuild records reach this destination. + pub internal_only: bool, + /// SQLBuild records reach this destination. + pub allow_internal: bool, + /// This destination receives SQL text, so it only accepts records that carry it. + pub include_sql_text: bool, +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/tests.rs b/crates/sqlbuild-core/src/diagnostic_routing/tests.rs new file mode 100644 index 0000000000..0f89c014b0 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_routing.rs"] +mod routing; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs new file mode 100644 index 0000000000..59b840ef14 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_routing.rs @@ -0,0 +1,200 @@ +//! Diagnostic routing decisions match the Python logging filter and compute log handler. + +use crate::diagnostic_routing::main::compute_log_admits::compute_log_admits; +use crate::diagnostic_routing::main::diagnostic_severity::diagnostic_severity; +use crate::diagnostic_routing::main::internal_logger::internal_logger; +use crate::diagnostic_routing::main::route_admits::route_admits; +use crate::diagnostic_routing::main::sql_digest::sql_digest; +use crate::diagnostic_routing::models::{RecordFacts, RouteOptions}; +use crate::diagnostic_routing::tests::test_types::{ + ComputeLogTestCase, DigestTestCase, InternalLoggerTestCase, RouteTestCase, SeverityTestCase, +}; + +const EXTERNAL_DEBUG: RecordFacts = RecordFacts { + internal: false, + level: 10, + has_sql_digest: false, + has_sql_text: false, +}; +const DIGEST_ONLY: RecordFacts = RecordFacts { + internal: true, + level: 10, + has_sql_digest: true, + has_sql_text: false, +}; +const CONSOLE: RouteOptions = RouteOptions { + internal_only: false, + allow_internal: true, + include_sql_text: false, +}; + +#[test] +fn given_logger_names_when_classifying_then_only_sqlbuild_family_is_internal() { + let test_cases = [ + InternalLoggerTestCase { + description: "root", + name: "sqlbuild", + expected_internal: true, + }, + InternalLoggerTestCase { + description: "child", + name: "sqlbuild.sql", + expected_internal: true, + }, + InternalLoggerTestCase { + description: "shared prefix without separator", + name: "sqlbuildx", + expected_internal: false, + }, + InternalLoggerTestCase { + description: "third party", + name: "other", + expected_internal: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + internal_logger(test_case.name), + test_case.expected_internal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_route_options_when_routing_then_family_level_and_sql_rules_apply() { + let test_cases = [ + RouteTestCase { + description: "external debug is below INFO", + options: CONSOLE, + record: EXTERNAL_DEBUG, + expected_admitted: false, + }, + RouteTestCase { + description: "internal debug reaches a destination allowing internal records", + options: CONSOLE, + record: DIGEST_ONLY, + expected_admitted: true, + }, + RouteTestCase { + description: "private SQL route skips digest-only records", + options: RouteOptions { + include_sql_text: true, + ..CONSOLE + }, + record: DIGEST_ONLY, + expected_admitted: false, + }, + RouteTestCase { + description: "internal records need allow_internal", + options: RouteOptions { + allow_internal: false, + ..CONSOLE + }, + record: DIGEST_ONLY, + expected_admitted: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + route_admits(test_case.options, test_case.record), + test_case.expected_admitted, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_compute_log_records_when_admitting_then_sql_without_text_is_dropped_with_text_retention() { + let test_cases = [ + ComputeLogTestCase { + description: "external debug is below INFO", + record: EXTERNAL_DEBUG, + include_sql_text: false, + expected_admitted: false, + }, + ComputeLogTestCase { + description: "digest-only SQL is kept without text retention", + record: DIGEST_ONLY, + include_sql_text: false, + expected_admitted: true, + }, + ComputeLogTestCase { + description: "digest-only SQL is skipped with text retention", + record: DIGEST_ONLY, + include_sql_text: true, + expected_admitted: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + compute_log_admits(test_case.record, test_case.include_sql_text), + test_case.expected_admitted, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_logging_levels_when_mapping_then_severity_names_follow_thresholds() { + let test_cases = [ + SeverityTestCase { + description: "below DEBUG", + level: 5, + expected_severity: "debug", + }, + SeverityTestCase { + description: "between INFO and WARNING", + level: 25, + expected_severity: "info", + }, + SeverityTestCase { + description: "WARNING", + level: 30, + expected_severity: "warning", + }, + SeverityTestCase { + description: "ERROR", + level: 40, + expected_severity: "error", + }, + SeverityTestCase { + description: "above CRITICAL", + level: 99, + expected_severity: "critical", + }, + ]; + + for test_case in test_cases { + assert_eq!( + diagnostic_severity(test_case.level), + test_case.expected_severity, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_when_digesting_then_prefixed_sha256_hex() { + let test_cases = [DigestTestCase { + description: "empty statement", + sql: "", + expected_digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }]; + + for test_case in test_cases { + assert_eq!( + sql_digest(test_case.sql), + test_case.expected_digest, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs new file mode 100644 index 0000000000..275aa0b285 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_routing/tests/test_types.rs @@ -0,0 +1,33 @@ +use crate::diagnostic_routing::models::{RecordFacts, RouteOptions}; + +pub(super) struct InternalLoggerTestCase { + pub(super) description: &'static str, + pub(super) name: &'static str, + pub(super) expected_internal: bool, +} + +pub(super) struct RouteTestCase { + pub(super) description: &'static str, + pub(super) options: RouteOptions, + pub(super) record: RecordFacts, + pub(super) expected_admitted: bool, +} + +pub(super) struct ComputeLogTestCase { + pub(super) description: &'static str, + pub(super) record: RecordFacts, + pub(super) include_sql_text: bool, + pub(super) expected_admitted: bool, +} + +pub(super) struct SeverityTestCase { + pub(super) description: &'static str, + pub(super) level: i64, + pub(super) expected_severity: &'static str, +} + +pub(super) struct DigestTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_digest: &'static str, +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs new file mode 100644 index 0000000000..b16f5f3402 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod record_parts; diff --git a/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs new file mode 100644 index 0000000000..3a347899cc --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/_helpers/record_parts.rs @@ -0,0 +1,58 @@ +//! The header pieces of a `--debug` console line. + +use crate::cli_text::models::CliStyle; +use crate::diagnostic_routing::constants::LOGGER_PREFIX; +use crate::diagnostic_text::models::DiagnosticRecordText; + +/// Whether Python's `str.isspace()` holds for `character`. +pub(crate) fn python_space(character: char) -> bool { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) +} + +/// The message with ` SQL` lowered to ` sql`. +fn normalized_message(message: &str) -> String { + message.replace(" SQL", " sql") +} + +/// The subject, name, event, context tokens and message, when the record has a subject. +fn context_message(record: &DiagnosticRecordText, style: CliStyle) -> Option { + let subject: &String = record.subject.as_ref()?; + let mut parts: Vec = vec![subject.clone()]; + if let Some(name) = &record.name { + parts.push(style.object_name(name)); + } + if let Some(event) = &record.event { + parts.push(event.clone()); + } + for (label, value) in &record.tokens { + parts.push(format!("{label}={value}")); + } + if let Some(duration_seconds) = record.duration_seconds { + parts.push(format!("duration={duration_seconds:.2}s")); + } + let message: String = normalized_message(&record.message); + if !message.is_empty() { + parts.push(message); + } + Some(parts.join(" ")) +} + +/// The level tag and context or logger scope that open a console line. +pub(crate) fn header(record: &DiagnosticRecordText, style: CliStyle) -> String { + let level_tag: String = format!("[{}]", record.level_name.to_lowercase()); + let context: String = context_message(record, style).unwrap_or_else(|| { + let scope: &str = record + .logger_name + .strip_prefix(LOGGER_PREFIX) + .unwrap_or(&record.logger_name); + format!( + "{} {}", + style.object_name(scope), + normalized_message(&record.message) + ) + }); + if !style.use_color { + return format!("{level_tag} {context}"); + } + format!("{} {context}", style.log_label(&level_tag)) +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/constants.rs b/crates/sqlbuild-core/src/diagnostic_text/constants.rs new file mode 100644 index 0000000000..536adb4da1 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/constants.rs @@ -0,0 +1,6 @@ +//! Statements and widths of `--debug` console lines. + +/// Statements a diagnostics console line shows inline instead of in a block. +pub(crate) const INLINE_SQL_STATEMENTS: [&str; 3] = ["BEGIN", "COMMIT", "ROLLBACK"]; +/// The rule above and below a block of logged SQL. +pub(crate) const SQL_SEPARATOR_WIDTH: usize = 80; diff --git a/crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs b/crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs new file mode 100644 index 0000000000..2b6e9c4571 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/main/format_diagnostic_record.rs @@ -0,0 +1,24 @@ +//! `format_diagnostic_record`: one `--debug` stderr line for a Python log record. + +use crate::cli_text::models::CliStyle; +use crate::diagnostic_text::_helpers::record_parts::{header, python_space}; +use crate::diagnostic_text::constants::{INLINE_SQL_STATEMENTS, SQL_SEPARATOR_WIDTH}; +use crate::diagnostic_text::models::DiagnosticRecordText; + +/// The header, then any logged SQL inline or between separator rules; no exception text. +#[must_use] +pub fn format_diagnostic_record(record: &DiagnosticRecordText, style: CliStyle) -> String { + let header: String = header(record, style); + let Some(sql) = &record.sql else { + return header; + }; + let stripped: &str = sql.trim_matches(python_space); + if INLINE_SQL_STATEMENTS.contains(&stripped.to_uppercase().as_str()) { + return format!("{header} {stripped}"); + } + let separator: String = style.muted(&"-".repeat(SQL_SEPARATOR_WIDTH)); + format!( + "{header}\n{separator}\n{}\n{separator}", + sql.trim_end_matches(python_space) + ) +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/main/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/main/mod.rs new file mode 100644 index 0000000000..e001a0c182 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/main/mod.rs @@ -0,0 +1 @@ +pub mod format_diagnostic_record; diff --git a/crates/sqlbuild-core/src/diagnostic_text/mod.rs b/crates/sqlbuild-core/src/diagnostic_text/mod.rs new file mode 100644 index 0000000000..f01181f853 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/diagnostic_text/models.rs b/crates/sqlbuild-core/src/diagnostic_text/models.rs new file mode 100644 index 0000000000..b7218c127b --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/models.rs @@ -0,0 +1,19 @@ +//! The log record fields a `--debug` console line shows. + +/// The fields of one Python log record a diagnostics console line shows. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DiagnosticRecordText { + pub level_name: String, + pub logger_name: String, + pub message: String, + /// `sqlbuild_subject`, `sqlbuild_name` and `sqlbuild_event` when they are strings. + pub subject: Option, + pub name: Option, + pub event: Option, + /// Each `label=value` context token whose record field is a string, in display order. + pub tokens: Vec<(&'static str, String)>, + /// `sqlbuild_duration_ms / 1000` when the field is an integer, as Python divides it. + pub duration_seconds: Option, + /// `sqlbuild_sql` when it is a string. + pub sql: Option, +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests.rs b/crates/sqlbuild-core/src/diagnostic_text/tests.rs new file mode 100644 index 0000000000..9069258726 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_format.rs"] +mod format; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs b/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs new file mode 100644 index 0000000000..624bebd3e9 --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/tests/test_format.rs @@ -0,0 +1,74 @@ +use crate::cli_text::models::CliStyle; +use crate::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; +use crate::diagnostic_text::models::DiagnosticRecordText; +use crate::diagnostic_text::tests::test_types::DiagnosticRecordTestCase; + +#[test] +fn given_log_record_fields_when_formatting_then_matches_console_line() { + let test_cases = [ + DiagnosticRecordTestCase { + description: "plain record drops the root logger prefix", + record: DiagnosticRecordText { + level_name: "DEBUG".to_owned(), + logger_name: "sqlbuild.compiler".to_owned(), + message: "Running SQL".to_owned(), + ..DiagnosticRecordText::default() + }, + use_color: false, + expected_text: "[debug] compiler Running sql", + }, + DiagnosticRecordTestCase { + description: "context tokens and rounded duration", + record: DiagnosticRecordText { + level_name: "INFO".to_owned(), + logger_name: "sqlbuild.exec".to_owned(), + message: "done".to_owned(), + subject: Some("model".to_owned()), + name: Some("orders".to_owned()), + tokens: vec![("phase", "run".to_owned())], + duration_seconds: Some(1.005), + ..DiagnosticRecordText::default() + }, + use_color: true, + expected_text: "\x1b[2m[info]\x1b[0m model orders phase=run duration=1.00s done", + }, + DiagnosticRecordTestCase { + description: "transaction statements stay inline", + record: DiagnosticRecordText { + level_name: "DEBUG".to_owned(), + logger_name: "sqlbuild.sql".to_owned(), + message: "Executing".to_owned(), + sql: Some(" commit\n".to_owned()), + ..DiagnosticRecordText::default() + }, + use_color: false, + expected_text: "[debug] sql Executing commit", + }, + DiagnosticRecordTestCase { + description: "other statements sit between separators", + record: DiagnosticRecordText { + level_name: "DEBUG".to_owned(), + logger_name: "other".to_owned(), + message: String::new(), + sql: Some("SELECT 1 \n".to_owned()), + ..DiagnosticRecordText::default() + }, + use_color: false, + expected_text: "[debug] other \n--------------------------------------------------------------------------------\nSELECT 1\n--------------------------------------------------------------------------------", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_diagnostic_record( + &test_case.record, + CliStyle { + use_color: test_case.use_color + } + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs b/crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs new file mode 100644 index 0000000000..46d67d836e --- /dev/null +++ b/crates/sqlbuild-core/src/diagnostic_text/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::diagnostic_text::models::DiagnosticRecordText; + +pub(crate) struct DiagnosticRecordTestCase { + pub(crate) description: &'static str, + pub(crate) record: DiagnosticRecordText, + pub(crate) use_color: bool, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs new file mode 100644 index 0000000000..08bdb0c93a --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/_helpers/credentials.rs @@ -0,0 +1,173 @@ +//! Destination credentials per warehouse adapter. + +use crate::dlt_destination::constants::{ + BIGQUERY_ADAPTER, BIGQUERY_ARGUMENTS, CREDENTIALS_ARGUMENT, DATABRICKS_ADAPTER, DLT_LOADER, + MOTHERDUCK_ADAPTER, POSTGRES_ADAPTER, SNOWFLAKE_ACCOUNT_SUFFIX, SNOWFLAKE_ADAPTER, + SNOWFLAKE_OPTIONAL_SETTINGS, SQLSERVER_ADAPTER, SQLSERVER_DEFAULT_PORT, SQLSERVER_FACTORY, +}; +use crate::dlt_destination::errors::DltDestinationError; +use crate::dlt_destination::models::DltValue; +use crate::loader_connection::constants::{DEFAULT_HOST, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT}; +use crate::loader_connection::main::connection_setting::connection_setting; +use crate::loader_connection::main::required_connection_setting::required_connection_setting; +use crate::loader_connection::models::LoaderConnection; + +/// Factory arguments, in call order. +type Arguments = Vec<(&'static str, DltValue)>; + +/// A required stripped connection setting of `adapter`. +pub(crate) fn required<'a>( + connection: &'a LoaderConnection, + key: &str, + adapter: &str, +) -> Result<&'a str, DltDestinationError> { + required_connection_setting(connection, key, adapter, DLT_LOADER) + .map_err(|message| DltDestinationError { message }) +} + +fn text(value: &str) -> DltValue { + DltValue::Text(value.to_owned()) +} + +/// The single `credentials` argument. +pub(crate) fn credentials(value: DltValue) -> Arguments { + vec![(CREDENTIALS_ARGUMENT, value)] +} + +fn snowflake_account(account: &str) -> &str { + account + .strip_suffix(SNOWFLAKE_ACCOUNT_SUFFIX) + .unwrap_or(account) +} + +fn motherduck(connection: &LoaderConnection) -> DltValue { + let database: &str = connection_setting(connection, "database").unwrap_or_default(); + let mut credentials: String = if database.starts_with(MOTHERDUCK_PREFIX) { + database.to_owned() + } else { + format!("{MOTHERDUCK_PREFIX}{database}") + }; + if let Some(token) = connection_setting(connection, "token") { + credentials = format!("{credentials}?motherduck_token={token}"); + } + DltValue::Text(credentials) +} + +fn postgres(connection: &LoaderConnection) -> Result { + let host: &str = connection_setting(connection, "host").unwrap_or(DEFAULT_HOST); + let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); + let user: &str = required(connection, "user", POSTGRES_ADAPTER)?; + let password: &str = connection_setting(connection, "password").unwrap_or_default(); + let database: &str = match connection_setting(connection, "dbname") { + Some(database) => database, + None => required(connection, "database", POSTGRES_ADAPTER)?, + }; + Ok(DltValue::Text(format!( + "postgresql://{user}:{password}@{host}:{port}/{database}" + ))) +} + +fn snowflake(connection: &LoaderConnection) -> Result { + let account: &str = required(connection, "account", SNOWFLAKE_ADAPTER)?; + let host: &str = connection_setting(connection, "host").unwrap_or(account); + let mut fields: Arguments = vec![ + ("account", text(snowflake_account(account))), + ("host", text(snowflake_account(host))), + ( + "user", + text(required(connection, "user", SNOWFLAKE_ADAPTER)?), + ), + ( + "database", + text(required(connection, "database", SNOWFLAKE_ADAPTER)?), + ), + ( + "warehouse", + text(required(connection, "warehouse", SNOWFLAKE_ADAPTER)?), + ), + ]; + for key in SNOWFLAKE_OPTIONAL_SETTINGS { + if let Some(value) = connection_setting(connection, key) { + fields.push((key, text(value))); + } + } + Ok(DltValue::Object(fields)) +} + +fn bigquery(connection: &LoaderConnection) -> Arguments { + let mut arguments: Arguments = Vec::new(); + for (argument, key) in BIGQUERY_ARGUMENTS { + if let Some(value) = connection_setting(connection, key) { + arguments.push((argument, text(value))); + } + } + arguments +} + +fn databricks(connection: &LoaderConnection) -> Result { + let mut fields: Arguments = vec![ + ( + "server_hostname", + text(required(connection, "server_hostname", DATABRICKS_ADAPTER)?), + ), + ( + "http_path", + text(required(connection, "http_path", DATABRICKS_ADAPTER)?), + ), + ( + "access_token", + text(required(connection, "token", DATABRICKS_ADAPTER)?), + ), + ]; + if let Some(catalog) = connection_setting(connection, "catalog") { + fields.push(("catalog", text(catalog))); + } + Ok(DltValue::Object(fields)) +} + +fn sqlserver(connection: &LoaderConnection) -> Result { + Ok(DltValue::Object(vec![ + ( + "host", + text(connection_setting(connection, "host").unwrap_or(DEFAULT_HOST)), + ), + ( + "port", + DltValue::Port { + default: SQLSERVER_DEFAULT_PORT, + }, + ), + ( + "database", + text(required(connection, "database", SQLSERVER_ADAPTER)?), + ), + ( + "username", + text(required(connection, "user", SQLSERVER_ADAPTER)?), + ), + ( + "password", + text(connection_setting(connection, "password").unwrap_or_default()), + ), + ( + "query", + DltValue::Object(vec![("TrustServerCertificate", text("yes"))]), + ), + ])) +} + +/// The factory and arguments of a non-DuckDB adapter, or `None` when dlt cannot load into it. +pub(crate) fn adapter_call( + adapter: &str, + connection: &LoaderConnection, +) -> Result, DltDestinationError> { + Ok(Some(match adapter { + MOTHERDUCK_ADAPTER => (MOTHERDUCK_ADAPTER, credentials(motherduck(connection))), + POSTGRES_ADAPTER => (POSTGRES_ADAPTER, credentials(postgres(connection)?)), + SNOWFLAKE_ADAPTER => (SNOWFLAKE_ADAPTER, credentials(snowflake(connection)?)), + BIGQUERY_ADAPTER => (BIGQUERY_ADAPTER, bigquery(connection)), + DATABRICKS_ADAPTER => (DATABRICKS_ADAPTER, credentials(databricks(connection)?)), + SQLSERVER_ADAPTER => (SQLSERVER_FACTORY, credentials(sqlserver(connection)?)), + _ => return Ok(None), + })) +} diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs new file mode 100644 index 0000000000..53f1c45a32 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod credentials; +pub(crate) mod request; diff --git a/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs b/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs new file mode 100644 index 0000000000..91f9b45ff2 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/_helpers/request.rs @@ -0,0 +1,48 @@ +//! Destination config and dataset rules checked before any adapter credentials. + +use crate::dlt_destination::constants::SQLBUILD_OWNED_DESTINATION_KEYS; +use crate::dlt_destination::errors::DltDestinationError; +use crate::dlt_destination::models::DltDatasetName; +use crate::text::main::python_strip::python_strip; + +/// Reject a destination config that sets keys SQLBuild owns. +pub(crate) fn check_destination_keys( + adapter: &str, + destination_keys: &[Option], +) -> Result<(), DltDestinationError> { + let mut blocked: Vec<&str> = destination_keys + .iter() + .flatten() + .map(String::as_str) + .filter(|key| SQLBUILD_OWNED_DESTINATION_KEYS.contains(key)) + .collect(); + if blocked.is_empty() { + return Ok(()); + } + blocked.sort_unstable(); + Err(DltDestinationError { + message: format!( + "Adapter '{adapter}' dlt destination config cannot define SQLBuild-owned key(s): {}", + blocked.join(", ") + ), + }) +} + +/// Reject a missing or blank dataset name, which raw landing requires. +pub(crate) fn check_dataset_name( + adapter: &str, + dataset_name: &DltDatasetName, +) -> Result<(), DltDestinationError> { + let named: bool = dataset_name + .text + .as_deref() + .is_some_and(|name| !python_strip(name).is_empty()); + if named { + return Ok(()); + } + Err(DltDestinationError { + message: format!( + "Adapter '{adapter}' requires an explicit dlt source schema for raw landing" + ), + }) +} diff --git a/crates/sqlbuild-core/src/dlt_destination/constants.rs b/crates/sqlbuild-core/src/dlt_destination/constants.rs new file mode 100644 index 0000000000..920e01ac55 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/constants.rs @@ -0,0 +1,36 @@ +//! dlt destination defaults, adapters and the keys SQLBuild owns. + +/// The integration loader name dlt connection errors cite. +pub(crate) const DLT_LOADER: &str = "dlt"; +/// Destination options SQLBuild passes unless the destination config overrides them. +pub const DEFAULT_DESTINATION_OPTIONS: [(&str, &str); 1] = [("naming_convention", "sql_ci_v1")]; +/// Destination config keys SQLBuild sets itself. +pub(crate) const SQLBUILD_OWNED_DESTINATION_KEYS: [&str; 4] = [ + "credentials", + "dataset_name", + "default_schema_name", + "project_id", +]; +/// The dataset DuckDB destinations land in without an explicit one. +pub(crate) const DUCKDB_DEFAULT_DATASET: &str = "main"; +/// The suffix Snowflake account and host settings drop. +pub(crate) const SNOWFLAKE_ACCOUNT_SUFFIX: &str = ".snowflakecomputing.com"; +/// SQL Server's default port number. +pub(crate) const SQLSERVER_DEFAULT_PORT: i64 = 1433; +/// Snowflake settings copied into credentials when set, in order. +pub(crate) const SNOWFLAKE_OPTIONAL_SETTINGS: [&str; 4] = + ["password", "role", "authenticator", "token"]; +/// BigQuery factory arguments and the connection settings they come from, in order. +pub(crate) const BIGQUERY_ARGUMENTS: [(&str, &str); 2] = + [("project_id", "project"), ("location", "location")]; +pub(crate) const DUCKDB_ADAPTER: &str = "duckdb"; +pub(crate) const MOTHERDUCK_ADAPTER: &str = "motherduck"; +pub(crate) const POSTGRES_ADAPTER: &str = "postgres"; +pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; +pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; +pub(crate) const DATABRICKS_ADAPTER: &str = "databricks"; +pub(crate) const SQLSERVER_ADAPTER: &str = "sqlserver"; +/// The dlt destination factory of SQL Server. +pub(crate) const SQLSERVER_FACTORY: &str = "mssql"; +/// The factory argument holding destination credentials. +pub(crate) const CREDENTIALS_ARGUMENT: &str = "credentials"; diff --git a/crates/sqlbuild-core/src/dlt_destination/errors.rs b/crates/sqlbuild-core/src/dlt_destination/errors.rs new file mode 100644 index 0000000000..d0ca821392 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/errors.rs @@ -0,0 +1,7 @@ +//! dlt destination construction errors. + +/// A destination the dlt integration cannot address, with its user-facing message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DltDestinationError { + pub message: String, +} diff --git a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs new file mode 100644 index 0000000000..3e06044eb3 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs @@ -0,0 +1,40 @@ +//! `dlt_destination_plan`: the dlt destination call for one adapter's connection. + +use crate::dlt_destination::_helpers::credentials::{adapter_call, credentials, required}; +use crate::dlt_destination::_helpers::request::{check_dataset_name, check_destination_keys}; +use crate::dlt_destination::constants::{DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET}; +use crate::dlt_destination::errors::DltDestinationError; +use crate::dlt_destination::models::{ + DltDataset, DltDestinationPlan, DltDestinationRequest, DltValue, +}; + +/// The dlt destination call for the request's adapter, connection and dataset. +pub fn dlt_destination_plan( + request: &DltDestinationRequest<'_>, +) -> Result { + let adapter: &str = request.adapter; + check_destination_keys(adapter, request.destination_keys)?; + if adapter == DUCKDB_ADAPTER { + let database: &str = required(request.connection, "database", DUCKDB_ADAPTER)?; + return Ok(DltDestinationPlan { + factory: DUCKDB_ADAPTER, + arguments: credentials(DltValue::Text(database.to_owned())), + dataset: if request.dataset_name.truthy { + DltDataset::Given + } else { + DltDataset::Default(DUCKDB_DEFAULT_DATASET) + }, + }); + } + check_dataset_name(adapter, request.dataset_name)?; + let Some((factory, arguments)) = adapter_call(adapter, request.connection)? else { + return Err(DltDestinationError { + message: format!("Adapter '{adapter}' does not support dlt integration loaders"), + }); + }; + Ok(DltDestinationPlan { + factory, + arguments, + dataset: DltDataset::Given, + }) +} diff --git a/crates/sqlbuild-core/src/dlt_destination/main/mod.rs b/crates/sqlbuild-core/src/dlt_destination/main/mod.rs new file mode 100644 index 0000000000..b8507d2941 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/main/mod.rs @@ -0,0 +1 @@ +pub mod dlt_destination_plan; diff --git a/crates/sqlbuild-core/src/dlt_destination/mod.rs b/crates/sqlbuild-core/src/dlt_destination/mod.rs new file mode 100644 index 0000000000..4be5b95029 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/mod.rs @@ -0,0 +1,10 @@ +//! The dlt destination factory and arguments for one SQLBuild warehouse adapter. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/dlt_destination/models.rs b/crates/sqlbuild-core/src/dlt_destination/models.rs new file mode 100644 index 0000000000..3e1d881496 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/models.rs @@ -0,0 +1,49 @@ +//! A dlt destination factory call and the facts it is built from. + +use crate::loader_connection::models::LoaderConnection; + +/// One argument value of a dlt destination factory call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DltValue { + Text(String), + /// The connection's raw `port` setting, or `default` when unset. + Port { + default: i64, + }, + Object(Vec<(&'static str, DltValue)>), +} + +/// The dataset a dlt destination lands in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DltDataset { + /// The dataset name the caller gave. + Given, + Default(&'static str), +} + +/// The caller's dataset name. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DltDatasetName { + /// The name when it is a string. + pub text: Option, + /// Python truthiness of the name. + pub truthy: bool, +} + +/// One `dlt.destinations.(**arguments, **options)` call and its dataset. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DltDestinationPlan { + pub factory: &'static str, + pub arguments: Vec<(&'static str, DltValue)>, + pub dataset: DltDataset, +} + +/// One adapter's connection, destination config keys and dataset name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DltDestinationRequest<'a> { + pub adapter: &'a str, + pub connection: &'a LoaderConnection, + /// The destination config's keys, each `None` when not a string. + pub destination_keys: &'a [Option], + pub dataset_name: &'a DltDatasetName, +} diff --git a/crates/sqlbuild-core/src/dlt_destination/tests.rs b/crates/sqlbuild-core/src/dlt_destination/tests.rs new file mode 100644 index 0000000000..48fa2dd946 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_dlt_destination_plan.rs"] +mod dlt_destination_plan; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs new file mode 100644 index 0000000000..418fd98879 --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs @@ -0,0 +1,108 @@ +//! dlt destination calls match the Python integration's. + +use crate::dlt_destination::main::dlt_destination_plan::dlt_destination_plan; +use crate::dlt_destination::models::{ + DltDataset, DltDatasetName, DltDestinationPlan, DltDestinationRequest, DltValue, +}; +use crate::dlt_destination::tests::test_types::DestinationPlanTestCase; +use crate::loader_connection::models::LoaderConnection; + +#[test] +fn given_adapter_connection_when_planning_dlt_destination_then_matches_python_mapping() { + let test_cases = [ + DestinationPlanTestCase { + description: "duckdb without dataset lands in main", + adapter: "duckdb", + strings: &[("database", " warehouse.duckdb ")], + port: None, + destination_keys: &[], + dataset_name: None, + expected_plan: Ok(DltDestinationPlan { + factory: "duckdb", + arguments: vec![("credentials", DltValue::Text("warehouse.duckdb".to_owned()))], + dataset: DltDataset::Default("main"), + }), + }, + DestinationPlanTestCase { + description: "postgres connection string uses host default and port", + adapter: "postgres", + strings: &[("user", "u"), ("database", "d")], + port: Some("6543"), + destination_keys: &["staging"], + dataset_name: Some("raw"), + expected_plan: Ok(DltDestinationPlan { + factory: "postgres", + arguments: vec![( + "credentials", + DltValue::Text("postgresql://u:@localhost:6543/d".to_owned()), + )], + dataset: DltDataset::Given, + }), + }, + DestinationPlanTestCase { + description: "owned destination keys are sorted in the error", + adapter: "postgres", + strings: &[], + port: None, + destination_keys: &["dataset_name", "credentials"], + dataset_name: Some("raw"), + expected_plan: Err( + "Adapter 'postgres' dlt destination config cannot define SQLBuild-owned key(s): credentials, dataset_name", + ), + }, + DestinationPlanTestCase { + description: "blank dataset is rejected before credentials", + adapter: "snowflake", + strings: &[], + port: None, + destination_keys: &[], + dataset_name: Some(" "), + expected_plan: Err( + "Adapter 'snowflake' requires an explicit dlt source schema for raw landing", + ), + }, + DestinationPlanTestCase { + description: "missing required setting names the dlt loader", + adapter: "snowflake", + strings: &[], + port: None, + destination_keys: &[], + dataset_name: Some("raw"), + expected_plan: Err( + "Adapter 'snowflake' requires connection.account for dlt integration loaders", + ), + }, + ]; + + for test_case in test_cases { + let connection = LoaderConnection { + strings: test_case + .strings + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + port: test_case.port.map(str::to_owned), + }; + let destination_keys: Vec> = test_case + .destination_keys + .iter() + .map(|key| Some((*key).to_owned())) + .collect(); + let dataset_name = DltDatasetName { + text: test_case.dataset_name.map(str::to_owned), + truthy: test_case.dataset_name.is_some_and(|name| !name.is_empty()), + }; + let request = DltDestinationRequest { + adapter: test_case.adapter, + connection: &connection, + destination_keys: &destination_keys, + dataset_name: &dataset_name, + }; + assert_eq!( + dlt_destination_plan(&request).map_err(|error| error.message), + test_case.expected_plan.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs new file mode 100644 index 0000000000..ea416660fd --- /dev/null +++ b/crates/sqlbuild-core/src/dlt_destination/tests/test_types.rs @@ -0,0 +1,11 @@ +use crate::dlt_destination::models::DltDestinationPlan; + +pub(super) struct DestinationPlanTestCase { + pub(super) description: &'static str, + pub(super) adapter: &'static str, + pub(super) strings: &'static [(&'static str, &'static str)], + pub(super) port: Option<&'static str>, + pub(super) destination_keys: &'static [&'static str], + pub(super) dataset_name: Option<&'static str>, + pub(super) expected_plan: Result, +} diff --git a/crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs b/crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs new file mode 100644 index 0000000000..6dabcf061c --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/is_lowercase_identifier.rs @@ -0,0 +1,9 @@ +//! Lowercase ASCII identifiers: `re.fullmatch(r"^[a-z][a-z0-9_]*$", value)`. + +/// Whether `value` is a lowercase ASCII letter followed by lowercase letters, digits or `_`. +#[must_use] +pub fn is_lowercase_identifier(value: &str) -> bool { + let mut bytes = value.bytes(); + bytes.next().is_some_and(|byte| byte.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') +} diff --git a/crates/sqlbuild-core/src/identifier_text/main/mod.rs b/crates/sqlbuild-core/src/identifier_text/main/mod.rs new file mode 100644 index 0000000000..a29dc793b1 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/main/mod.rs @@ -0,0 +1 @@ +pub mod is_lowercase_identifier; diff --git a/crates/sqlbuild-core/src/identifier_text/mod.rs b/crates/sqlbuild-core/src/identifier_text/mod.rs new file mode 100644 index 0000000000..5db00853a7 --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/mod.rs @@ -0,0 +1,6 @@ +//! Identifier spellings authored names must follow. + +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/identifier_text/tests.rs b/crates/sqlbuild-core/src/identifier_text/tests.rs new file mode 100644 index 0000000000..149a53f52e --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_lowercase_identifier.rs"] +mod lowercase_identifier; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs b/crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs new file mode 100644 index 0000000000..b06b3563da --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/tests/test_lowercase_identifier.rs @@ -0,0 +1,76 @@ +use crate::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; +use crate::identifier_text::tests::test_types::LowercaseIdentifierTestCase; + +#[test] +fn given_names_when_checked_then_only_lowercase_ascii_identifiers_pass() { + let test_cases = [ + LowercaseIdentifierTestCase { + description: "a plain name", + value: "orders", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "one letter", + value: "a", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "digits and underscores", + value: "a1_b", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "a trailing underscore", + value: "abc_", + expected_valid: true, + }, + LowercaseIdentifierTestCase { + description: "an empty name", + value: "", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "an uppercase letter", + value: "A", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a leading digit", + value: "1a", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a leading underscore", + value: "_a", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a hyphen", + value: "a-b", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a trailing newline", + value: "ab\n", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "an accented letter", + value: "\u{e9}", + expected_valid: false, + }, + LowercaseIdentifierTestCase { + description: "a dotless i", + value: "a\u{131}", + expected_valid: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + is_lowercase_identifier(test_case.value), + test_case.expected_valid, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/identifier_text/tests/test_types.rs b/crates/sqlbuild-core/src/identifier_text/tests/test_types.rs new file mode 100644 index 0000000000..8b57aa418f --- /dev/null +++ b/crates/sqlbuild-core/src/identifier_text/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(crate) struct LowercaseIdentifierTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) expected_valid: bool, +} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs new file mode 100644 index 0000000000..6a1951b26e --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/connection.rs @@ -0,0 +1,37 @@ +//! Connection settings read the way the ingestr destination expects them. + +use crate::ingestr_command::constants::INGESTR_LOADER; +use crate::ingestr_command::errors::IngestrCommandError; +use crate::loader_connection::main::connection_setting::connection_setting; +use crate::loader_connection::main::required_connection_setting::required_connection_setting; +use crate::loader_connection::models::LoaderConnection; + +/// The stripped string setting `key`, or `None` when unset, not a string or blank. +pub(crate) fn optional<'a>(connection: &'a LoaderConnection, key: &str) -> Option<&'a str> { + connection_setting(connection, key) +} + +/// The stripped string setting `key`, which the adapter's ingestr destination requires. +pub(crate) fn required<'a>( + connection: &'a LoaderConnection, + key: &str, + adapter_name: &str, +) -> Result<&'a str, IngestrCommandError> { + required_connection_setting(connection, key, adapter_name, INGESTR_LOADER) + .map_err(|message| IngestrCommandError { message }) +} + +/// `(key, value)` when the setting `key` is set. +fn setting<'a>(connection: &'a LoaderConnection, key: &'a str) -> Option<(&'a str, &'a str)> { + optional(connection, key).map(|value| (key, value)) +} + +/// The settings among `keys` that are set, in `keys` order. +pub(crate) fn present<'a>( + connection: &'a LoaderConnection, + keys: &[&'a str], +) -> Vec<(&'a str, &'a str)> { + keys.iter() + .filter_map(|key| setting(connection, key)) + .collect() +} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs new file mode 100644 index 0000000000..86b6532aa8 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs @@ -0,0 +1,131 @@ +//! Destination URIs per warehouse adapter. + +use crate::ingestr_command::_helpers::connection::{optional, present, required}; +use crate::ingestr_command::_helpers::url_quote::{query_suffix, quote, urlencode}; +use crate::ingestr_command::constants::{ + BIGQUERY_PARAMETERS, DATABRICKS_PARAMETERS, SNOWFLAKE_PARAMETERS, SQLSERVER_DEFAULT_DRIVER, + SQLSERVER_DEFAULT_PORT, SQLSERVER_DRIVER_PARAMETER, SQLSERVER_PARAMETERS, + SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, +}; +use crate::ingestr_command::errors::IngestrCommandError; +use crate::loader_connection::constants::{DEFAULT_HOST, MOTHERDUCK_PREFIX, POSTGRES_DEFAULT_PORT}; +use crate::loader_connection::models::LoaderConnection; + +/// DuckDB's database file URI. +pub(crate) fn duckdb(connection: &LoaderConnection) -> Result { + Ok(format!( + "duckdb:///{}", + required(connection, "database", "duckdb")? + )) +} + +/// MotherDuck's database URI with its optional token. +pub(crate) fn motherduck(connection: &LoaderConnection) -> String { + let database: &str = optional(connection, "database").unwrap_or_default(); + let database: &str = database.strip_prefix(MOTHERDUCK_PREFIX).unwrap_or(database); + let token: Vec<(&str, &str)> = optional(connection, "token") + .map(|token| ("token", token)) + .into_iter() + .collect(); + format!("motherduck://{}{}", quote(database), query_suffix(&token)) +} + +/// Postgres's URI with its optional SSL mode. +pub(crate) fn postgres(connection: &LoaderConnection) -> Result { + let host: &str = optional(connection, "host").unwrap_or(DEFAULT_HOST); + let port: &str = connection.port.as_deref().unwrap_or(POSTGRES_DEFAULT_PORT); + let user: &str = required(connection, "user", "postgres")?; + let password: &str = optional(connection, "password").unwrap_or_default(); + let database: &str = match optional(connection, "dbname") { + Some(database) => database, + None => required(connection, "database", "postgres")?, + }; + let sslmode: Vec<(&str, &str)> = present(connection, &["sslmode"]); + Ok(format!( + "postgresql://{}:{}@{host}:{port}/{}{}", + quote(user), + quote(password), + quote(database), + query_suffix(&sslmode) + )) +} + +/// BigQuery's project URI with its optional credentials. +pub(crate) fn bigquery(connection: &LoaderConnection) -> Result { + let project: &str = match optional(connection, "project") { + Some(project) => project, + None => required(connection, "database", "bigquery")?, + }; + Ok(format!( + "bigquery://{project}{}", + query_suffix(&present(connection, &BIGQUERY_PARAMETERS)) + )) +} + +/// Snowflake's account URI with its optional session settings. +pub(crate) fn snowflake(connection: &LoaderConnection) -> Result { + let account: &str = required(connection, "account", "snowflake")?; + let user: &str = required(connection, "user", "snowflake")?; + let password: &str = optional(connection, "password").unwrap_or_default(); + let database: &str = required(connection, "database", "snowflake")?; + let schema: &str = required(connection, "schema", "snowflake")?; + Ok(format!( + "snowflake://{}:{}@{account}/{}/{}{}", + quote(user), + quote(password), + quote(database), + quote(schema), + query_suffix(&present(connection, &SNOWFLAKE_PARAMETERS)) + )) +} + +/// Databricks's token URI with its warehouse path. +pub(crate) fn databricks(connection: &LoaderConnection) -> Result { + let host: &str = required(connection, "server_hostname", "databricks")?; + let token: &str = required(connection, "token", "databricks")?; + let mut parameters: Vec<(&str, &str)> = vec![( + "http_path", + required(connection, "http_path", "databricks")?, + )]; + parameters.extend(present(connection, &DATABRICKS_PARAMETERS)); + Ok(format!( + "databricks://token:{}@{host}?{}", + quote(token), + urlencode(¶meters) + )) +} + +/// SQL Server's URI with its ODBC defaults. +pub(crate) fn sqlserver(connection: &LoaderConnection) -> String { + let first = |keys: [&str; 2], default: &'static str| -> String { + keys.iter() + .find_map(|key| optional(connection, key)) + .unwrap_or(default) + .to_owned() + }; + let host: String = first(["host", "server"], DEFAULT_HOST); + let port: &str = connection.port.as_deref().unwrap_or(SQLSERVER_DEFAULT_PORT); + let user: String = first(["user", "username"], "sa"); + let password: &str = optional(connection, "password").unwrap_or_default(); + let database: String = first(["database", "dbname"], "master"); + let mut parameters: Vec<(&str, &str)> = present(connection, &SQLSERVER_PARAMETERS); + let defaults: [(&str, &str); 2] = [ + (SQLSERVER_DRIVER_PARAMETER, SQLSERVER_DEFAULT_DRIVER), + (SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, "yes"), + ]; + for (key, value) in defaults { + if !parameters + .iter() + .any(|(present_key, _)| *present_key == key) + { + parameters.push((key, value)); + } + } + format!( + "mssql://{}:{}@{host}:{port}/{}{}", + quote(&user), + quote(password), + quote(&database), + query_suffix(¶meters) + ) +} diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs new file mode 100644 index 0000000000..0b39ed4d18 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod connection; +pub(crate) mod destinations; +pub(crate) mod url_quote; diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs new file mode 100644 index 0000000000..d2566d4c35 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/url_quote.rs @@ -0,0 +1,42 @@ +//! Python's `urllib.parse.quote` and `urlencode` over string pairs. + +use crate::ingestr_command::constants::URL_ALWAYS_SAFE; + +fn quote_with(text: &str, safe: &[u8]) -> String { + let mut quoted: String = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || URL_ALWAYS_SAFE.contains(&byte) || safe.contains(&byte) { + quoted.push(char::from(byte)); + } else { + quoted.push_str(&format!("%{byte:02X}")); + } + } + quoted +} + +/// Python's `urllib.parse.quote(text)`, which keeps `/`. +pub(crate) fn quote(text: &str) -> String { + quote_with(text, b"/") +} + +fn quote_plus(text: &str) -> String { + quote_with(text, b" ").replace(' ', "+") +} + +/// Python's `urlencode` of ordered string pairs. +pub(crate) fn urlencode(pairs: &[(&str, &str)]) -> String { + pairs + .iter() + .map(|(key, value)| format!("{}={}", quote_plus(key), quote_plus(value))) + .collect::>() + .join("&") +} + +/// `?` and the encoded pairs, or nothing when there are none. +pub(crate) fn query_suffix(pairs: &[(&str, &str)]) -> String { + if pairs.is_empty() { + String::new() + } else { + format!("?{}", urlencode(pairs)) + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/constants.rs b/crates/sqlbuild-core/src/ingestr_command/constants.rs new file mode 100644 index 0000000000..12475e6949 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/constants.rs @@ -0,0 +1,35 @@ +//! Ingestr command vocabulary and destination defaults. + +/// The fixed leading arguments of every ingestr run. +pub(crate) const COMMAND_PREFIX: [&str; 2] = ["ingestr", "ingest"]; +/// Progress reporting arguments ingestr runs with. +pub(crate) const COMMAND_PROGRESS: [&str; 3] = ["--yes", "--progress", "log"]; +/// Characters Python's `urllib.parse.quote` never escapes. +pub(crate) const URL_ALWAYS_SAFE: &[u8] = b"_.-~"; +/// SQL Server's default port. +pub(crate) const SQLSERVER_DEFAULT_PORT: &str = "1433"; +/// The SQL Server ODBC driver ingestr uses unless the connection names one. +pub(crate) const SQLSERVER_DEFAULT_DRIVER: &str = "ODBC Driver 18 for SQL Server"; +/// Optional BigQuery query parameters, in URI order. +pub(crate) const BIGQUERY_PARAMETERS: [&str; 3] = + ["location", "credentials_path", "credentials_base64"]; +/// Optional Snowflake query parameters, in URI order. +pub(crate) const SNOWFLAKE_PARAMETERS: [&str; 6] = [ + "warehouse", + "role", + "token", + "private_key", + "private_key_passphrase", + "authenticator", +]; +/// Optional Databricks query parameters after `http_path`, in URI order. +pub(crate) const DATABRICKS_PARAMETERS: [&str; 2] = ["catalog", "schema"]; +/// Optional SQL Server query parameters, in URI order. +pub(crate) const SQLSERVER_PARAMETERS: [&str; 3] = + ["driver", "TrustServerCertificate", "Authentication"]; +/// The SQL Server driver query parameter. +pub(crate) const SQLSERVER_DRIVER_PARAMETER: &str = "driver"; +/// The SQL Server certificate trust query parameter. +pub(crate) const SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER: &str = "TrustServerCertificate"; +/// The integration loader name ingestr connection errors cite. +pub(crate) const INGESTR_LOADER: &str = "ingestr"; diff --git a/crates/sqlbuild-core/src/ingestr_command/errors.rs b/crates/sqlbuild-core/src/ingestr_command/errors.rs new file mode 100644 index 0000000000..bb10c5e272 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/errors.rs @@ -0,0 +1,7 @@ +//! Ingestr command construction errors. + +/// A destination the ingestr integration cannot address, with its user-facing message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IngestrCommandError { + pub message: String, +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs new file mode 100644 index 0000000000..dc8c3ed27d --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/format_ingestr_command.rs @@ -0,0 +1,13 @@ +//! The display form of an ingestr command line. + +use crate::shell_text::main::shell_quote::shell_quote; + +/// The arguments shell-quoted and joined with spaces, like Python's `shlex.join`. +#[must_use] +pub fn format_ingestr_command(arguments: &[String]) -> String { + arguments + .iter() + .map(|argument| shell_quote(argument)) + .collect::>() + .join(" ") +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs new file mode 100644 index 0000000000..34eabc624e --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_command_arguments.rs @@ -0,0 +1,43 @@ +//! The `ingestr ingest` arguments for one source run. + +use crate::ingestr_command::constants::{COMMAND_PREFIX, COMMAND_PROGRESS}; +use crate::ingestr_command::models::{IngestrDestination, IngestrSource}; + +/// The ingestr command line that loads `source` into `destination`. +#[must_use] +pub fn ingestr_command_arguments( + source: &IngestrSource, + destination: &IngestrDestination, +) -> Vec { + let mut arguments: Vec = COMMAND_PREFIX.map(str::to_owned).to_vec(); + let pairs: [(&str, &str); 4] = [ + ("--source-uri", &source.source_uri), + ("--source-table", &source.source_table), + ("--dest-uri", &destination.uri), + ("--dest-table", &destination.table), + ]; + let optional: [(&str, Option<&String>); 2] = [ + ("--incremental-strategy", source.strategy.as_ref()), + ("--incremental-key", source.incremental_key.as_ref()), + ]; + for (flag, value) in pairs { + arguments.extend([flag.to_owned(), value.to_owned()]); + } + arguments.extend(COMMAND_PROGRESS.map(str::to_owned)); + for (flag, value) in optional { + if let Some(value) = value { + arguments.extend([flag.to_owned(), value.clone()]); + } + } + for primary_key in &source.primary_key { + arguments.extend(["--primary-key".to_owned(), primary_key.clone()]); + } + if let Some(columns) = &source.columns { + arguments.extend(["--columns".to_owned(), columns.clone()]); + } + if destination.is_reload { + arguments.push("--full-refresh".to_owned()); + } + arguments.extend(source.extra_args.iter().cloned()); + arguments +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs new file mode 100644 index 0000000000..67fd30a1f4 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_table.rs @@ -0,0 +1,10 @@ +//! The relation an ingestr source run writes. + +/// `schema.target_name` when the source declares a schema, otherwise `target_name`. +#[must_use] +pub fn ingestr_destination_table(schema: Option<&str>, target_name: &str) -> String { + match schema { + Some(schema) => format!("{schema}.{target_name}"), + None => target_name.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs new file mode 100644 index 0000000000..198bf35e36 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs @@ -0,0 +1,28 @@ +//! The ingestr destination URI for a SQLBuild warehouse connection. + +use crate::ingestr_command::_helpers::destinations::{ + bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlserver, +}; +use crate::ingestr_command::errors::IngestrCommandError; +use crate::loader_connection::models::LoaderConnection; + +/// The ingestr destination URI for the connection of the adapter named `adapter_name`. +pub fn ingestr_destination_uri( + adapter_name: &str, + connection: &LoaderConnection, +) -> Result { + match adapter_name { + "duckdb" => duckdb(connection), + "motherduck" => Ok(motherduck(connection)), + "postgres" => postgres(connection), + "bigquery" => bigquery(connection), + "snowflake" => snowflake(connection), + "databricks" => databricks(connection), + "sqlserver" => Ok(sqlserver(connection)), + _ => Err(IngestrCommandError { + message: format!( + "Adapter '{adapter_name}' does not support ingestr integration loaders" + ), + }), + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/main/mod.rs b/crates/sqlbuild-core/src/ingestr_command/main/mod.rs new file mode 100644 index 0000000000..68329553b8 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/main/mod.rs @@ -0,0 +1,4 @@ +pub mod format_ingestr_command; +pub mod ingestr_command_arguments; +pub mod ingestr_destination_table; +pub mod ingestr_destination_uri; diff --git a/crates/sqlbuild-core/src/ingestr_command/mod.rs b/crates/sqlbuild-core/src/ingestr_command/mod.rs new file mode 100644 index 0000000000..db623d7553 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/mod.rs @@ -0,0 +1,10 @@ +//! The `ingestr ingest` command line for one declarative ingestr source and its warehouse. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/ingestr_command/models.rs b/crates/sqlbuild-core/src/ingestr_command/models.rs new file mode 100644 index 0000000000..bbfc6e6af1 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/models.rs @@ -0,0 +1,22 @@ +//! Inputs of one ingestr command line. + +/// A resolved ingestr source configuration. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct IngestrSource { + pub source_uri: String, + pub source_table: String, + pub strategy: Option, + pub incremental_key: Option, + pub primary_key: Vec, + pub columns: Option, + pub extra_args: Vec, +} + +/// Where one ingestr run writes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IngestrDestination { + pub uri: String, + pub table: String, + /// The run rebuilds the destination from scratch. + pub is_reload: bool, +} diff --git a/crates/sqlbuild-core/src/ingestr_command/tests.rs b/crates/sqlbuild-core/src/ingestr_command/tests.rs new file mode 100644 index 0000000000..ed3facc731 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_ingestr_command.rs"] +mod ingestr_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs new file mode 100644 index 0000000000..e35dd77651 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs @@ -0,0 +1,125 @@ +//! Ingestr destination URIs and command lines match the Python integration's. + +use crate::ingestr_command::main::format_ingestr_command::format_ingestr_command; +use crate::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; +use crate::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; +use crate::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; +use crate::ingestr_command::models::{IngestrDestination, IngestrSource}; +use crate::ingestr_command::tests::test_types::{CommandTestCase, DestinationUriTestCase}; +use crate::loader_connection::models::LoaderConnection; + +#[test] +fn given_connection_when_building_destination_uri_then_matches_python_quoting() { + let test_cases = [ + DestinationUriTestCase { + description: "duckdb path", + adapter_name: "duckdb", + strings: &[("database", " warehouse.duckdb ")], + port: None, + expected_uri: Ok("duckdb:///warehouse.duckdb"), + }, + DestinationUriTestCase { + description: "motherduck prefix and token", + adapter_name: "motherduck", + strings: &[("database", "md:my db/é"), ("token", "t o/k=n&")], + port: None, + expected_uri: Ok("motherduck://my%20db/%C3%A9?token=t+o%2Fk%3Dn%26"), + }, + DestinationUriTestCase { + description: "postgres with sslmode and port", + adapter_name: "postgres", + strings: &[ + ("user", "ana lyst"), + ("password", "p@ss:/w"), + ("dbname", "orders"), + ], + port: Some("6543"), + expected_uri: Ok("postgresql://ana%20lyst:p%40ss%3A/w@localhost:6543/orders"), + }, + DestinationUriTestCase { + description: "sqlserver defaults", + adapter_name: "sqlserver", + strings: &[("password", "Sqlbuild!2026"), ("database", "analytics")], + port: None, + expected_uri: Ok("mssql://sa:Sqlbuild%212026@localhost:1433/analytics\ + ?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes"), + }, + DestinationUriTestCase { + description: "snowflake requires an account", + adapter_name: "snowflake", + strings: &[("account", " ")], + port: None, + expected_uri: Err( + "Adapter 'snowflake' requires connection.account for ingestr integration loaders", + ), + }, + DestinationUriTestCase { + description: "unsupported adapter", + adapter_name: "fabric", + strings: &[], + port: None, + expected_uri: Err("Adapter 'fabric' does not support ingestr integration loaders"), + }, + ]; + for test_case in test_cases { + let connection: LoaderConnection = LoaderConnection { + strings: test_case + .strings + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect(), + port: test_case.port.map(str::to_owned), + }; + let uri: Result = + ingestr_destination_uri(test_case.adapter_name, &connection) + .map_err(|error| error.message); + assert_eq!( + uri.as_deref().map_err(String::as_str), + test_case.expected_uri, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_source_when_building_command_then_orders_arguments_like_python() { + let test_cases = [ + CommandTestCase { + description: "reload with schema", + schema: Some("raw"), + is_reload: true, + expected_display: "ingestr ingest --source-uri 'csv://a b' --source-table t \ + --dest-uri duckdb:///w.duckdb --dest-table raw.orders --yes --progress log \ + --primary-key id --full-refresh '--note=it'\"'\"'s'", + }, + CommandTestCase { + description: "incremental without schema", + schema: None, + is_reload: false, + expected_display: "ingestr ingest --source-uri 'csv://a b' --source-table t \ + --dest-uri duckdb:///w.duckdb --dest-table orders --yes --progress log \ + --primary-key id '--note=it'\"'\"'s'", + }, + ]; + let source: IngestrSource = IngestrSource { + source_uri: "csv://a b".to_owned(), + source_table: "t".to_owned(), + primary_key: vec!["id".to_owned()], + extra_args: vec!["--note=it's".to_owned()], + ..IngestrSource::default() + }; + for test_case in test_cases { + let destination: IngestrDestination = IngestrDestination { + uri: "duckdb:///w.duckdb".to_owned(), + table: ingestr_destination_table(test_case.schema, "orders"), + is_reload: test_case.is_reload, + }; + assert_eq!( + format_ingestr_command(&ingestr_command_arguments(&source, &destination)), + test_case.expected_display, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs new file mode 100644 index 0000000000..e406920491 --- /dev/null +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(super) struct DestinationUriTestCase { + pub(super) description: &'static str, + pub(super) adapter_name: &'static str, + pub(super) strings: &'static [(&'static str, &'static str)], + pub(super) port: Option<&'static str>, + pub(super) expected_uri: Result<&'static str, &'static str>, +} + +pub(super) struct CommandTestCase { + pub(super) description: &'static str, + pub(super) schema: Option<&'static str>, + pub(super) is_reload: bool, + pub(super) expected_display: &'static str, +} diff --git a/crates/sqlbuild-core/src/init_command/constants.rs b/crates/sqlbuild-core/src/init_command/constants.rs index efd035371f..340fd783a4 100644 --- a/crates/sqlbuild-core/src/init_command/constants.rs +++ b/crates/sqlbuild-core/src/init_command/constants.rs @@ -19,9 +19,6 @@ pub const SCAFFOLD_DIRECTORIES: [&str; 15] = [ "audits/singular", ]; -/// The project configuration file name. -pub const PROJECT_FILE: &str = "sqlbuild_project.toml"; - /// The placeholder file kept in otherwise empty directories. pub const GITKEEP_FILE: &str = ".gitkeep"; diff --git a/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs b/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs index 300bf21589..6f40e2863f 100644 --- a/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs +++ b/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs @@ -1,6 +1,7 @@ //! The summary printed after a project is created. use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::init_command::constants::NEXT_STEPS; /// The created header, project and config fields, and the numbered next steps. @@ -10,7 +11,7 @@ pub fn render_init_summary(project_name: &str, style: CliStyle) -> String { document.header("SQLBuild project created", None); document.lines.push(String::new()); document.field("Project", (project_name, " ")); - document.field("Config", ("sqlbuild_project.toml", " ")); + document.field("Config", (PROJECT_CONFIG_FILENAME, " ")); document.lines.push(String::new()); document.section("Next steps"); document diff --git a/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs b/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs index 79fabfc34e..b87556f24f 100644 --- a/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs +++ b/crates/sqlbuild-core/src/janitor_text/main/janitor_disabled_text.rs @@ -1,6 +1,7 @@ //! Guidance when janitor is not enabled for the project. use crate::cli_text::models::CliStyle; +use crate::constants::PROJECT_CONFIG_FILENAME; use crate::janitor_text::constants::DISABLED_DETAIL; /// How to enable janitor and preview cleanup. @@ -11,7 +12,7 @@ pub fn janitor_disabled_text(style: CliStyle) -> String { After enabling, run janitor again to preview cleanup:\n {}\n", style.warning_strong("Janitor is disabled for this project."), style.warning(DISABLED_DETAIL), - style.object_name("sqlbuild_project.toml"), + style.object_name(PROJECT_CONFIG_FILENAME), style.accent("sqb janitor") ) } diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 4d3832c03d..99383f712f 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -24,14 +24,22 @@ pub mod dbt_graph; pub mod dbt_plan; pub mod dbt_process; pub mod debug_report; +pub mod diagnostic_routing; +pub mod diagnostic_text; +pub mod dlt_destination; pub mod event_export; pub mod format_command; pub mod freshness_text; +pub mod identifier_text; +pub mod ingestr_command; pub mod init_command; pub mod integration_results; pub mod janitor_text; pub mod json; +pub mod lifecycle_events; pub mod load_command; +pub mod loader_connection; +pub mod migration_values; pub mod models; pub mod nested_progress; pub mod panics; @@ -42,6 +50,7 @@ pub mod query_command; pub mod redaction; pub mod scenario_command; pub mod seed_command; +pub mod shell_text; pub mod terminal_events; pub mod test_command; pub mod text; diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs new file mode 100644 index 0000000000..d6366eb7b5 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/envelope.rs @@ -0,0 +1,20 @@ +//! The catalogue definition and correlations of one event envelope. + +use crate::lifecycle_events::main::catalog::catalog; +use crate::lifecycle_events::models::{EventDefinition, KnownEventFacts}; + +/// Whether the named correlation field is set on the event. +pub(crate) fn correlation_set(event: &KnownEventFacts, name: &str) -> bool { + event + .correlations + .iter() + .any(|(field, set)| *field == name && *set) +} + +/// The catalogue definition of the event's type and schema version, if catalogued. +pub(crate) fn definition(event: &KnownEventFacts) -> Option { + let entries: Vec = catalog(event.schema_version)?; + entries + .into_iter() + .find(|entry| entry.event_type == event.event_type) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs new file mode 100644 index 0000000000..6ef184878e --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/fields.rs @@ -0,0 +1,123 @@ +//! Payload value predicates and the per-field type contract. + +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; +use crate::lifecycle_events::constants::{ + BOOLEAN_PAYLOAD_FIELDS, DURATION_MS_FIELD, EXIT_CODE_FIELD, FINITE_NUMBER_PAYLOAD_FIELDS, + JSON_PAYLOAD_FIELDS, MAX_METADATA_BYTES, METADATA_FIELD, MINIMUM_DURATION_MS, + NONNEGATIVE_INTEGER_PAYLOAD_FIELDS, STRING_PAYLOAD_FIELDS, +}; +use crate::lifecycle_events::models::PayloadValue; +use crate::text::main::python_str_repr::python_str_repr; + +/// `payload.get(name)`, treating a JSON null like an absent field. +pub(crate) fn get<'a>( + payload: &'a [(String, PayloadValue)], + name: &str, +) -> Option<&'a PayloadValue> { + payload + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value) + .filter(|value| !matches!(value, PayloadValue::Null)) +} + +/// The integer a value holds, excluding booleans. +pub(crate) fn integer(value: Option<&PayloadValue>) -> Option { + match value { + Some(PayloadValue::Integer { value, .. }) => Some(*value), + _ => None, + } +} + +/// Whether a value is text listed in `allowed`. +pub(crate) fn catalogued(value: Option<&PayloadValue>, allowed: &[&str]) -> bool { + matches!(value, Some(PayloadValue::Text(text)) if allowed.contains(&text.as_str())) +} + +fn finite_number(value: &PayloadValue) -> Option { + match value { + PayloadValue::Integer { value, .. } => Some(*value as f64), + PayloadValue::Float(number) if number.is_finite() => Some(*number), + _ => None, + } +} + +fn json(value: &PayloadValue) -> JsonValue { + match value { + PayloadValue::Null => JsonValue::Null, + PayloadValue::Bool(flag) => JsonValue::Bool(*flag), + PayloadValue::Integer { text, .. } => { + JsonInteger::parse(text).map_or(JsonValue::Null, JsonValue::Integer) + } + PayloadValue::Float(number) => JsonValue::Float(*number), + PayloadValue::Text(text) => JsonValue::String(text.clone()), + PayloadValue::Array(items) => JsonValue::Array(items.iter().map(json).collect()), + PayloadValue::Object(entries) => JsonValue::Object( + entries + .iter() + .map(|(key, item)| (key.clone(), json(item))) + .collect(), + ), + } +} + +/// The byte length of `json.dumps(value, sort_keys=True, separators=(",", ":"))`. +fn encoded_length(value: &PayloadValue) -> usize { + let options = StdlibJsonOptions { + sort_keys: true, + allow_nan: false, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + dumps(&json(value), &JsonDialect::Stdlib(options)).map_or(0, |text| text.len()) +} + +/// The type contract of one payload field, as the first failing message. +pub(crate) fn field_problem(name: &str, value: &PayloadValue) -> Option { + let quoted: String = python_str_repr(name); + if STRING_PAYLOAD_FIELDS.contains(&name) { + let valid: bool = matches!(value, PayloadValue::Text(text) if !text.is_empty()); + return (!valid).then(|| format!("payload field {quoted} must be a non-empty string")); + } + if NONNEGATIVE_INTEGER_PAYLOAD_FIELDS.contains(&name) { + let valid: bool = integer(Some(value)).is_some_and(|number| number >= 0); + return (!valid).then(|| { + format!("payload field {quoted} must be a nonnegative integer excluding bool") + }); + } + if FINITE_NUMBER_PAYLOAD_FIELDS.contains(&name) { + return finite_number(value) + .is_none() + .then(|| format!("payload field {quoted} must be a finite number excluding bool")); + } + if BOOLEAN_PAYLOAD_FIELDS.contains(&name) { + return (!matches!(value, PayloadValue::Bool(_))) + .then(|| format!("payload field {quoted} must be a boolean")); + } + if JSON_PAYLOAD_FIELDS.contains(&name) { + return None; + } + if name == DURATION_MS_FIELD { + let valid: bool = finite_number(value).is_some_and(|number| number >= MINIMUM_DURATION_MS); + return (!valid).then(|| { + "payload field 'duration_ms' must be a nonnegative finite number excluding bool" + .to_owned() + }); + } + if name == EXIT_CODE_FIELD { + return integer(Some(value)) + .is_none() + .then(|| "payload field 'exit_code' must be an integer excluding bool".to_owned()); + } + if name == METADATA_FIELD { + if !matches!(value, PayloadValue::Object(_)) { + return Some("payload field 'metadata' must be a JSON object".to_owned()); + } + if encoded_length(value) > MAX_METADATA_BYTES { + return Some(format!( + "payload field 'metadata' must encode to at most {MAX_METADATA_BYTES} bytes" + )); + } + } + None +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs new file mode 100644 index 0000000000..7c28c938e5 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod envelope; +pub(crate) mod fields; +pub(crate) mod payloads; diff --git a/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs new file mode 100644 index 0000000000..e57df01e02 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/_helpers/payloads.rs @@ -0,0 +1,248 @@ +//! Event-family payload rules: statements, operations, microbatches, skips, retries and runs. + +use std::collections::BTreeSet; + +use crate::lifecycle_events::_helpers::fields::field_problem; +use crate::lifecycle_events::_helpers::fields::{catalogued, get, integer}; +use crate::lifecycle_events::constants::{ + AUDIT_RUN_COUNT_FIELDS, CONFIGURED_CONCURRENCY_FIELD, FORBIDDEN_STATEMENT_PAYLOAD_FIELDS, + HOOK_NAME_MAX_TAIL, HOOK_PHASES, HOOK_TYPES, METADATA_FIELD, MICROBATCH_EVENT_PREFIX, + MICROBATCH_KINDS, MICROBATCH_RUN_TYPES, MICROBATCH_STRATEGIES, OPERATION_ADAPTERS, + OPERATION_EVENT_PREFIX, OPERATION_KINDS, OPERATION_METADATA_FIELDS, OPERATION_NAMES, + OPERATION_PHASES, OPERATION_SCOPES, OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, + RESOURCE_ATTEMPT_SKIPPED_EVENT, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, + RETRY_SCHEDULED_EVENT, RUN_STARTED_EVENT, RUN_TERMINALS, STATEMENT_EVENT_PREFIX, +}; +use crate::lifecycle_events::models::{KnownEventFacts, PayloadValue}; +use crate::lifecycle_events::types::PayloadCheck; + +type Payload = [(String, PayloadValue)]; + +/// Every forbidden SQL or parameter key anywhere in a statement payload. +pub(crate) fn forbidden_statement_fields(value: &PayloadValue) -> BTreeSet { + let mut found: BTreeSet = BTreeSet::new(); + match value { + PayloadValue::Object(entries) => { + for (key, item) in entries { + if FORBIDDEN_STATEMENT_PAYLOAD_FIELDS.contains(&key.as_str()) { + found.insert(key.clone()); + } + found.extend(forbidden_statement_fields(item)); + } + } + PayloadValue::Array(items) => { + for item in items { + found.extend(forbidden_statement_fields(item)); + } + } + _ => {} + } + found +} + +pub(crate) fn safe_hook_name(name: &str) -> bool { + let mut characters = name.chars(); + let Some(first) = characters.next() else { + return false; + }; + (first.is_ascii_alphabetic() || first == '_') + && name.len() <= HOOK_NAME_MAX_TAIL + 1 + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +fn uncatalogued(payload: &Payload, name: &str, allowed: &[&str]) -> bool { + get(payload, name).is_some_and(|value| !catalogued(Some(value), allowed)) +} + +/// Catalogued operation dimensions, hook identity and allowlisted metadata counts. +pub(crate) fn operation_problem(payload: &Payload) -> Option { + if uncatalogued(payload, "operation_kind", &OPERATION_KINDS) { + return Some("operation_kind must be a catalogued value".to_owned()); + } + if uncatalogued(payload, "operation_name", &OPERATION_NAMES) { + return Some("operation_name must be a catalogued value".to_owned()); + } + let dimensions: [(&str, &[&str]); 5] = [ + ("phase", &OPERATION_PHASES), + ("strategy", &OPERATION_STRATEGIES), + ("target_kind", &OPERATION_TARGET_KINDS), + ("adapter", &OPERATION_ADAPTERS), + ("scope", &OPERATION_SCOPES), + ]; + for (name, allowed) in dimensions { + if uncatalogued(payload, name, allowed) { + return Some(format!("{name} must be a catalogued value")); + } + } + let hook_fields = ["hook_phase", "hook_type", "hook_index", "hook_name"]; + if hook_fields.iter().any(|name| get(payload, name).is_some()) { + if !catalogued(get(payload, "hook_phase"), &HOOK_PHASES) { + return Some("hook_phase must be a catalogued value".to_owned()); + } + if !catalogued(get(payload, "hook_type"), &HOOK_TYPES) { + return Some("hook_type must be a catalogued value".to_owned()); + } + if !integer(get(payload, "hook_index")).is_some_and(|index| index >= 0) { + return Some("hook_index must be a nonnegative integer".to_owned()); + } + let unsafe_name: bool = match get(payload, "hook_name") { + None => false, + Some(PayloadValue::Text(name)) => !safe_hook_name(name), + Some(_) => true, + }; + if unsafe_name { + return Some("hook_name must be a non-empty safe token".to_owned()); + } + } + let entries: &Vec<(String, PayloadValue)> = match get(payload, METADATA_FIELD)? { + PayloadValue::Object(entries) => entries, + _ => return Some("operation metadata must be a JSON object".to_owned()), + }; + if entries + .iter() + .any(|(key, _)| !OPERATION_METADATA_FIELDS.contains(&key.as_str())) + { + return Some("operation metadata contains a non-allowlisted field".to_owned()); + } + entries + .iter() + .any(|(_, value)| !integer(Some(value)).is_some_and(|count| count >= 0)) + .then(|| "operation metadata values must be nonnegative integers excluding bool".to_owned()) +} + +/// Positive, ordered batch positions and catalogued microbatch dimensions. +pub(crate) fn microbatch_problem(payload: &Payload) -> Option { + let Some(index) = integer(get(payload, "batch_index")).filter(|index| *index >= 1) else { + return Some("batch_index must be a positive integer".to_owned()); + }; + let Some(count) = integer(get(payload, "batch_count")).filter(|count| *count >= 1) else { + return Some("batch_count must be a positive integer".to_owned()); + }; + if index > count { + return Some("batch_index must not exceed batch_count".to_owned()); + } + let dimensions: [(&str, &[&str]); 3] = [ + ("microbatch_strategy", &MICROBATCH_STRATEGIES), + ("microbatch_run_type", &MICROBATCH_RUN_TYPES), + ("batch_kind", &MICROBATCH_KINDS), + ]; + dimensions + .into_iter() + .find(|(name, allowed)| !catalogued(get(payload, name), allowed)) + .map(|(name, _)| format!("{name} must be a catalogued value")) +} + +/// A catalogued skip code and, when present, skip mode. +pub(crate) fn skip_problem(payload: &Payload) -> Option { + if !catalogued(get(payload, "skip_code"), &RESOURCE_SKIP_CODES) { + return Some("skip_code must be a catalogued value".to_owned()); + } + uncatalogued(payload, "skip_mode", &RESOURCE_SKIP_MODES) + .then(|| "skip_mode must be a catalogued value".to_owned()) +} + +/// A positive failed attempt followed by the next attempt number. +pub(crate) fn retry_problem(payload: &Payload) -> Option { + let Some(failed) = integer(get(payload, "failed_attempt_number")).filter(|number| *number > 0) + else { + return Some("failed_attempt_number must be a positive integer".to_owned()); + }; + (integer(get(payload, "next_attempt_number")) != Some(failed + 1)) + .then(|| "next_attempt_number must equal failed_attempt_number + 1".to_owned()) +} + +fn run_count(payload: &Payload, name: &str) -> Result { + integer(get(payload, name)) + .filter(|count| *count >= 0) + .ok_or_else(|| format!("{name} must be a nonnegative integer excluding bool")) +} + +/// Worker counts of a started run and audit totals of a finished run. +pub(crate) fn run_problem(event_type: &str, payload: &Payload) -> Option { + let present = |name: &str| payload.iter().any(|(key, _)| key == name); + if event_type == RUN_STARTED_EVENT && present(CONFIGURED_CONCURRENCY_FIELD) { + let Some(configured) = + integer(get(payload, CONFIGURED_CONCURRENCY_FIELD)).filter(|count| *count >= 1) + else { + return Some( + "configured_concurrency must be a positive integer excluding bool".to_owned(), + ); + }; + let Some(selected) = integer(get(payload, "selected_count")).filter(|count| *count >= 0) + else { + return Some("selected_count must be a nonnegative integer excluding bool".to_owned()); + }; + let Some(workers) = integer(get(payload, "worker_count")) else { + return Some("worker_count must be an integer excluding bool".to_owned()); + }; + if workers != configured.min(selected) { + return Some( + "worker_count must equal min(configured_concurrency, selected_count)".to_owned(), + ); + } + } + if !RUN_TERMINALS.contains(&event_type) + || !AUDIT_RUN_COUNT_FIELDS.iter().any(|name| present(name)) + { + return None; + } + if !AUDIT_RUN_COUNT_FIELDS.iter().all(|name| present(name)) { + return Some( + "audit run terminals must include pass_count, warn_count, and fail_count together" + .to_owned(), + ); + } + let counts = || -> Result, String> { + let pass: i128 = run_count(payload, "pass_count")?; + let warn: i128 = run_count(payload, "warn_count")?; + let fail: i128 = run_count(payload, "fail_count")?; + let succeeded: i128 = run_count(payload, "succeeded_count")?; + let failed: i128 = run_count(payload, "failed_count")?; + if succeeded != pass + warn { + return Ok(Some( + "succeeded_count must equal pass_count + warn_count".to_owned(), + )); + } + Ok((failed != fail).then(|| "failed_count must equal fail_count".to_owned())) + }; + counts().unwrap_or_else(Some) +} + +/// The first statement, operation, microbatch, skip, retry, field or run violation. +pub(crate) fn family_problem(event: &KnownEventFacts) -> Option { + let event_type: &str = &event.event_type; + if event_type.starts_with(STATEMENT_EVENT_PREFIX) { + let forbidden: BTreeSet = + forbidden_statement_fields(&PayloadValue::Object(event.payload.clone())); + if !forbidden.is_empty() { + let names: Vec = forbidden.into_iter().collect(); + return Some(format!( + "statement lifecycle payload must not contain SQL or parameter values; \ +forbidden field(s): {}", + names.join(", ") + )); + } + } + let checks: [(bool, PayloadCheck); 4] = [ + ( + event_type.starts_with(OPERATION_EVENT_PREFIX), + operation_problem, + ), + ( + event_type.starts_with(MICROBATCH_EVENT_PREFIX), + microbatch_problem, + ), + (event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT, skip_problem), + (event_type == RETRY_SCHEDULED_EVENT, retry_problem), + ]; + for (applies, check) in checks { + if let Some(problem) = applies.then(|| check(&event.payload)).flatten() { + return Some(problem); + } + } + event + .payload + .iter() + .find_map(|(name, value)| field_problem(name, value)) + .or_else(|| run_problem(event_type, &event.payload)) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs new file mode 100644 index 0000000000..8743401fb4 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -0,0 +1,1078 @@ +//! The versioned lifecycle event catalogue and the payload value contracts it validates. + +use crate::lifecycle_events::types::CatalogEntry; + +/// The lifecycle event schema version SQLBuild produces. +pub const CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: i64 = 2; +/// Every lifecycle event schema version with a catalogue, oldest first. +pub const LIFECYCLE_EVENT_SCHEMA_VERSIONS: [i64; 2] = [1, 2]; +/// The only lifecycle event schema version without invocation metadata. +pub(crate) const UNSEQUENCED_SCHEMA_VERSION: i128 = 1; +/// The diagnostic log schema version SQLBuild produces and reads. +pub const CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: i64 = 1; +/// Every diagnostic log severity. +pub const DIAGNOSTIC_SEVERITIES: [&str; 6] = + ["critical", "debug", "error", "info", "trace", "warning"]; +/// The producer whose version SQLBuild supplies itself. +pub const DEFAULT_PRODUCER: &str = "sqlbuild"; +/// The message for an invalid invocation sequence. +pub const INVOCATION_SEQUENCE_PROBLEM: &str = + "invocation_sequence must be a non-negative integer excluding bool"; +pub(crate) const MAX_METADATA_BYTES: usize = 4096; +/// The smallest duration a payload may carry. +pub(crate) const MINIMUM_DURATION_MS: f64 = 0.0; +/// The most characters a hook name may hold after its first. +pub(crate) const HOOK_NAME_MAX_TAIL: usize = 127; +pub(crate) const DURATION_MS_FIELD: &str = "duration_ms"; +pub(crate) const EXIT_CODE_FIELD: &str = "exit_code"; +pub(crate) const METADATA_FIELD: &str = "metadata"; +pub(crate) const CONFIGURED_CONCURRENCY_FIELD: &str = "configured_concurrency"; +pub(crate) const RUN_STARTED_EVENT: &str = "run_started"; +pub(crate) const STATEMENT_EVENT_PREFIX: &str = "statement_"; +pub(crate) const OPERATION_EVENT_PREFIX: &str = "operation_"; +pub(crate) const MICROBATCH_EVENT_PREFIX: &str = "microbatch_"; +pub(crate) const RESOURCE_ATTEMPT_SKIPPED_EVENT: &str = "resource_attempt_skipped"; +pub(crate) const RETRY_SCHEDULED_EVENT: &str = "retry_scheduled"; +pub(crate) const RUN_TERMINALS: [&str; 2] = ["run_completed", "run_failed"]; +pub(crate) const AUDIT_RUN_COUNT_FIELDS: [&str; 3] = ["fail_count", "pass_count", "warn_count"]; +pub(crate) const MICROBATCH_STRATEGIES: [&str; 2] = ["rolling_window", "watermark"]; +pub(crate) const MICROBATCH_RUN_TYPES: [&str; 3] = ["backfill", "normal", "replay_on_change"]; +pub(crate) const MICROBATCH_KINDS: [&str; 2] = ["ordinary", "recovery"]; +pub const OPERATION_TARGET_KINDS: [&str; 7] = [ + "audit", + "namespace", + "relation", + "source", + "sql_test", + "staging_relation", + "state_batch", +]; +pub const OPERATION_SCOPES: [&str; 7] = [ + "delta", + "end", + "final", + "model", + "seed", + "source", + "standalone", +]; +pub const OPERATION_PHASES: [&str; 11] = [ + "apply", + "assert", + "convert", + "create", + "evaluate", + "inspect", + "observe", + "promote", + "publish", + "reconcile", + "setup", +]; +pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [ + "bindings", + "full_sql", + "parameter_values", + "parameters", + "params", + "query_sql", + "sql", + "statement_sql", +]; +pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ + "adapter", + "attached_column_name", + "attached_target_kind", + "attached_target_name", + "attachment_kind", + "audit_definition_name", + "audit_description", + "audit_name", + "batch_kind", + "binding_key", + "command", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "definition_fingerprint", + "effective_batch_size", + "error_code", + "error_type", + "evaluation_mode", + "evidence_error", + "evidence_sql", + "executed_sql", + "execution_error", + "execution_fingerprint", + "hook_name", + "hook_phase", + "hook_type", + "intent", + "job_id", + "measurement_sql", + "microbatch_run_type", + "microbatch_strategy", + "operation_kind", + "operation_name", + "outcome", + "planned_cursor_end_exclusive", + "planned_cursor_start", + "query_id", + "resource_kind", + "resource_name", + "result_id", + "run_kind", + "run_scope_phase", + "sample_unit", + "scope", + "severity", + "sql_digest", + "statement_kind", + "target_database", + "target_name", + "target_schema", +]; +pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 25] = [ + "affected_rows", + "attempt_number", + "batch_count", + "batch_index", + "batch_size", + "configured_concurrency", + "delay_ms", + "evidence_count", + "fail_count", + "failed_attempt_number", + "failed_count", + "hook_index", + "minimum_samples", + "next_attempt_number", + "pass_count", + "process_id", + "row_count", + "sample_count", + "selected_count", + "signal_number", + "skipped_count", + "succeeded_count", + "violation_count", + "warn_count", + "worker_count", +]; +pub(crate) const FINITE_NUMBER_PAYLOAD_FIELDS: [&str; 1] = ["measured_value"]; +pub(crate) const BOOLEAN_PAYLOAD_FIELDS: [&str; 2] = ["evidence_truncated", "reused"]; +pub(crate) const JSON_PAYLOAD_FIELDS: [&str; 2] = ["evidence", "thresholds"]; +pub const RESOURCE_SKIP_CODES: [&str; 4] = ["dependency", "explicit", "fan_in", "scheduler"]; +pub const RESOURCE_SKIP_MODES: [&str; 2] = ["hard", "soft"]; +pub const HOOK_PHASES: [&str; 2] = ["post_hooks", "pre_hooks"]; +pub const HOOK_TYPES: [&str; 2] = ["python", "sql"]; +pub const OPERATION_KINDS: [&str; 10] = [ + "clone", + "freshness", + "janitor", + "loader", + "project", + "python_node", + "quality", + "scenario", + "subprocess", + "warehouse", +]; +pub const OPERATION_NAMES: [&str; 55] = [ + "audit_evaluation", + "clone_execution", + "clone_finalization", + "clone_namespace_preparation", + "clone_relation_inspection", + "clone_relation_transfer", + "clone_retention_reconciliation", + "clone_state_connection", + "clone_state_inspection", + "clone_target_connection", + "dbt_command", + "discovery_declaration_parse", + "discovery_filesystem_walk", + "discovery_project_assembly", + "discovery_python_import", + "external_manifest_discovery", + "external_source_load", + "ingestr_command", + "janitor_candidate_planning", + "janitor_cleanup_action", + "janitor_execution", + "janitor_state_inspection", + "janitor_target_connection", + "janitor_warehouse_inspection", + "managed_source_load", + "project_compile", + "project_discovery", + "python_asset", + "python_check", + "python_hook", + "python_materialization", + "python_task", + "relation_promotion", + "retention_application", + "retention_inspection", + "runtime_schema_inspection", + "scenario_capture", + "scenario_cleanup", + "scenario_execution", + "scenario_relation_read", + "scenario_schema_inspection", + "scenario_snapshot_serialization", + "scenario_snapshot_write", + "scenario_target_connection", + "schema_synchronization", + "source_freshness_metadata_observation", + "source_freshness_publication", + "source_freshness_query_observation", + "sql_hook", + "sql_test_assertion", + "sql_test_planning", + "sql_test_setup", + "staging_creation", + "table_type_conversion", + "table_type_inspection", +]; +pub const OPERATION_METADATA_FIELDS: [&str; 9] = [ + "added_count", + "altered_count", + "attempt_number", + "byte_count", + "changed_count", + "item_count", + "removed_count", + "row_count", + "unknown_count", +]; +pub const OPERATION_STRATEGIES: [&str; 10] = [ + "adapter", + "append_new_columns", + "atomic_replace", + "atomic_swap", + "build_aside", + "column", + "create_new", + "rename", + "sql", + "sync_all_columns", +]; +pub const OPERATION_ADAPTERS: [&str; 8] = [ + "bigquery", + "custom", + "databricks", + "duckdb", + "motherduck", + "postgres", + "snowflake", + "sqlserver", +]; + +/// The schema version 1 catalogue in declaration order. +pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ + ("invocation_started", &[], &[], &["command"], false), + ( + "invocation_completed", + &[], + &[], + &["command", "duration_ms", "exit_code"], + true, + ), + ( + "invocation_failed", + &[], + &[], + &[ + "command", + "duration_ms", + "error_code", + "error_type", + "exit_code", + ], + true, + ), + ( + "run_started", + &["run_id"], + &[], + &[ + "configured_concurrency", + "run_kind", + "selected_count", + "worker_count", + ], + false, + ), + ( + "run_completed", + &["run_id"], + &[], + &[ + "duration_ms", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "run_failed", + &["run_id"], + &[], + &[ + "duration_ms", + "error_code", + "error_type", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "resource_attempt_started", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &["attempt_number", "resource_kind", "resource_name"], + false, + ), + ( + "resource_attempt_completed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_failed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "error_code", + "error_type", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_skipped", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "attempt_number", + "resource_kind", + "resource_name", + "skip_code", + ], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + "skip_code", + "skip_mode", + ], + true, + ), + ( + "operation_started", + &["operation_id"], + &[], + &[ + "adapter", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "scope", + "strategy", + "target_kind", + ], + false, + ), + ( + "operation_completed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "operation_failed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "error_code", + "error_type", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "retry_scheduled", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "delay_ms", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + &[ + "delay_ms", + "error_code", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + false, + ), + ( + "audit_completed", + &["run_id"], + &[ + "attachment_kind", + "audit_definition_name", + "audit_name", + "evaluation_mode", + "outcome", + "run_scope_phase", + "severity", + ], + &[ + "attached_column_name", + "attached_target_kind", + "attached_target_name", + "attachment_kind", + "audit_definition_name", + "audit_description", + "audit_name", + "binding_key", + "definition_fingerprint", + "evaluation_mode", + "evidence", + "evidence_count", + "evidence_error", + "evidence_sql", + "evidence_truncated", + "executed_sql", + "execution_error", + "execution_fingerprint", + "measured_value", + "measurement_sql", + "minimum_samples", + "outcome", + "result_id", + "reused", + "run_scope_phase", + "sample_count", + "sample_unit", + "severity", + "sql_digest", + "target_database", + "target_name", + "target_schema", + "thresholds", + "violation_count", + ], + true, + ), + ( + "statement_started", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_submitted", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_heartbeat", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_completed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), + ( + "statement_failed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "error_code", + "error_type", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), +]; +/// The schema version 2 catalogue in declaration order. +pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ + ("invocation_started", &[], &[], &["command"], false), + ( + "invocation_completed", + &[], + &[], + &["command", "duration_ms", "exit_code"], + true, + ), + ( + "invocation_failed", + &[], + &[], + &[ + "command", + "duration_ms", + "error_code", + "error_type", + "exit_code", + ], + true, + ), + ( + "run_started", + &["run_id"], + &[], + &[ + "configured_concurrency", + "run_kind", + "selected_count", + "worker_count", + ], + false, + ), + ( + "run_completed", + &["run_id"], + &[], + &[ + "duration_ms", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "run_failed", + &["run_id"], + &[], + &[ + "duration_ms", + "error_code", + "error_type", + "fail_count", + "failed_count", + "pass_count", + "run_kind", + "skipped_count", + "succeeded_count", + "warn_count", + ], + true, + ), + ( + "resource_attempt_started", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &["attempt_number", "resource_kind", "resource_name"], + false, + ), + ( + "resource_attempt_completed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_failed", + &["resource_attempt_id", "resource_id", "run_id"], + &[], + &[ + "attempt_number", + "duration_ms", + "error_code", + "error_type", + "resource_kind", + "resource_name", + ], + true, + ), + ( + "resource_attempt_skipped", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "attempt_number", + "resource_kind", + "resource_name", + "skip_code", + ], + &[ + "attempt_number", + "duration_ms", + "resource_kind", + "resource_name", + "skip_code", + "skip_mode", + ], + true, + ), + ( + "operation_started", + &["operation_id"], + &[], + &[ + "adapter", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "scope", + "strategy", + "target_kind", + ], + false, + ), + ( + "operation_completed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "operation_failed", + &["operation_id"], + &[], + &[ + "adapter", + "duration_ms", + "error_code", + "error_type", + "exit_code", + "hook_index", + "hook_name", + "hook_phase", + "hook_type", + "metadata", + "operation_kind", + "operation_name", + "phase", + "process_id", + "scope", + "signal_number", + "strategy", + "target_kind", + ], + true, + ), + ( + "retry_scheduled", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "delay_ms", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + &[ + "delay_ms", + "error_code", + "error_type", + "failed_attempt_number", + "next_attempt_number", + ], + false, + ), + ( + "audit_completed", + &["run_id"], + &[ + "attachment_kind", + "audit_definition_name", + "audit_name", + "evaluation_mode", + "outcome", + "run_scope_phase", + "severity", + ], + &[ + "attached_column_name", + "attached_target_kind", + "attached_target_name", + "attachment_kind", + "audit_definition_name", + "audit_description", + "audit_name", + "binding_key", + "definition_fingerprint", + "evaluation_mode", + "evidence", + "evidence_count", + "evidence_error", + "evidence_sql", + "evidence_truncated", + "executed_sql", + "execution_error", + "execution_fingerprint", + "measured_value", + "measurement_sql", + "minimum_samples", + "outcome", + "result_id", + "reused", + "run_scope_phase", + "sample_count", + "sample_unit", + "severity", + "sql_digest", + "target_database", + "target_name", + "target_schema", + "thresholds", + "violation_count", + ], + true, + ), + ( + "statement_started", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_submitted", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "intent", + "job_id", + "metadata", + "query_id", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_heartbeat", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + false, + ), + ( + "statement_completed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), + ( + "statement_failed", + &["statement_id"], + &[], + &[ + "adapter", + "affected_rows", + "batch_size", + "duration_ms", + "error_code", + "error_type", + "intent", + "job_id", + "metadata", + "query_id", + "row_count", + "sql_digest", + "statement_kind", + ], + true, + ), + ( + "microbatch_started", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + false, + ), + ( + "microbatch_completed", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + &[ + "affected_rows", + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "duration_ms", + "effective_batch_size", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + true, + ), + ( + "microbatch_failed", + &["resource_attempt_id", "resource_id", "run_id"], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "effective_batch_size", + "error_type", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + &[ + "batch_count", + "batch_index", + "batch_kind", + "configured_batch_size", + "cursor_end_exclusive", + "cursor_start", + "duration_ms", + "effective_batch_size", + "error_code", + "error_type", + "microbatch_run_type", + "microbatch_strategy", + "planned_cursor_end_exclusive", + "planned_cursor_start", + ], + true, + ), +]; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs new file mode 100644 index 0000000000..74d374d42a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs @@ -0,0 +1,27 @@ +//! `catalog`: the catalogued event definitions of one schema version, in declaration order. + +use crate::lifecycle_events::constants::{LIFECYCLE_EVENT_CATALOG_V1, LIFECYCLE_EVENT_CATALOG_V2}; +use crate::lifecycle_events::models::EventDefinition; +use crate::lifecycle_events::types::CatalogEntry; + +fn definition(entry: &CatalogEntry) -> EventDefinition { + let (event_type, required_correlations, required_payload, allowed_payload, terminal) = *entry; + EventDefinition { + event_type, + required_correlations, + required_payload, + allowed_payload, + terminal, + } +} + +/// Every definition of `schema_version`, or `None` for an uncatalogued version. +#[must_use] +pub fn catalog(schema_version: i64) -> Option> { + let entries: &[CatalogEntry] = match schema_version { + 1 => &LIFECYCLE_EVENT_CATALOG_V1, + 2 => &LIFECYCLE_EVENT_CATALOG_V2, + _ => return None, + }; + Some(entries.iter().map(definition).collect()) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs new file mode 100644 index 0000000000..d19e85f0ad --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_severity_problem.rs @@ -0,0 +1,10 @@ +//! `diagnostic_severity_problem`: the severity rule of a diagnostic log. + +use crate::lifecycle_events::constants::DIAGNOSTIC_SEVERITIES; + +/// The rule a diagnostic severity breaks. +#[must_use] +pub fn diagnostic_severity_problem(severity: &str) -> Option<&'static str> { + (!DIAGNOSTIC_SEVERITIES.contains(&severity)) + .then_some("severity must be one of trace, debug, info, warning, error, critical") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs new file mode 100644 index 0000000000..8f50767a3a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/diagnostic_version_problem.rs @@ -0,0 +1,11 @@ +//! `diagnostic_version_problem`: the schema version rule of a diagnostic log. + +use crate::lifecycle_events::constants::CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION; + +/// The rule a diagnostic log's schema version breaks. +#[must_use] +pub fn diagnostic_version_problem(schema_version: i128) -> Option { + (schema_version != i128::from(CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION)).then(|| { + format!("diagnostic schema_version must be {CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION}") + }) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs new file mode 100644 index 0000000000..e3b76f7602 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/hook_problem.rs @@ -0,0 +1,19 @@ +//! `hook_problem`: the catalogue rules of an operation's hook fields. + +use crate::lifecycle_events::constants::{HOOK_PHASES, HOOK_TYPES}; +use crate::lifecycle_events::models::HookFacts; + +/// The first rule supplied hook fields break. +#[must_use] +pub fn hook_problem(hook: HookFacts<'_>) -> Option<&'static str> { + if !hook.phase.is_some_and(|phase| HOOK_PHASES.contains(&phase)) { + return Some("hook_phase must be a catalogued value"); + } + if !hook + .hook_type + .is_some_and(|hook_type| HOOK_TYPES.contains(&hook_type)) + { + return Some("hook_type must be a catalogued value"); + } + (!hook.valid_index).then_some("hook_index must be a nonnegative integer") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs new file mode 100644 index 0000000000..cf90f4ef27 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/lifecycle_version_problem.rs @@ -0,0 +1,32 @@ +//! `lifecycle_version_problem`: the schema version and invocation metadata rules of an event. + +use crate::lifecycle_events::constants::{ + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, INVOCATION_SEQUENCE_PROBLEM, + LIFECYCLE_EVENT_SCHEMA_VERSIONS, UNSEQUENCED_SCHEMA_VERSION, +}; +use crate::lifecycle_events::models::SequenceFact; + +/// The first rule a lifecycle event's schema version and invocation metadata break. +#[must_use] +pub fn lifecycle_version_problem( + schema_version: i128, + sequence: SequenceFact, + has_external_context: bool, +) -> Option { + let known: bool = LIFECYCLE_EVENT_SCHEMA_VERSIONS + .iter() + .any(|version| i128::from(*version) == schema_version); + if !known { + return Some(format!( + "LifecycleEvent only represents known schema versions through \ + {CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION}; decode other versions as \ + OpaqueLifecycleEvent" + )); + } + if schema_version == UNSEQUENCED_SCHEMA_VERSION { + return (sequence != SequenceFact::Absent || has_external_context).then(|| { + "schema version 1 lifecycle events cannot contain invocation metadata".to_owned() + }); + } + (sequence != SequenceFact::Valid).then(|| INVOCATION_SEQUENCE_PROBLEM.to_owned()) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs new file mode 100644 index 0000000000..421bd96e7c --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/mod.rs @@ -0,0 +1,10 @@ +pub mod catalog; +pub mod diagnostic_severity_problem; +pub mod diagnostic_version_problem; +pub mod hook_problem; +pub mod lifecycle_version_problem; +pub mod operation_metadata_field; +pub mod producer_version_problem; +pub mod publishable_hook_name; +pub mod run_bounds_problem; +pub mod validate_known_event; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs b/crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs new file mode 100644 index 0000000000..a2ce662f67 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/operation_metadata_field.rs @@ -0,0 +1,9 @@ +//! `operation_metadata_field`: whether a key is an allow-listed operation metadata field. + +use crate::lifecycle_events::constants::OPERATION_METADATA_FIELDS; + +/// Whether a metadata key, when a string, is an allow-listed operation metadata field. +#[must_use] +pub fn operation_metadata_field(key: Option<&str>) -> bool { + key.is_some_and(|key| OPERATION_METADATA_FIELDS.contains(&key)) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs new file mode 100644 index 0000000000..de76192a3a --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/producer_version_problem.rs @@ -0,0 +1,10 @@ +//! `producer_version_problem`: the producer version rule of a lifecycle event. + +use crate::lifecycle_events::constants::DEFAULT_PRODUCER; + +/// The rule a lifecycle event's producer breaks: other producers must name their version. +#[must_use] +pub fn producer_version_problem(producer: Option<&str>, has_version: bool) -> Option<&'static str> { + (producer != Some(DEFAULT_PRODUCER) && !has_version) + .then_some("producer_version is required when producer is not 'sqlbuild'") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs b/crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs new file mode 100644 index 0000000000..77fe9c7417 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/publishable_hook_name.rs @@ -0,0 +1,9 @@ +//! `publishable_hook_name`: whether an operation's hook name is safe to publish. + +use crate::lifecycle_events::_helpers::payloads::safe_hook_name; + +/// Whether a hook name is an identifier of at most 128 characters. +#[must_use] +pub fn publishable_hook_name(name: &str) -> bool { + safe_hook_name(name) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs b/crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs new file mode 100644 index 0000000000..ed15cb02b3 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/run_bounds_problem.rs @@ -0,0 +1,18 @@ +//! `run_bounds_problem`: the selection, concurrency and worker rules of a run. + +/// The rule a run's selection, concurrency and worker counts break. +#[must_use] +pub fn run_bounds_problem( + selected_count: i128, + configured_concurrency: i128, + worker_count: i128, +) -> Option<&'static str> { + if selected_count < 0 { + return Some("selected_count must be nonnegative"); + } + if configured_concurrency < 1 { + return Some("configured_concurrency must be positive"); + } + (worker_count != configured_concurrency.min(selected_count)) + .then_some("worker_count must equal min(configured_concurrency, selected_count)") +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs b/crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs new file mode 100644 index 0000000000..311a971fd0 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/main/validate_known_event.rs @@ -0,0 +1,60 @@ +//! `validate_known_event`: correlations and safe payload fields of one catalogued event. + +use std::collections::BTreeSet; + +use crate::lifecycle_events::_helpers::envelope::{correlation_set, definition}; +use crate::lifecycle_events::_helpers::payloads::family_problem; +use crate::lifecycle_events::models::KnownEventFacts; +use crate::text::main::python_str_repr::python_str_repr; + +/// The first contract violation of a catalogued event, as its validation message. +pub fn validate_known_event(event: &KnownEventFacts) -> Result<(), String> { + let quoted: String = python_str_repr(&event.event_type); + let Some(definition) = definition(event) else { + return Err(format!( + "unknown lifecycle event_type {quoted}; decode it as an opaque envelope" + )); + }; + let missing: Vec<&str> = definition + .required_correlations + .iter() + .copied() + .filter(|name| !correlation_set(event, name)) + .collect(); + if !missing.is_empty() { + return Err(format!( + "known event {quoted} requires correlation field(s): {}", + missing.join(", ") + )); + } + let keys: BTreeSet<&str> = event.payload.iter().map(|(key, _)| key.as_str()).collect(); + let missing_payload: Vec<&str> = definition + .required_payload + .iter() + .copied() + .filter(|name| !keys.contains(name)) + .collect(); + if !missing_payload.is_empty() { + return Err(format!( + "known event {quoted} requires payload field(s): {}", + missing_payload.join(", ") + )); + } + let unexpected: Vec<&str> = keys + .iter() + .copied() + .filter(|key| !definition.allowed_payload.contains(key)) + .collect(); + if !unexpected.is_empty() { + let allowed: String = if definition.allowed_payload.is_empty() { + "none".to_owned() + } else { + definition.allowed_payload.join(", ") + }; + return Err(format!( + "payload field(s) not allowed for known event {quoted}: {}; allowed fields: {allowed}", + unexpected.join(", ") + )); + } + family_problem(event).map_or(Ok(()), Err) +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/mod.rs b/crates/sqlbuild-core/src/lifecycle_events/mod.rs new file mode 100644 index 0000000000..5853224abe --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/lifecycle_events/models.rs b/crates/sqlbuild-core/src/lifecycle_events/models.rs new file mode 100644 index 0000000000..4fc84dd22e --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/models.rs @@ -0,0 +1,57 @@ +//! Lifecycle event definitions and the JSON payload values validation reads. + +/// The validation contract of one catalogued lifecycle event type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EventDefinition { + pub event_type: &'static str, + pub required_correlations: &'static [&'static str], + pub required_payload: &'static [&'static str], + pub allowed_payload: &'static [&'static str], + pub terminal: bool, +} + +/// One frozen JSON payload value. +#[derive(Clone, Debug, PartialEq)] +pub enum PayloadValue { + Null, + Bool(bool), + /// An integer, saturated into `i128`, with its exact decimal text. + Integer { + value: i128, + text: String, + }, + Float(f64), + Text(String), + Array(Vec), + Object(Vec<(String, PayloadValue)>), +} + +/// The envelope facts of one lifecycle event that catalogue validation reads. +#[derive(Clone, Debug, PartialEq)] +pub struct KnownEventFacts { + pub schema_version: i64, + pub event_type: String, + /// Each correlation field name with whether it is set. + pub correlations: Vec<(&'static str, bool)>, + pub payload: Vec<(String, PayloadValue)>, +} + +/// The invocation sequence an event carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SequenceFact { + Absent, + /// A non-negative integer other than a bool. + Valid, + Invalid, +} + +/// The supplied hook phase, type and index of an operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HookFacts<'a> { + /// The phase when it is a string. + pub phase: Option<&'a str>, + /// The hook type when it is a string. + pub hook_type: Option<&'a str>, + /// The index is a non-negative integer other than a bool. + pub valid_index: bool, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests.rs b/crates/sqlbuild-core/src/lifecycle_events/tests.rs new file mode 100644 index 0000000000..b110b2a88e --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_envelope_rules.rs"] +mod envelope_rules; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_validate.rs"] +mod validate; diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs new file mode 100644 index 0000000000..7945ca3222 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/helpers.rs @@ -0,0 +1,33 @@ +use crate::lifecycle_events::models::{KnownEventFacts, PayloadValue}; + +/// An integer payload value. +pub(crate) fn int(value: i128) -> PayloadValue { + PayloadValue::Integer { + value, + text: value.to_string(), + } +} + +/// A text payload value. +pub(crate) fn text(value: &str) -> PayloadValue { + PayloadValue::Text(value.to_owned()) +} + +/// A version 2 event of `event_type` with every correlation set. +pub(crate) fn event(event_type: &str, payload: Vec<(&str, PayloadValue)>) -> KnownEventFacts { + KnownEventFacts { + schema_version: 2, + event_type: event_type.to_owned(), + correlations: vec![ + ("run_id", true), + ("resource_id", true), + ("resource_attempt_id", true), + ("operation_id", true), + ("statement_id", true), + ], + payload: payload + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs new file mode 100644 index 0000000000..b37fd38f69 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_envelope_rules.rs @@ -0,0 +1,160 @@ +use crate::lifecycle_events::main::hook_problem::hook_problem; +use crate::lifecycle_events::main::lifecycle_version_problem::lifecycle_version_problem; +use crate::lifecycle_events::main::run_bounds_problem::run_bounds_problem; +use crate::lifecycle_events::models::{HookFacts, SequenceFact}; +use crate::lifecycle_events::tests::test_types::{ + HookTestCase, LifecycleVersionTestCase, RunBoundsTestCase, +}; + +const SEQUENCE_PROBLEM: &str = "invocation_sequence must be a non-negative integer excluding bool"; +const VALID_HOOK: HookFacts<'static> = HookFacts { + phase: Some("pre_hooks"), + hook_type: Some("sql"), + valid_index: true, +}; + +#[test] +fn given_schema_versions_and_metadata_when_validating_then_version_rules_apply() { + let test_cases = [ + LifecycleVersionTestCase { + description: "unknown version", + schema_version: 3, + sequence: SequenceFact::Valid, + has_external_context: false, + expected_problem: Some( + "LifecycleEvent only represents known schema versions through 2; decode other versions as OpaqueLifecycleEvent", + ), + }, + LifecycleVersionTestCase { + description: "version 1 without metadata", + schema_version: 1, + sequence: SequenceFact::Absent, + has_external_context: false, + expected_problem: None, + }, + LifecycleVersionTestCase { + description: "version 1 with external context", + schema_version: 1, + sequence: SequenceFact::Absent, + has_external_context: true, + expected_problem: Some( + "schema version 1 lifecycle events cannot contain invocation metadata", + ), + }, + LifecycleVersionTestCase { + description: "version 2 requires a sequence", + schema_version: 2, + sequence: SequenceFact::Absent, + has_external_context: false, + expected_problem: Some(SEQUENCE_PROBLEM), + }, + LifecycleVersionTestCase { + description: "version 2 with sequence and context", + schema_version: 2, + sequence: SequenceFact::Valid, + has_external_context: true, + expected_problem: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + lifecycle_version_problem( + test_case.schema_version, + test_case.sequence, + test_case.has_external_context + ) + .as_deref(), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_run_bounds_when_validating_then_worker_count_matches_capped_concurrency() { + let test_cases = [ + RunBoundsTestCase { + description: "empty selection", + counts: (0, 1, 0), + expected_problem: None, + }, + RunBoundsTestCase { + description: "concurrency caps workers", + counts: (5, 2, 2), + expected_problem: None, + }, + RunBoundsTestCase { + description: "negative selection", + counts: (-1, 1, 0), + expected_problem: Some("selected_count must be nonnegative"), + }, + RunBoundsTestCase { + description: "zero concurrency", + counts: (1, 0, 0), + expected_problem: Some("configured_concurrency must be positive"), + }, + RunBoundsTestCase { + description: "too many workers", + counts: (5, 2, 5), + expected_problem: Some( + "worker_count must equal min(configured_concurrency, selected_count)", + ), + }, + ]; + + for test_case in test_cases { + let (selected, configured, workers) = test_case.counts; + assert_eq!( + run_bounds_problem(selected, configured, workers), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_hook_fields_when_validating_then_catalogue_rules_apply_in_order() { + let test_cases = [ + HookTestCase { + description: "catalogued hook", + hook: VALID_HOOK, + expected_problem: None, + }, + HookTestCase { + description: "missing phase", + hook: HookFacts { + phase: None, + ..VALID_HOOK + }, + expected_problem: Some("hook_phase must be a catalogued value"), + }, + HookTestCase { + description: "uncatalogued type", + hook: HookFacts { + hook_type: Some("shell"), + ..VALID_HOOK + }, + expected_problem: Some("hook_type must be a catalogued value"), + }, + HookTestCase { + description: "invalid index", + hook: HookFacts { + valid_index: false, + ..VALID_HOOK + }, + expected_problem: Some("hook_index must be a nonnegative integer"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + hook_problem(test_case.hook), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs new file mode 100644 index 0000000000..e843112479 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_types.rs @@ -0,0 +1,27 @@ +use crate::lifecycle_events::models::{HookFacts, KnownEventFacts, SequenceFact}; + +pub(crate) struct ValidateTestCase { + pub(crate) description: &'static str, + pub(crate) event: KnownEventFacts, + pub(crate) expected_result: Result<(), &'static str>, +} + +pub(crate) struct LifecycleVersionTestCase { + pub(crate) description: &'static str, + pub(crate) schema_version: i128, + pub(crate) sequence: SequenceFact, + pub(crate) has_external_context: bool, + pub(crate) expected_problem: Option<&'static str>, +} + +pub(crate) struct RunBoundsTestCase { + pub(crate) description: &'static str, + pub(crate) counts: (i128, i128, i128), + pub(crate) expected_problem: Option<&'static str>, +} + +pub(crate) struct HookTestCase { + pub(crate) description: &'static str, + pub(crate) hook: HookFacts<'static>, + pub(crate) expected_problem: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs new file mode 100644 index 0000000000..8d98a1c185 --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs @@ -0,0 +1,62 @@ +use crate::lifecycle_events::main::validate_known_event::validate_known_event; +use crate::lifecycle_events::tests::helpers::{event, int, text}; +use crate::lifecycle_events::tests::test_types::ValidateTestCase; + +#[test] +fn given_lifecycle_event_when_validating_then_first_contract_violation_is_reported() { + let test_cases = [ + ValidateTestCase { + description: "unknown type", + event: event("custom", Vec::new()), + expected_result: Err( + "unknown lifecycle event_type 'custom'; decode it as an opaque envelope", + ), + }, + ValidateTestCase { + description: "unexpected payload lists allowed fields", + event: event("invocation_started", vec![("x", int(1))]), + expected_result: Err( + "payload field(s) not allowed for known event 'invocation_started': x; allowed fields: command", + ), + }, + ValidateTestCase { + description: "retry numbering", + event: event( + "retry_scheduled", + vec![ + ("failed_attempt_number", int(1)), + ("next_attempt_number", int(3)), + ("delay_ms", int(5)), + ("error_type", text("E")), + ], + ), + expected_result: Err("next_attempt_number must equal failed_attempt_number + 1"), + }, + ValidateTestCase { + description: "uncatalogued operation adapter", + event: event("operation_started", vec![("adapter", text("other"))]), + expected_result: Err("adapter must be a catalogued value"), + }, + ValidateTestCase { + description: "valid run start", + event: event( + "run_started", + vec![ + ("configured_concurrency", int(4)), + ("selected_count", int(2)), + ("worker_count", int(2)), + ], + ), + expected_result: Ok(()), + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_known_event(&test_case.event), + test_case.expected_result.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/types.rs b/crates/sqlbuild-core/src/lifecycle_events/types.rs new file mode 100644 index 0000000000..2af438e15f --- /dev/null +++ b/crates/sqlbuild-core/src/lifecycle_events/types.rs @@ -0,0 +1,14 @@ +//! The shape of one catalogue entry. + +/// One catalogued event: type, required correlations, required and allowed payload, terminal. +pub(crate) type CatalogEntry = ( + &'static str, + &'static [&'static str], + &'static [&'static str], + &'static [&'static str], + bool, +); + +/// One event-family payload rule: the first violation in a payload, if any. +pub(crate) type PayloadCheck = + fn(&[(String, crate::lifecycle_events::models::PayloadValue)]) -> Option; diff --git a/crates/sqlbuild-core/src/loader_connection/constants.rs b/crates/sqlbuild-core/src/loader_connection/constants.rs new file mode 100644 index 0000000000..115683dca6 --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/constants.rs @@ -0,0 +1,8 @@ +//! Loader destination connection defaults. + +/// The host destinations connect to without one. +pub const DEFAULT_HOST: &str = "localhost"; +/// Postgres's default port. +pub const POSTGRES_DEFAULT_PORT: &str = "5432"; +/// The MotherDuck database prefix. +pub const MOTHERDUCK_PREFIX: &str = "md:"; diff --git a/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs b/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs new file mode 100644 index 0000000000..141c17c681 --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/main/connection_setting.rs @@ -0,0 +1,14 @@ +//! `connection_setting`: one stripped string connection setting. + +use crate::loader_connection::models::LoaderConnection; +use crate::text::main::python_strip::python_strip; + +/// The stripped string setting `key`, or `None` when unset, not a string or blank. +#[must_use] +pub fn connection_setting<'a>(connection: &'a LoaderConnection, key: &str) -> Option<&'a str> { + connection + .strings + .get(key) + .map(|value| python_strip(value)) + .filter(|value| !value.is_empty()) +} diff --git a/crates/sqlbuild-core/src/loader_connection/main/mod.rs b/crates/sqlbuild-core/src/loader_connection/main/mod.rs new file mode 100644 index 0000000000..4ea9c16cda --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/main/mod.rs @@ -0,0 +1,2 @@ +pub mod connection_setting; +pub mod required_connection_setting; diff --git a/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs b/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs new file mode 100644 index 0000000000..cb6ac0aecb --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/main/required_connection_setting.rs @@ -0,0 +1,18 @@ +//! `required_connection_setting`: one connection setting a loader destination requires. + +use crate::loader_connection::main::connection_setting::connection_setting; +use crate::loader_connection::models::LoaderConnection; + +/// The stripped string setting `key`, which the adapter's `loader` destination requires. +pub fn required_connection_setting<'a>( + connection: &'a LoaderConnection, + key: &str, + adapter_name: &str, + loader: &str, +) -> Result<&'a str, String> { + connection_setting(connection, key).ok_or_else(|| { + format!( + "Adapter '{adapter_name}' requires connection.{key} for {loader} integration loaders" + ) + }) +} diff --git a/crates/sqlbuild-core/src/loader_connection/mod.rs b/crates/sqlbuild-core/src/loader_connection/mod.rs new file mode 100644 index 0000000000..28a7bc49f9 --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/mod.rs @@ -0,0 +1,5 @@ +//! Warehouse connection settings that integration loader destinations read. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-core/src/loader_connection/models.rs b/crates/sqlbuild-core/src/loader_connection/models.rs new file mode 100644 index 0000000000..ed4f4d15bd --- /dev/null +++ b/crates/sqlbuild-core/src/loader_connection/models.rs @@ -0,0 +1,12 @@ +//! The adapter connection settings a loader destination reads. + +use std::collections::BTreeMap; + +/// The adapter connection settings an ingestr or dlt destination reads. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoaderConnection { + /// Connection settings whose values are Python strings, unstripped. + pub strings: BTreeMap, + /// `str(connection["port"])` when the connection sets a port. + pub port: Option, +} diff --git a/crates/sqlbuild-core/src/migration_values/constants.rs b/crates/sqlbuild-core/src/migration_values/constants.rs new file mode 100644 index 0000000000..e1bcf6c806 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/constants.rs @@ -0,0 +1,30 @@ +//! Migration decision vocabularies. + +/// Model decisions that clone the origin over the destination. +pub(crate) const MOVING_DECISIONS: [&str; 4] = + ["migrate", "redo", "superseded_replace", "forced_replace"]; +/// Model decisions that stop a build regardless of compatibility. +pub(crate) const BLOCKING_DECISIONS: [&str; 2] = ["conflict", "origin_missing"]; +pub(crate) const CONFLICT_DECISION: &str = "conflict"; +/// Column decisions that append a column migration event. +pub(crate) const RECORDING_COLUMN_DECISIONS: [&str; 2] = ["rename", "record"]; +pub(crate) const RENAME_COLUMN_DECISION: &str = "rename"; +/// Column decisions that stop a build before any execution. +pub(crate) const BLOCKING_COLUMN_DECISIONS: [&str; 4] = [ + "conflict", + "source_missing", + "still_produced", + "unsupported", +]; +/// Plan labels of the column decisions. +pub(crate) const COLUMN_DECISION_LABELS: [(&str, &str); 7] = [ + ("rename", "rename in place"), + ("record", "already renamed, record"), + ("done", "done"), + ("conflict", "conflict"), + ("source_missing", "source missing"), + ("still_produced", "source still produced"), + ("unsupported", "unsupported"), +]; +/// Snowflake table types by whether the table is transient. +pub(crate) const STORAGE_LABELS: [&str; 2] = ["permanent", "transient"]; diff --git a/crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs b/crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs new file mode 100644 index 0000000000..358d9fe467 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/column_decision_traits.rs @@ -0,0 +1,21 @@ +//! What one column migration decision does. + +use crate::migration_values::constants::{ + BLOCKING_COLUMN_DECISIONS, COLUMN_DECISION_LABELS, RECORDING_COLUMN_DECISIONS, + RENAME_COLUMN_DECISION, +}; +use crate::migration_values::models::ColumnDecisionTraits; + +/// The behaviour of the column migration decision whose value is `decision`. +#[must_use] +pub fn column_decision_traits(decision: &str) -> ColumnDecisionTraits { + ColumnDecisionTraits { + renames: decision == RENAME_COLUMN_DECISION, + records_event: RECORDING_COLUMN_DECISIONS.contains(&decision), + blocks_build: BLOCKING_COLUMN_DECISIONS.contains(&decision), + label: COLUMN_DECISION_LABELS + .iter() + .find(|(value, _)| *value == decision) + .map(|(_, label)| *label), + } +} diff --git a/crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs b/crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs new file mode 100644 index 0000000000..75976c0ff1 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/migration_decision_traits.rs @@ -0,0 +1,15 @@ +//! What one model migration decision does. + +use crate::migration_values::constants::{BLOCKING_DECISIONS, CONFLICT_DECISION, MOVING_DECISIONS}; +use crate::migration_values::models::DecisionTraits; + +/// The behaviour of the model migration decision whose value is `decision`. +#[must_use] +pub fn migration_decision_traits(decision: &str) -> DecisionTraits { + let moves_data: bool = MOVING_DECISIONS.contains(&decision); + DecisionTraits { + moves_data, + blocks_build: BLOCKING_DECISIONS.contains(&decision), + checks_compatibility: moves_data || decision == CONFLICT_DECISION, + } +} diff --git a/crates/sqlbuild-core/src/migration_values/main/mod.rs b/crates/sqlbuild-core/src/migration_values/main/mod.rs new file mode 100644 index 0000000000..c8c8b9703d --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/mod.rs @@ -0,0 +1,5 @@ +pub mod column_decision_traits; +pub mod migration_decision_traits; +pub mod model_migration_blocks_build; +pub mod plan_label; +pub mod storage_transition; diff --git a/crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs b/crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs new file mode 100644 index 0000000000..3e11b5c365 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/model_migration_blocks_build.rs @@ -0,0 +1,12 @@ +//! Whether one planned model migration stops a build before any execution. + +use crate::migration_values::models::ModelMigrationFacts; + +/// A missing origin blocks when tracked or denied; otherwise the decision or an incompatible move blocks. +#[must_use] +pub fn model_migration_blocks_build(facts: &ModelMigrationFacts) -> bool { + if facts.origin_missing { + return facts.origin_tracked || facts.deny_missing_origin; + } + facts.decision.blocks_build || (facts.decision.moves_data && facts.incompatible) +} diff --git a/crates/sqlbuild-core/src/migration_values/main/plan_label.rs b/crates/sqlbuild-core/src/migration_values/main/plan_label.rs new file mode 100644 index 0000000000..83701478df --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/plan_label.rs @@ -0,0 +1,7 @@ +//! The plan label of a model migration decision or promotion. + +/// `value` with underscores shown as spaces. +#[must_use] +pub fn plan_label(value: &str) -> String { + value.replace('_', " ") +} diff --git a/crates/sqlbuild-core/src/migration_values/main/storage_transition.rs b/crates/sqlbuild-core/src/migration_values/main/storage_transition.rs new file mode 100644 index 0000000000..dfd8cf5aad --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/main/storage_transition.rs @@ -0,0 +1,17 @@ +//! The Snowflake table-type transition one model migration makes. + +use crate::migration_values::constants::STORAGE_LABELS; + +/// `origin -> stage` table types, or `None` when the stage's type is unknown. +#[must_use] +pub fn storage_transition( + origin_is_transient: bool, + stage_is_transient: Option, +) -> Option { + let stage: bool = stage_is_transient?; + Some(format!( + "{} -> {}", + STORAGE_LABELS[usize::from(origin_is_transient)], + STORAGE_LABELS[usize::from(stage)] + )) +} diff --git a/crates/sqlbuild-core/src/migration_values/mod.rs b/crates/sqlbuild-core/src/migration_values/mod.rs new file mode 100644 index 0000000000..19c466d2dd --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/mod.rs @@ -0,0 +1,8 @@ +//! Model and column migration decisions: what each decision does and when it blocks a build. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/migration_values/models.rs b/crates/sqlbuild-core/src/migration_values/models.rs new file mode 100644 index 0000000000..409ef75c16 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/models.rs @@ -0,0 +1,35 @@ +//! What one migration decision does. + +/// The behaviour of one model migration decision. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DecisionTraits { + /// The decision clones the origin over the destination. + pub moves_data: bool, + /// The decision stops a build regardless of compatibility. + pub blocks_build: bool, + /// The decision evaluated the origin against the destination. + pub checks_compatibility: bool, +} + +/// The behaviour of one column migration decision. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ColumnDecisionTraits { + pub renames: bool, + pub records_event: bool, + pub blocks_build: bool, + /// The plan label, or `None` for an unknown decision. + pub label: Option<&'static str>, +} + +/// The facts of one planned model migration that decide whether it blocks a build. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ModelMigrationFacts { + pub origin_missing: bool, + /// The missing origin has build history in this target. + pub origin_tracked: bool, + /// `missing_migration_origin` is `deny`. + pub deny_missing_origin: bool, + pub decision: DecisionTraits, + /// The origin is incompatible with the destination. + pub incompatible: bool, +} diff --git a/crates/sqlbuild-core/src/migration_values/tests.rs b/crates/sqlbuild-core/src/migration_values/tests.rs new file mode 100644 index 0000000000..04bacf2250 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_migration_values.rs"] +mod migration_values; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs b/crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs new file mode 100644 index 0000000000..3ee22cd32a --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/tests/test_migration_values.rs @@ -0,0 +1,146 @@ +use crate::migration_values::main::column_decision_traits::column_decision_traits; +use crate::migration_values::main::migration_decision_traits::migration_decision_traits; +use crate::migration_values::main::model_migration_blocks_build::model_migration_blocks_build; +use crate::migration_values::main::storage_transition::storage_transition; +use crate::migration_values::models::{ColumnDecisionTraits, DecisionTraits, ModelMigrationFacts}; +use crate::migration_values::tests::test_types::{ + BlocksBuildTestCase, ColumnDecisionTestCase, DecisionTestCase, TransitionTestCase, +}; + +const MOVING: DecisionTraits = DecisionTraits { + moves_data: true, + blocks_build: false, + checks_compatibility: true, +}; + +#[test] +fn given_model_decision_when_classifying_then_matches_python_properties() { + let test_cases = [ + DecisionTestCase { + description: "migrate moves data and checks compatibility", + decision: "migrate", + expected_traits: MOVING, + }, + DecisionTestCase { + description: "conflict blocks and checks compatibility", + decision: "conflict", + expected_traits: DecisionTraits { + moves_data: false, + blocks_build: true, + checks_compatibility: true, + }, + }, + DecisionTestCase { + description: "done does nothing", + decision: "done", + expected_traits: DecisionTraits { + moves_data: false, + blocks_build: false, + checks_compatibility: false, + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + migration_decision_traits(test_case.decision), + test_case.expected_traits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_column_decision_when_classifying_then_matches_python_properties() { + let test_cases = [ + ColumnDecisionTestCase { + description: "rename renames and records", + decision: "rename", + expected_traits: ColumnDecisionTraits { + renames: true, + records_event: true, + blocks_build: false, + label: Some("rename in place"), + }, + }, + ColumnDecisionTestCase { + description: "a produced source blocks", + decision: "still_produced", + expected_traits: ColumnDecisionTraits { + renames: false, + records_event: false, + blocks_build: true, + label: Some("source still produced"), + }, + }, + ]; + for test_case in test_cases { + assert_eq!( + column_decision_traits(test_case.decision), + test_case.expected_traits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_migration_facts_when_checking_then_blocks_as_python_did() { + let facts = + |origin_missing, origin_tracked, deny_missing_origin, incompatible| ModelMigrationFacts { + origin_missing, + origin_tracked, + deny_missing_origin, + decision: MOVING, + incompatible, + }; + let test_cases = [ + BlocksBuildTestCase { + description: "a tracked missing origin blocks", + facts: facts(true, true, false, false), + expected_blocks: true, + }, + BlocksBuildTestCase { + description: "an untracked allowed missing origin passes", + facts: facts(true, false, false, true), + expected_blocks: false, + }, + BlocksBuildTestCase { + description: "an incompatible move blocks", + facts: facts(false, false, false, true), + expected_blocks: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + model_migration_blocks_build(&test_case.facts), + test_case.expected_blocks, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_table_types_when_labelling_then_shows_the_transition() { + let test_cases = [ + TransitionTestCase { + description: "a permanent origin staged as transient", + types: (false, Some(true)), + expected_transition: Some("permanent -> transient"), + }, + TransitionTestCase { + description: "an unknown stage type has no label", + types: (true, None), + expected_transition: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + storage_transition(test_case.types.0, test_case.types.1).as_deref(), + test_case.expected_transition, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/migration_values/tests/test_types.rs b/crates/sqlbuild-core/src/migration_values/tests/test_types.rs new file mode 100644 index 0000000000..871d0a9408 --- /dev/null +++ b/crates/sqlbuild-core/src/migration_values/tests/test_types.rs @@ -0,0 +1,25 @@ +use crate::migration_values::models::{ColumnDecisionTraits, DecisionTraits, ModelMigrationFacts}; + +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) decision: &'static str, + pub(super) expected_traits: DecisionTraits, +} + +pub(super) struct ColumnDecisionTestCase { + pub(super) description: &'static str, + pub(super) decision: &'static str, + pub(super) expected_traits: ColumnDecisionTraits, +} + +pub(super) struct BlocksBuildTestCase { + pub(super) description: &'static str, + pub(super) facts: ModelMigrationFacts, + pub(super) expected_blocks: bool, +} + +pub(super) struct TransitionTestCase { + pub(super) description: &'static str, + pub(super) types: (bool, Option), + pub(super) expected_transition: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/shell_text/main/mod.rs b/crates/sqlbuild-core/src/shell_text/main/mod.rs new file mode 100644 index 0000000000..3079457ba4 --- /dev/null +++ b/crates/sqlbuild-core/src/shell_text/main/mod.rs @@ -0,0 +1 @@ +pub mod shell_quote; diff --git a/crates/sqlbuild-core/src/shell_text/main/shell_quote.rs b/crates/sqlbuild-core/src/shell_text/main/shell_quote.rs new file mode 100644 index 0000000000..3e7e57a811 --- /dev/null +++ b/crates/sqlbuild-core/src/shell_text/main/shell_quote.rs @@ -0,0 +1,16 @@ +//! Python's `shlex.quote`. + +/// Python's `shlex.quote(text)`: safe ASCII text unchanged, anything else single-quoted. +#[must_use] +pub fn shell_quote(text: &str) -> String { + if text.is_empty() { + return "''".to_owned(); + } + let safe: bool = text + .chars() + .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); + if safe { + return text.to_owned(); + } + format!("'{}'", text.replace('\'', "'\"'\"'")) +} diff --git a/crates/sqlbuild-core/src/shell_text/mod.rs b/crates/sqlbuild-core/src/shell_text/mod.rs new file mode 100644 index 0000000000..66ee6a2194 --- /dev/null +++ b/crates/sqlbuild-core/src/shell_text/mod.rs @@ -0,0 +1,3 @@ +//! Shell-quoted command text. + +pub mod main; diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index 1c58b4febe..f1bb0e5c90 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -1,4 +1,4 @@ -//! Project layout names shared by native discovery, mirroring the Python discovery constants. +//! Project layout names shared by native discovery. /// Root of SQL model files. pub const MODELS_DIRECTORY: &str = "models"; @@ -41,8 +41,6 @@ pub const GROUPED_NAMED_DECLARATION_DIRECTORIES: [&str; 6] = [ pub const GLOBAL_NAMED_DECLARATION_DIRECTORIES: [&str; 3] = ["audits", "schemas", "hooks"]; /// Suffix of macro files. pub const PYTHON_FILE_SUFFIX: &str = ".py"; -/// The stem of package initialisers, which are not macro files. -pub const PYTHON_INIT_MODULE_STEM: &str = "__init__"; /// The project-wide audit role directory. pub const AUDIT_ROLE_DIRECTORY: &str = "audits"; /// The folder-local audit role directory inside a declaration group. diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs index c2b151ceb5..c1c8242eb2 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/module_paths.rs @@ -1,6 +1,7 @@ //! Which `*.py` paths below a root Python module discovery keeps. -use crate::declaration_files::constants::{INIT_MODULE_STEM, PRIVATE_PREFIX}; +use crate::declaration_files::constants::PRIVATE_PREFIX; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM as INIT_MODULE_STEM; /// Whether `relative_path` below `root` is a listed module: never `__init__`, and public if asked. pub(crate) fn is_listed_module(relative_path: &str, root: &str, public_only: bool) -> bool { diff --git a/crates/sqlbuild-discovery/src/declaration_files/constants.rs b/crates/sqlbuild-discovery/src/declaration_files/constants.rs index 35ae950b15..769949697b 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/constants.rs @@ -2,7 +2,5 @@ /// The directory whose `*.py` files define warehouse Python functions. pub const PYTHON_FUNCTIONS_ROOT: &str = "functions/python"; -/// The stem of a package `__init__` module, which module discovery skips. -pub const INIT_MODULE_STEM: &str = "__init__"; /// The prefix of a path part that keeps a module out of public module discovery. pub const PRIVATE_PREFIX: char = '_'; diff --git a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs index 48431079ca..191540fa06 100644 --- a/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs +++ b/crates/sqlbuild-discovery/src/declarations/_helpers/declaration_tree.rs @@ -2,8 +2,8 @@ use crate::constants::{ CANONICAL_AUTHORED_ROOTS, DECLARATION_GROUP_DIRECTORY, GLOBAL_DECLARATION_DIRECTORIES, - LOCAL_DECLARATION_DIRECTORIES, PYTHON_FILE_SUFFIX, PYTHON_INIT_MODULE_STEM, - SCOPED_DECLARATION_DIRECTORIES, SQL_FILE_SUFFIX, + LOCAL_DECLARATION_DIRECTORIES, PYTHON_FILE_SUFFIX, SCOPED_DECLARATION_DIRECTORIES, + SQL_FILE_SUFFIX, }; use crate::declarations::_helpers::declaration_groups::validate_declaration_groups; use crate::declarations::_helpers::paths::{directory_facts, name, parent, stem}; @@ -14,6 +14,7 @@ use crate::tree::main::directories::directories; use crate::tree::main::path_order::compare_posix_text; use crate::tree::main::rglob::rglob; use crate::tree::models::ProjectTree; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; /// Where a declaration root sits and who owns it. struct DeclarationRoot<'a> { diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs index 47ebf99d0a..91ea0a3256 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/loaders.rs @@ -6,10 +6,10 @@ use crate::project_validation::_helpers::conflicts::{conflict, node_path, unique use crate::project_validation::_helpers::python_nodes::{ validate_check_dependencies, validate_loader_dependencies, validate_python_node_dependencies, }; -use crate::project_validation::constants::DLT_LOADER_KIND; use crate::project_validation::models::{ NamedPath, NodeFact, NodeKind, ProjectFacts, SourceFileFact, ValidationFailure, }; +use crate::yaml_entries::main::integration_loader_name::integration_loader_name; /// Validate loader and node names, node dependencies, then the loaders sources name. pub(crate) fn validate_loaders(facts: &ProjectFacts) -> Result<(), ValidationFailure> { @@ -64,15 +64,6 @@ fn source_loader_name_collisions( Ok(()) } -/// Python's `integration_loader_name`. -fn integration_loader_name(kind: &str, source_name: &str) -> String { - if kind == DLT_LOADER_KIND { - source_name.to_owned() - } else { - format!("{kind}__{source_name}") - } -} - fn source_loader_references( sources: &[SourceFileFact], loaders: &[&NodeFact], diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index 8ee3042fb3..6122e20503 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -1,7 +1,5 @@ //! Names and limits cross-file discovery validation checks against. -/// The loader kind whose generated loader takes the source's own name. -pub(crate) const DLT_LOADER_KIND: &str = "dlt"; /// The prefix discovery strips from model paths before matching path defaults. pub(crate) const MODELS_PREFIX: &str = "models/"; /// `str(Path(...).parent)` of a path with no folder. diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs index a2730f2ab2..1e100c626e 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/thresholds.rs @@ -100,7 +100,7 @@ fn finite_limit( } /// Check `MeasurementThresholds.__post_init__` for the parsed bounds. -fn check_pair( +pub(crate) fn check_pair( warn: Option, error: Option, ) -> Result<(), &'static str> { diff --git a/crates/sqlbuild-model-config/src/header_metadata/constants.rs b/crates/sqlbuild-model-config/src/header_metadata/constants.rs index 387bd87f93..949cb97071 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/constants.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/constants.rs @@ -20,7 +20,9 @@ pub const AUDIT_OPTION_KEYS: [&str; 8] = [ pub const THRESHOLD_WARN_KEY: &str = "warn"; pub const THRESHOLD_ERROR_KEY: &str = "error"; /// Threshold operators, as `ThresholdOperator` values. -pub const THRESHOLD_OPERATORS: [&str; 3] = ["below", "above", "outside"]; +pub const THRESHOLD_OPERATORS: [&str; 3] = [BELOW_OPERATOR, ABOVE_OPERATOR, "outside"]; +pub const BELOW_OPERATOR: &str = "below"; +pub const ABOVE_OPERATOR: &str = "above"; /// Audit severities a header may name. pub const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; /// The audit that contradicts `nullable = true`. diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs b/crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs new file mode 100644 index 0000000000..84faae9b98 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/check_measurement_contract.rs @@ -0,0 +1,21 @@ +//! `MeasurementContract.__post_init__`: the measurement query's named columns and unit. + +use sqlbuild_core::text::main::python_strip::python_strip; + +/// Whether every named contract field is non-blank, or the measurement audit error text. +pub fn check_measurement_contract( + value_column: &str, + sample_count_column: Option<&str>, + sample_unit: Option<&str>, +) -> Result<(), &'static str> { + if python_strip(value_column).is_empty() { + return Err("measurement contract value_column must be non-empty"); + } + if sample_count_column.is_some_and(|column| python_strip(column).is_empty()) { + return Err("measurement contract sample_count_column must be non-empty"); + } + if sample_unit.is_some_and(|unit| python_strip(unit).is_empty()) { + return Err("measurement contract sample_unit must be non-empty"); + } + Ok(()) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs new file mode 100644 index 0000000000..55af2103e8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_bound.rs @@ -0,0 +1,52 @@ +//! `MeasurementThresholdBound.__post_init__`: one directional bound and its limits. + +use crate::header_metadata::constants::{ABOVE_OPERATOR, BELOW_OPERATOR}; +use crate::header_metadata::models::{ThresholdBound, ThresholdInput}; + +/// The bound `operator` declares over `(limit, lower, upper)`, or the measurement audit error text. +pub fn check_threshold_bound( + operator: &str, + (limit, lower, upper): (ThresholdInput, ThresholdInput, ThresholdInput), +) -> Result { + if operator == BELOW_OPERATOR || operator == ABOVE_OPERATOR { + let limit: f64 = match (limit, lower, upper) { + (ThresholdInput::Number(limit), ThresholdInput::Absent, ThresholdInput::Absent) => { + limit + } + (ThresholdInput::Boolean, ThresholdInput::Absent, ThresholdInput::Absent) => { + return Err("measurement threshold limit must not be boolean".to_owned()); + } + _ => return Err(format!("{operator} threshold requires limit only")), + }; + if !limit.is_finite() { + return Err("measurement threshold limit must be finite".to_owned()); + } + return Ok(if operator == BELOW_OPERATOR { + ThresholdBound::Below(limit) + } else { + ThresholdBound::Above(limit) + }); + } + if limit != ThresholdInput::Absent + || lower == ThresholdInput::Absent + || upper == ThresholdInput::Absent + { + return Err("outside threshold requires lower and upper only".to_owned()); + } + let ThresholdInput::Number(lower) = lower else { + return Err("measurement threshold lower must not be boolean".to_owned()); + }; + let ThresholdInput::Number(upper) = upper else { + return Err("measurement threshold upper must not be boolean".to_owned()); + }; + if !lower.is_finite() { + return Err("measurement threshold lower must be finite".to_owned()); + } + if !upper.is_finite() { + return Err("measurement threshold upper must be finite".to_owned()); + } + if lower > upper { + return Err("outside threshold lower must be less than or equal to upper".to_owned()); + } + Ok(ThresholdBound::Outside(lower, upper)) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs new file mode 100644 index 0000000000..002a96426d --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/check_threshold_pair.rs @@ -0,0 +1,12 @@ +//! `MeasurementThresholds.__post_init__`: the warn and error bounds of one audit. + +use crate::header_metadata::_helpers::thresholds::check_pair; +use crate::header_metadata::models::ThresholdBound; + +/// Whether `warn` and `error` form a valid threshold policy, or the measurement audit error text. +pub fn check_threshold_pair( + warn: Option, + error: Option, +) -> Result<(), &'static str> { + check_pair(warn, error) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs index c7b2438638..d195d8f0d8 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs @@ -1,4 +1,7 @@ pub mod audit_factory_references; +pub mod check_measurement_contract; +pub mod check_threshold_bound; +pub mod check_threshold_pair; pub mod merged_audit_identities; pub mod parse_audit_list; pub mod parse_dynamic_columns; diff --git a/crates/sqlbuild-model-config/src/header_metadata/models.rs b/crates/sqlbuild-model-config/src/header_metadata/models.rs index 165b823b9b..9a1ea2e896 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/models.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/models.rs @@ -28,6 +28,14 @@ pub enum ThresholdBound { Outside(f64, f64), } +/// One limit passed to a measurement threshold bound, as Python sees its type. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ThresholdInput { + Absent, + Boolean, + Number(f64), +} + /// An audit's warn and error thresholds; at least one is set. #[derive(Clone, Copy, Debug, PartialEq)] pub struct ParsedThresholds { diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests.rs b/crates/sqlbuild-model-config/src/header_metadata/tests.rs index fed845e0be..bedc46c47f 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests.rs @@ -2,6 +2,8 @@ mod header_extensions; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_measurement_checks.rs"] +mod measurement_checks; #[path = "tests/test_parse_header_metadata.rs"] mod parse_header_metadata; #[path = "tests/test_parse_schema_columns.rs"] diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs new file mode 100644 index 0000000000..acdad37ca4 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_measurement_checks.rs @@ -0,0 +1,130 @@ +use crate::header_metadata::main::check_measurement_contract::check_measurement_contract; +use crate::header_metadata::main::check_threshold_bound::check_threshold_bound; +use crate::header_metadata::models::{ThresholdBound, ThresholdInput}; +use crate::header_metadata::tests::test_types::{ + MeasurementContractTestCase, ThresholdBoundTestCase, +}; + +#[test] +fn given_threshold_limits_when_checking_bound_then_matches_python_validation() { + let test_cases = [ + ThresholdBoundTestCase { + description: "a below limit becomes the bound", + operator: "below", + inputs: ( + ThresholdInput::Number(2.0), + ThresholdInput::Absent, + ThresholdInput::Absent, + ), + expected_bound: Ok(ThresholdBound::Below(2.0)), + }, + ThresholdBoundTestCase { + description: "a boolean limit is rejected after the shape check", + operator: "above", + inputs: ( + ThresholdInput::Boolean, + ThresholdInput::Absent, + ThresholdInput::Absent, + ), + expected_bound: Err("measurement threshold limit must not be boolean"), + }, + ThresholdBoundTestCase { + description: "a directional bound with a lower limit is rejected", + operator: "above", + inputs: ( + ThresholdInput::Number(1.0), + ThresholdInput::Number(0.0), + ThresholdInput::Absent, + ), + expected_bound: Err("above threshold requires limit only"), + }, + ThresholdBoundTestCase { + description: "an infinite limit is rejected", + operator: "below", + inputs: ( + ThresholdInput::Number(f64::INFINITY), + ThresholdInput::Absent, + ThresholdInput::Absent, + ), + expected_bound: Err("measurement threshold limit must be finite"), + }, + ThresholdBoundTestCase { + description: "an outside bound without an upper limit fails the shape check first", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Boolean, + ThresholdInput::Absent, + ), + expected_bound: Err("outside threshold requires lower and upper only"), + }, + ThresholdBoundTestCase { + description: "a boolean upper limit is rejected", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Number(1.0), + ThresholdInput::Boolean, + ), + expected_bound: Err("measurement threshold upper must not be boolean"), + }, + ThresholdBoundTestCase { + description: "a reversed outside range is rejected", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Number(3.0), + ThresholdInput::Number(1.0), + ), + expected_bound: Err("outside threshold lower must be less than or equal to upper"), + }, + ThresholdBoundTestCase { + description: "an outside range becomes the bound", + operator: "outside", + inputs: ( + ThresholdInput::Absent, + ThresholdInput::Number(1.0), + ThresholdInput::Number(3.0), + ), + expected_bound: Ok(ThresholdBound::Outside(1.0, 3.0)), + }, + ]; + for test_case in test_cases { + assert_eq!( + check_threshold_bound(test_case.operator, test_case.inputs), + test_case.expected_bound.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_contract_fields_when_checking_then_blank_names_are_rejected() { + let test_cases = [ + MeasurementContractTestCase { + description: "named columns are accepted", + fields: ("value", Some("samples"), None), + expected_result: Ok(()), + }, + MeasurementContractTestCase { + description: "a blank value column is rejected", + fields: (" ", None, None), + expected_result: Err("measurement contract value_column must be non-empty"), + }, + MeasurementContractTestCase { + description: "a blank sample unit is rejected", + fields: ("value", None, Some("")), + expected_result: Err("measurement contract sample_unit must be non-empty"), + }, + ]; + for test_case in test_cases { + let (value_column, sample_count_column, sample_unit) = test_case.fields; + assert_eq!( + check_measurement_contract(value_column, sample_count_column, sample_unit), + test_case.expected_result, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs index 1d6136ac43..0a09443e13 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::header_metadata::models::{ThresholdBound, ThresholdInput}; use crate::tests::test_types::Value; /// One header's `columns` and `audits` values and the parse the native parser must produce. @@ -31,3 +32,18 @@ pub(super) struct SchemaColumnsTestCase { /// The comma-separated column names, or the first error message. pub(super) expected_summary: &'static str, } + +/// One threshold bound's operator and limits and the bound or error it produces. +pub(super) struct ThresholdBoundTestCase { + pub(super) description: &'static str, + pub(super) operator: &'static str, + pub(super) inputs: (ThresholdInput, ThresholdInput, ThresholdInput), + pub(super) expected_bound: Result, +} + +/// One measurement contract's fields and the error it produces, if any. +pub(super) struct MeasurementContractTestCase { + pub(super) description: &'static str, + pub(super) fields: (&'static str, Option<&'static str>, Option<&'static str>), + pub(super) expected_result: Result<(), &'static str>, +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs index 53c23c6e17..28c2055977 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/incremental.rs @@ -16,11 +16,12 @@ use crate::model_validation::constants::{ ENFORCED_CONTRACT, FUTURE_CURSOR_ACTIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_STRATEGIES, MERGE_STRATEGY, ON_SCHEMA_CHANGE_EXAMPLE, ON_SCHEMA_CHANGE_POLICIES, REPLAY_ON_CHANGE_ACTIONS, REPLAY_ON_CHANGE_EXAMPLE, REPLAY_ON_CHANGE_VALID_VALUES, TABLE_MATERIALIZATION, - TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, ZERO_DAY_DURATION, + TIMESTAMP_CURSOR, VIEW_MATERIALIZATION, }; use crate::model_validation::models::{ModelValidationFacts, ValidationStop}; use crate::model_validation::types::Check; use crate::types::{AuthoredNode, NodeKind}; +use sqlbuild_core::constants::ZERO_DAY_DURATION; const ON_SCHEMA_CHANGE_KEY: &str = "on_schema_change"; const REPLAY_ON_CHANGE_KEY: &str = "replay_on_change"; diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 0249e5c8c0..341d1f0cf4 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -113,8 +113,6 @@ sqlbuild_project.toml:\n [settings]\n microbatch_concurren pub const SETTING_SNIPPET_INDENT: &str = " "; /// The duration-disabled cursor policy value. pub const CURSOR_POLICY_DISABLED: &str = "disabled"; -/// The zero-day cursor duration. -pub const ZERO_DAY_DURATION: &str = "0d"; /// The `batch_size` token that follows the cursor grain. pub const EFFECTIVE_BATCH_SIZE: &str = "effective"; /// The SQL wildcard column. diff --git a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs index 44b20d041c..e60ee1765b 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/retention_override.rs @@ -4,12 +4,11 @@ use crate::errors::ConfigError; use crate::model_validation::_helpers::config::model_header_help; use crate::model_validation::_helpers::config::text_if_string; use crate::model_validation::_helpers::durations::parse_duration; -use crate::model_validation::constants::{ - DISABLED_RETENTION, INHERIT_POLICY, RETENTION_KEY, ZERO_DAY_DURATION, -}; +use crate::model_validation::constants::{DISABLED_RETENTION, INHERIT_POLICY, RETENTION_KEY}; use crate::model_validation::errors::DurationNumberError; use crate::model_validation::models::RetentionOverride; use crate::types::{AuthoredNode, NodeKind}; +use sqlbuild_core::constants::ZERO_DAY_DURATION; /// Return the header retention override, or the error it raises. pub fn retention_override( diff --git a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs index 2b87ef6f22..c929921954 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/_helpers/paths.rs @@ -7,8 +7,9 @@ use ruff_python_ast::Stmt; use crate::cpython_tree::models::CpythonNode; use crate::rule_evidence::constants::{ - INIT_FILE_NAME, INIT_MODULE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, + INIT_FILE_NAME, PYTHON_SUFFIX, RULES_DIRECTORY_NAME, STAR_IMPORT_NAME, }; +use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM as INIT_MODULE_NAME; /// `Path.resolve()`: symlinks resolved through the existing prefix, the rest kept lexically. pub(crate) fn resolve(path: &Path) -> PathBuf { diff --git a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs index 831fdd337a..3f0c19ea5a 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/constants.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/constants.rs @@ -6,7 +6,6 @@ pub(crate) const RULE_CASE: &str = "RuleCase"; pub(crate) const RULE_KEYWORD: &str = "rule"; pub(crate) const TEST_CASE_KEYWORD: &str = "test_case"; pub(crate) const CONDITION_KEYWORD: &str = "condition"; -pub(crate) const INIT_MODULE_NAME: &str = "__init__"; pub(crate) const INIT_FILE_NAME: &str = "__init__.py"; pub(crate) const RULES_DIRECTORY_NAME: &str = "rules"; pub(crate) const TESTS_DIRECTORY_NAME: &str = "tests"; @@ -23,3 +22,37 @@ pub(crate) const TEST_FILE_PREFIX: &str = "test_"; pub(crate) const TEST_FILE_SUFFIX: &str = "_test.py"; pub(crate) const TEST_CLASS_PREFIX: &str = "Test"; pub(crate) const FLOAT_ZERO: f64 = 0.0_f64; +/// The modules, and their submodules, custom-rule code may import. +pub(crate) const CUSTOM_RULE_IMPORT_ROOTS: [&str; 29] = [ + "__future__", + "abc", + "bisect", + "collections", + "copy", + "dataclasses", + "decimal", + "difflib", + "enum", + "fnmatch", + "fractions", + "functools", + "graphlib", + "hashlib", + "heapq", + "itertools", + "json", + "math", + "operator", + "pathlib", + "re", + "rules", + "sqlbuild.rules", + "statistics", + "string", + "textwrap", + "types", + "typing", + "unicodedata", +]; +/// The separator between a module and its submodules. +pub(crate) const MODULE_SEPARATOR: char = '.'; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs new file mode 100644 index 0000000000..4c6c8cb625 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/allowed_custom_rule_import.rs @@ -0,0 +1,13 @@ +//! The deterministic import allowlist of custom-rule code. + +use crate::rule_evidence::constants::{CUSTOM_RULE_IMPORT_ROOTS, MODULE_SEPARATOR}; + +/// Whether custom-rule code may import the absolute module `module`. +#[must_use] +pub fn allowed_custom_rule_import(module: &str) -> bool { + CUSTOM_RULE_IMPORT_ROOTS.iter().any(|root| { + module + .strip_prefix(root) + .is_some_and(|rest| rest.is_empty() || rest.starts_with(MODULE_SEPARATOR)) + }) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs new file mode 100644 index 0000000000..6f0e972915 --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/disallowed_import.rs @@ -0,0 +1,49 @@ +//! The first import a custom-rule source makes outside the allowlist, in `ast.walk` order. + +use ruff_python_ast::Stmt; +use ruff_text_size::Ranged; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; +use crate::rule_evidence::models::{DisallowedImport, UnparsedSource}; + +/// The first disallowed absolute import in breadth-first order, or `UnparsedSource`. +pub fn disallowed_import( + source: &str, + python_version: (u8, u8), +) -> Result, UnparsedSource> { + let parsed: ParsedSource = + parse_cpython_module(source, python_version).ok_or(UnparsedSource)?; + let roots: Vec> = parsed.module.body.iter().map(CpythonNode::Stmt).collect(); + for node in walk_breadth_first(roots) { + let (statement, modules): (&Stmt, Vec<&str>) = match node { + CpythonNode::Stmt(statement @ Stmt::Import(import)) => ( + statement, + import + .names + .iter() + .map(|alias| alias.name.as_str()) + .collect(), + ), + CpythonNode::Stmt(statement @ Stmt::ImportFrom(import)) if import.level == 0 => ( + statement, + vec![import.module.as_ref().map_or("", |module| module.as_str())], + ), + _ => continue, + }; + if let Some(module) = modules + .into_iter() + .find(|module| !allowed_custom_rule_import(module)) + { + let (line, _) = source_position(&parsed.lines, statement.start()); + return Ok(Some(DisallowedImport { + line, + module: module.to_owned(), + })); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs index e8b3ce047f..449b4c6551 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/main/mod.rs @@ -1,3 +1,5 @@ +pub mod allowed_custom_rule_import; +pub mod disallowed_import; pub mod implementation_fingerprint; pub mod import_closure; pub mod test_evidence; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/models.rs b/crates/sqlbuild-pysource/src/rule_evidence/models.rs index 68acd03cdb..f18c81f752 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/models.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/models.rs @@ -15,3 +15,14 @@ pub(crate) struct RuleTarget<'a> { pub(crate) modules: Vec, pub(crate) check_name: &'a str, } + +/// An import outside the custom-rule allowlist: its statement's line and the module. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DisallowedImport { + pub line: usize, + pub module: String, +} + +/// A source CPython's parser would reject. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct UnparsedSource; diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests.rs index 52b71770bf..1d5ac43a78 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_hermeticity.rs"] +mod hermeticity; #[path = "tests/test_rule_evidence.rs"] mod rule_evidence; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs new file mode 100644 index 0000000000..536347293e --- /dev/null +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_hermeticity.rs @@ -0,0 +1,121 @@ +use crate::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; +use crate::rule_evidence::main::disallowed_import::disallowed_import; +use crate::rule_evidence::models::{DisallowedImport, UnparsedSource}; +use crate::rule_evidence::tests::test_types::{AllowedImportTestCase, DisallowedImportTestCase}; + +const PYTHON: (u8, u8) = (3, 12); + +#[test] +fn given_module_names_when_checked_then_only_allowlisted_roots_and_submodules_pass() { + let test_cases = [ + AllowedImportTestCase { + description: "an allowed root", + module: "json", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "an allowed submodule", + module: "json.decoder", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "an allowed dotted root", + module: "sqlbuild.rules", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "a submodule of an allowed package", + module: "collections.abc", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "a longer name sharing a prefix", + module: "jsonx", + expected_allowed: false, + }, + AllowedImportTestCase { + description: "the parent of an allowed dotted root", + module: "sqlbuild", + expected_allowed: false, + }, + AllowedImportTestCase { + description: "an empty module", + module: "", + expected_allowed: false, + }, + AllowedImportTestCase { + description: "a trailing dot", + module: "json.", + expected_allowed: true, + }, + AllowedImportTestCase { + description: "a leading dot", + module: ".json", + expected_allowed: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + allowed_custom_rule_import(test_case.module), + test_case.expected_allowed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rule_sources_when_scanned_then_the_first_disallowed_import_in_walk_order_is_found() { + let test_cases = [ + DisallowedImportTestCase { + description: "only allowed and relative imports", + source: "import json\nfrom . import helpers\nfrom re import x\n", + expected_import: Ok(None), + }, + DisallowedImportTestCase { + description: "the first disallowed top-level import", + source: "import json\nimport os\nimport subprocess\n", + expected_import: Ok(Some(DisallowedImport { + line: 2, + module: "os".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "top-level statements are visited before nested ones", + source: "def f():\n import socket\n\nfrom urllib import request\n", + expected_import: Ok(Some(DisallowedImport { + line: 4, + module: "urllib".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "each alias of one statement in order", + source: "import json, os.path as p, re\n", + expected_import: Ok(Some(DisallowedImport { + line: 1, + module: "os.path".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "CRLF lines and continuations", + source: "import json\r\n\r\nimport \\\n os\r\n", + expected_import: Ok(Some(DisallowedImport { + line: 3, + module: "os".to_owned(), + })), + }, + DisallowedImportTestCase { + description: "a syntax error", + source: "def broken(:\n", + expected_import: Err(UnparsedSource), + }, + ]; + for test_case in test_cases { + assert_eq!( + disallowed_import(test_case.source, PYTHON), + test_case.expected_import, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs index 6fc509556a..fe7802a5cc 100644 --- a/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs +++ b/crates/sqlbuild-pysource/src/rule_evidence/tests/test_types.rs @@ -5,3 +5,18 @@ pub(crate) struct EvidenceTestCase { pub(crate) expected_closure: Vec<&'static str>, pub(crate) expected_cases: Vec<(usize, usize, &'static str)>, } + +pub(crate) struct AllowedImportTestCase { + pub(crate) description: &'static str, + pub(crate) module: &'static str, + pub(crate) expected_allowed: bool, +} + +pub(crate) struct DisallowedImportTestCase { + pub(crate) description: &'static str, + pub(crate) source: &'static str, + pub(crate) expected_import: Result< + Option, + crate::rule_evidence::models::UnparsedSource, + >, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index ea8a364d13..c9b585108b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -30,12 +30,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const LOADER_DEFINITION_MODULE: &str = "sqlbuild.python_nodes.main.read_loader_definition"; const NATIVE_MODULE: &str = "sqlbuild._native"; const BUILTINS_MODULE: &str = "builtins"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs index d03eb26660..81e054e539 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::explicit_references::models::{ }; use sqlbuild_analysis::explicit_references::types::ResourceKey; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs index c7620efbe8..99bd354073 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/required_columns.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::types::CompilerDetach; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; /// CTE-aware column reads of `query_sql`, resolved to the external resources it references. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs index 2ff4d0fb6d..1c6dfec894 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_columns.rs @@ -19,7 +19,7 @@ use crate::bindings::types::CompilerDetach; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; const LEGACY_MODE: &str = "legacy"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs rename to crates/sqlbuild-python/src/bindings/_helpers/analysis_session/fixture_inferences.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index baca587c27..5773febd9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod fixture_columns; +pub(crate) mod fixture_inferences; pub(crate) mod model_analyses; pub(crate) mod nullability_rules; pub(crate) mod oracle; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 9dadfd103b..73b4c55997 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -13,10 +13,11 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_s use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const LOG_DEBUG_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; const LOGGING_MODULE: &str = "logging"; @@ -308,8 +309,8 @@ fn model_rows<'py>( || (!entry.is_none() && entry.getattr("type_enforcement")?.is_truthy()?); let snapshot_columns: Option<(String, String)> = if get("materialized")?.eq(&snapshot)? { Some(( - or_default(&get("valid_from_column")?, "valid_from")?, - or_default(&get("valid_to_column")?, "valid_to")?, + or_default(&get("valid_from_column")?, VALID_FROM_DEFAULT)?, + or_default(&get("valid_to_column")?, VALID_TO_DEFAULT)?, )) } else { None diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index 28a3b907ff..9769df9baa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -7,10 +7,13 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + confidence_member, transform_kind_member, +}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const LINEAGE_NATIVE: &str = "native"; @@ -233,7 +236,6 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { let column_class: Bound<'py, PyAny> = shape_class(models.as_any(), "CompiledLineageColumnFact")?; let source_class: Bound<'py, PyAny> = models.getattr("CompiledLineageSourceFact")?; - let compact: Bound<'py, PyAny> = models.getattr("CompactLineageFacts")?; let mut facts: Vec> = Vec::new(); for row in rows.try_iter()? { let (output_column, transform_code, confidence_code, sources): ( @@ -265,11 +267,11 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { ("upstream_columns", PyTuple::new(py, upstream)?.into_any()), ( "transform_kind", - compact.call_method1("transform_kind", (transform_code,))?, + transform_kind_member(py, transform_code.extract()?)?.into_bound(py), ), ( "confidence", - compact.call_method1("confidence", (confidence_code,))?, + confidence_member(py, confidence_code.extract()?)?.into_bound(py), ), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index 04b7a39550..0c43aea023 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -1,15 +1,16 @@ //! Register the native model analysis session functions. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::analysis_session::{ - fixture_columns, model_analyses, oracle, outcome_objects, query_columns, session, - stage_functions, + fixture_columns, fixture_inferences, model_analyses, oracle, outcome_objects, query_columns, + session, stage_functions, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; fixture_columns::register(module)?; + module.add_class::()?; model_analyses::register(module)?; outcome_objects::register(module)?; query_columns::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index ef50a1c090..3635f6b608 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -33,7 +33,7 @@ const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ ("table_fn", "table_fn"), ]; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; /// Each query's `(succeeded, column rows, has star)`; a native internal failure raises. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index 332a25bccd..262ad5285a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -10,7 +10,7 @@ use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CURSOR_START: &str = "__cursor_start"; const CURSOR_START_CALL: &str = "__cursor_start()"; @@ -20,7 +20,11 @@ const INTEGER_ANALYSIS_BOUND: &str = "CAST(0 AS BIGINT)"; const TIMESTAMP_ANALYSIS_BOUND: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; const MODEL_SQL_CONTEXT: &str = "Model SQL"; const SQL_CONTEXT: &str = "SQL"; -const RESERVED_CURSOR_MARKERS: [&str; 2] = ["__SQB_CURSOR_START__", "__SQB_CURSOR_END__"]; +/// Executor placeholder replaced with a microbatch's start bound. +pub(crate) const MICROBATCH_START_SENTINEL: &str = "__SQB_CURSOR_START__"; +/// Executor placeholder replaced with a microbatch's end bound. +pub(crate) const MICROBATCH_END_SENTINEL: &str = "__SQB_CURSOR_END__"; +const RESERVED_CURSOR_MARKERS: [&str; 2] = [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL]; /// The reserved executor bound markers no authored SQL may contain. pub(crate) fn reserved_cursor_markers() -> Vec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs index a71a07949d..849c6f09cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs @@ -9,7 +9,7 @@ use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// The reference function and diagnostic code of each marker kind, in the scanner's order. const MARKER_KINDS: [(&str, &str); 5] = [ ("__ref", "R001"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs index d0b0490370..2bdbaebbfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::auditing::models::{ GateIdentity, MeasurementFacts, PlannedAudit, ThresholdBound, Thresholds, }; -const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const MEASUREMENT_MODE: &str = "measurement"; fn enum_value(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs index 8e07eaa020..dca87c66e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/mod.rs @@ -1,3 +1,4 @@ mod fields; pub(crate) mod project_vars; +mod python_nodes; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs new file mode 100644 index 0000000000..9a09f3ab30 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -0,0 +1,215 @@ +//! Python-node authoring checks: retry policies, audit cases and hook reads, in Python's operators. + +use pyo3::exceptions::{PyBaseException, PyTypeError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyDict, PyDictMethods, PyInt, PyMemoryView, PyString, PyStringMethods, PyTuple, PyType, +}; +use pyo3::types::{PyTupleMethods, PyTypeMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; +const SHARED_INPUT_ERROR: &str = "SharedInputError"; +const MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; +const RESOURCE_REF_CLASS: &str = "SqlResourceRef"; +const RANDOM_MODULE: &str = "random"; +const MAX_ATTEMPTS_TOO_LOW: &str = "RetryPolicy max_attempts must be at least 1"; +const INITIAL_DELAY_NEGATIVE: &str = "RetryPolicy initial_delay_seconds must be non-negative"; +const BACKOFF_TOO_LOW: &str = "RetryPolicy backoff_multiplier must be at least 1"; +const MAX_DELAY_NEGATIVE: &str = "RetryPolicy max_delay_seconds must be non-negative"; +const MAX_ELAPSED_NEGATIVE: &str = "RetryPolicy max_elapsed_seconds must be non-negative"; +const RETRY_ON_NOT_EXCEPTIONS: &str = "RetryPolicy retry_on must contain exception classes"; +const AUDIT_CASE_NAME_INVALID: &str = "AuditCase name must be a non-empty lowercase identifier \ + using letters, numbers, and underscores"; +const AUDIT_CASE_DEFINITION_EMPTY: &str = "AuditCase definition must be non-empty"; +const AUDIT_CASE_MINIMUM_SAMPLES: &str = "AuditCase minimum_samples must be a non-negative integer"; +const AUDIT_CASE_EVIDENCE_LIMIT: &str = "AuditCase evidence_limit must be a non-negative integer"; +const BYTES_NAME_ERROR: &str = "cannot use a string pattern on a bytes-like object"; +const HOOK_READS_INVALID: &str = + "@hook(reads=...) accepts model(...), source(...), and seed(...) references only; got "; + +/// `SharedInputError(message)`. +fn shared_input_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, ERRORS_MODULE, SHARED_INPUT_ERROR).and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `value < bound` under Python's comparison. +fn below(value: &Bound<'_, PyAny>, bound: i64) -> PyResult { + value.lt(bound) +} + +/// Whether `value` is an exception class. +fn exception_class(value: &Bound<'_, PyAny>) -> PyResult { + match value.downcast::() { + Ok(class) => class.is_subclass_of::(), + Err(_) => Ok(false), + } +} + +/// The validated `retry_on` exception classes of a retry policy. +#[pyfunction] +#[pyo3(signature = (**request))] +fn retry_policy_exceptions<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("retry_policy_exceptions", request)?; + let py: Python<'py> = request.py(); + let max_delay_seconds: Bound<'py, PyAny> = request.get("max_delay_seconds")?; + let max_elapsed_seconds: Bound<'py, PyAny> = request.get("max_elapsed_seconds")?; + if below(&request.get("max_attempts")?, 1)? { + return Err(shared_input_error(py, MAX_ATTEMPTS_TOO_LOW)); + } + if below(&request.get("initial_delay_seconds")?, 0)? { + return Err(shared_input_error(py, INITIAL_DELAY_NEGATIVE)); + } + if below(&request.get("backoff_multiplier")?, 1)? { + return Err(shared_input_error(py, BACKOFF_TOO_LOW)); + } + if !max_delay_seconds.is_none() && below(&max_delay_seconds, 0)? { + return Err(shared_input_error(py, MAX_DELAY_NEGATIVE)); + } + if !max_elapsed_seconds.is_none() && below(&max_elapsed_seconds, 0)? { + return Err(shared_input_error(py, MAX_ELAPSED_NEGATIVE)); + } + let retry_on: Bound<'py, PyAny> = request.get("retry_on")?; + if exception_class(&retry_on)? { + return PyTuple::new(py, [retry_on]); + } + let classes: Bound<'py, PyTuple> = PyTuple::new( + py, + retry_on + .try_iter()? + .collect::>>>()?, + )?; + for class in classes.iter() { + if !exception_class(&class)? { + return Err(shared_input_error(py, RETRY_ON_NOT_EXCEPTIONS)); + } + } + Ok(classes) +} + +/// The delay before retry `retry_index`: exponential backoff, capped, and jittered by `random`. +#[pyfunction] +#[pyo3(signature = (*, retry_policy, retry_index))] +fn retry_delay_seconds<'py>( + retry_policy: &Bound<'py, PyAny>, + retry_index: &Bound<'py, PyAny>, +) -> PyResult> { + let initial: Bound<'py, PyAny> = retry_policy.getattr("initial_delay_seconds")?; + let growth: Bound<'py, PyAny> = retry_policy + .getattr("backoff_multiplier")? + .pow(retry_index, retry_policy.py().None())?; + let mut delay: Bound<'py, PyAny> = initial.mul(growth)?; + let maximum: Bound<'py, PyAny> = retry_policy.getattr("max_delay_seconds")?; + if !maximum.is_none() && maximum.lt(&delay)? { + delay = maximum; + } + if retry_policy.getattr("jitter")?.is_truthy()? { + return imported(retry_policy.py(), RANDOM_MODULE, "uniform")?.call1((0, delay)); + } + Ok(delay) +} + +/// The `re.fullmatch` failure for a name that is not a string. +fn name_type_error(name: &Bound<'_, PyAny>) -> PyResult { + match PyMemoryView::from(name) { + Ok(_) => return Ok(PyTypeError::new_err(BYTES_NAME_ERROR)), + Err(error) if error.is_instance_of::(name.py()) => {} + Err(error) => return Err(error), + } + Ok(PyTypeError::new_err(format!( + "expected string or bytes-like object, got '{}'", + name.get_type().name()? + ))) +} + +/// Whether an optional count is absent or a non-negative integer other than a boolean. +fn valid_count(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(true); + } + if !value.is_instance_of::() || value.is_instance_of::() { + return Ok(false); + } + Ok(!below(value, 0)?) +} + +/// Reject an audit case without a lowercase name, a definition, or valid optional counts. +#[pyfunction] +#[pyo3(signature = (*, name, definition, minimum_samples, evidence_limit))] +fn validate_audit_case( + name: &Bound<'_, PyAny>, + definition: &Bound<'_, PyAny>, + minimum_samples: &Bound<'_, PyAny>, + evidence_limit: &Bound<'_, PyAny>, +) -> Result<(), PyErr> { + let py: Python<'_> = name.py(); + let Ok(text) = name.downcast::() else { + return Err(name_type_error(name)?); + }; + if !is_lowercase_identifier(&text.to_string_lossy()) { + return Err(shared_input_error(py, AUDIT_CASE_NAME_INVALID)); + } + if !definition.call_method0("strip")?.is_truthy()? { + return Err(shared_input_error(py, AUDIT_CASE_DEFINITION_EMPTY)); + } + if !valid_count(minimum_samples)? { + return Err(shared_input_error(py, AUDIT_CASE_MINIMUM_SAMPLES)); + } + if !valid_count(evidence_limit)? { + return Err(shared_input_error(py, AUDIT_CASE_EVIDENCE_LIMIT)); + } + Ok(()) +} + +/// `@hook(reads=...)` references in first-seen order; a leading invalid `None` is tolerated. +#[pyfunction] +fn normalize_hook_reads<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let reference: Bound<'py, PyAny> = imported(py, MODELS_MODULE, RESOURCE_REF_CLASS)?; + let reads: Vec> = if value.is_instance(&reference)? { + vec![value.clone()] + } else { + value.try_iter()?.collect::>()? + }; + for item in &reads { + if !item.is_instance(&reference)? { + if item.is_none() { + break; + } + let message: String = format!("{HOOK_READS_INVALID}{}", item.get_type().name()?); + return Err(shared_input_error(py, &message)); + } + } + let distinct: Bound<'py, PyDict> = PyDict::new(py); + let mut ordered: Vec> = Vec::new(); + for item in reads { + if !item.is_instance(&reference)? { + continue; + } + if !distinct.contains(&item)? { + distinct.set_item(&item, py.None())?; + ordered.push(item); + } + } + PyTuple::new(py, ordered) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(retry_policy_exceptions, module)?)?; + module.add_function(wrap_pyfunction!(retry_delay_seconds, module)?)?; + module.add_function(wrap_pyfunction!(validate_audit_case, module)?)?; + module.add_function(wrap_pyfunction!(normalize_hook_reads, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs index d33a3a7ff8..c1c11ce4c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/registration.rs @@ -2,5 +2,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::authored_values::fields::register(module)?; - crate::bindings::_helpers::authored_values::project_vars::register(module) + crate::bindings::_helpers::authored_values::project_vars::register(module)?; + crate::bindings::_helpers::authored_values::python_nodes::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index 1f81f7cb57..7b183ca425 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -20,7 +20,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const CLONE_TYPES: &str = "sqlbuild.executor.clone.types"; -const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const SPEC_TYPES: &str = "sqlbuild._native"; const TRANSIENT_MODELS_MODULE: &str = "sqlbuild.executor.clone.main._build_destination_transient_models"; const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 7b59a30828..b5e49386a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -20,14 +20,14 @@ use crate::bindings::_helpers::check::check_core::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; +use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; const NATIVE_MODULE: &str = "sqlbuild._native"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; -const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; const COST_CONTEXT_MODULE: &str = "sqlbuild._native"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; @@ -403,19 +403,11 @@ pub(crate) fn execute_build_plan<'py>( ("initial_state", initial_state), ], )?; - call_with( - &imported( - py, - FRESHNESS_MODULE, - "append_eligible_direct_source_freshness_records", - )?, - &[ - ("plan", plan), - ("result", result.clone()), - ("adapter", adapter), - ("connection_config", connection_config), - ("run_id", project.getattr("run_id")?), - ], + append_freshness_records( + (&plan, &result), + &adapter, + &connection_config, + &project.getattr("run_id")?, )?; lifecycle.finalize(py)?; Ok(Outcome { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 0845f9e4b9..b7dd57683a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -18,7 +18,7 @@ const SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS: &str = "sqlbuild._native"; const DEFER_CONFLICT_CODE: &str = "C408"; /// One build's request, resolved invocation and output settings. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index f1e02d7543..968c5fc8c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -21,7 +21,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const LOADER_DAG_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; const READ_SIDE_MODULE: &str = "sqlbuild.executor.python_nodes.main.read_side"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 3a9c5024c0..7e3ff2fac1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -34,7 +34,7 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const RELATION_PLAN_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index aa61b40fff..b6bec490be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -35,9 +35,9 @@ const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exce const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild._native"; const SKILLS_MODULE: &str = "sqlbuild._native"; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const LINT_ERRORS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CLI_LOGGER: &str = "sqlbuild.cli"; const UNKNOWN_COMMAND: &str = "unknown"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index fa12657b63..a0f6fd6b81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::clone::clone_progress::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const CLONE_MODELS: &str = "sqlbuild.executor.clone.models"; const RETENTION_REQUESTS_MODULE: &str = "sqlbuild.executor.clone.main._build_retention_requests"; @@ -24,7 +24,7 @@ const TRANSIENT_MODELS_MODULE: &str = const FINGERPRINTING_MODULE: &str = "sqlbuild.executor.clone.main._fingerprinting"; const EXECUTE_MODULE: &str = "sqlbuild.executor.clone.main.execute"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; -const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_TYPES: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 094d94104d..0530f4ec01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -7,10 +7,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ CONNECTION_DATABASE_KEY, CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, - DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, PROJECT_CONFIG_FILENAME, + DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, }; use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error; use sqlbuild_config::command_connection::main::snowflake_key_warning::snowflake_key_warning; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use crate::bindings::_helpers::command_runtime::runtime_errors::raised; use crate::bindings::_helpers::command_runtime::warehouses::{WarehouseInputs, apply_warehouse}; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs index 1d8619cffc..33da4de19c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const CLI_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// The `CliUserError` or `ProjectConfigError` for `error`. pub(crate) fn raised(py: Python<'_>, error: CommandConnectionError) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index 208e5184f4..25cd0771dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; -const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild.spec.contracts.types", "WarehouseGroup"); +const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild._native", "WarehouseGroup"); const COMMAND_TYPES: &str = "sqlbuild._native"; const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 0b24d368a5..83238fb68c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -28,6 +28,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; +/// The `WarehouseGroup` whose warehouse is a target's `warehouses.build`; any other reads `query`. +const BUILD_GROUP: &str = "build"; +const QUERY_GROUP: &str = "query"; + const ENTRY_MODELS: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -119,11 +123,12 @@ fn group_value(target: &Bound<'_, PyAny>, group: &Bound<'_, PyAny>) -> PyResult< if target.is_none() { return Ok(None); } - optional_text( - &target - .getattr("warehouses")? - .call_method1("for_group", (group,))?, - ) + let attribute: &str = if group.str()?.extract::()? == BUILD_GROUP { + BUILD_GROUP + } else { + QUERY_GROUP + }; + optional_text(&target.getattr("warehouses")?.getattr(attribute)?) } /// The active target's authored, unexpanded warehouse for one command group. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index 00e47e1ad6..30b2b65cc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -15,7 +15,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs index 96bf6589cd..0a524f17fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::model_config::template_data::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const CONNECTION_LABEL: &str = "effective connection"; /// Merge base project connection with the selected environment overrides. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 49da3e115f..42bf143684 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -13,8 +13,8 @@ use crate::bindings::_helpers::model_config::unicode_environment::unicode_enviro const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; /// Capture the project inputs every model's native config build and validation read. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 4afff18d09..a8a4f3bac9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -19,16 +19,16 @@ use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_c const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; /// The explicit reference call kinds and the `SqlResourceRefKind` member each resolves to. @@ -211,11 +211,7 @@ pub(crate) fn build_model_inputs<'py>( } else { imported(py, "pathlib", "Path")?.call0()? }; - let project_config_path: Bound<'py, PyAny> = root.div(imported( - py, - DISCOVERY_CONSTANTS_MODULE, - "PROJECT_CONFIG_FILENAME", - )?)?; + let project_config_path: Bound<'py, PyAny> = root.div(PROJECT_CONFIG_FILENAME)?; let config_inputs: Bound<'py, PyAny> = call_with( &imported(py, NATIVE_RECORDS_MODULE, "NativeModelConfigInputs")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 7f8b50d169..bf28cee411 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -12,7 +12,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; /// Build the declaration scope of `discovered`; a static scope lends only its index to macros. fn declaration_scope<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs index 462c10e10f..9d91bb850e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/storage_policies.rs @@ -5,8 +5,8 @@ use pyo3::types::{PyBool, PyString, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; /// `value`, unless it is `None`. fn present(value: Bound<'_, PyAny>) -> Option> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs index 5d1a7f85fc..649bcd6d24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -1,5 +1,9 @@ //! Compile reuse storage, fingerprint, environment and notice constants. +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + +use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; + /// The directory under the compiler cache directory holding the native reuse store. pub(crate) const NATIVE_REUSE_DIRECTORY_NAME: &str = "project-reuse-v2"; /// The reuse directory an older release kept stored compiles in. @@ -11,14 +15,12 @@ pub(crate) const REUSE_DISABLE_VALUE: &str = "1"; pub(crate) const REUSE_LOGGER_NAME: &str = "sqlbuild.compile.reuse"; pub(crate) const TRACKED_ENVIRONMENT_PREFIXES: [&str; 2] = ["SQLBUILD_", "SQB_"]; /// Sorted; tracked-prefix variables that never invalidate a stored compile. -pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = [ - "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR", - "SQLBUILD_DISABLE_COMPILE_REUSE", -]; +pub(crate) const UNTRACKED_ENVIRONMENT_NAMES: [&str; 2] = + [CAPTURE_DIR_ENV_VAR, REUSE_DISABLE_ENV_VAR]; pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = "\0missing"; pub(crate) const MISSING_FILE_DIGEST: &str = "\0missing"; pub(crate) const PROJECT_CONFIG_FILENAMES: [&str; 2] = - ["sqlbuild_project.toml", "sqlbuild_project.yml"]; + [PROJECT_CONFIG_FILENAME, "sqlbuild_project.yml"]; /// The `sqlbuild._native` attribute the reuse check reads its racy window from. pub(crate) const RACY_WINDOW_ATTRIBUTE: &str = "COMPILE_REUSE_RACY_WINDOW_NS"; /// The `sqlbuild._native` attribute the reuse check reads its excluded root directories from. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index 73544bca28..a10d21a019 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -6,6 +6,8 @@ pub(crate) mod registration; pub(crate) mod report; pub(crate) mod reuse_attempt; pub(crate) mod reuse_recording; +pub(crate) mod runtime_identity; +pub(crate) mod startup_inputs; pub(crate) mod surrogates; pub(crate) mod target_writer; pub(crate) mod test_artifact_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index 8e6f6d42e9..6b1d20cd31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -5,7 +5,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, + prepared_artifacts, report, reuse_attempt, reuse_recording, runtime_identity, startup_inputs, + target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -13,6 +14,8 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { report::register(module)?; reuse_attempt::register(module)?; reuse_recording::register(module)?; + runtime_identity::register(module)?; + startup_inputs::register(module)?; target_writer::register(module)?; prepared_artifacts::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index fcbb4a85bf..eeb8de479d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const METADATA_MODULE: &str = "importlib.metadata"; const TARGET_DIRECTORY: &str = "target"; const COMPILED_DIRECTORY: &str = "compiled"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index c8271b4702..dfaef4e9b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; +use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FORMAT}; @@ -26,6 +27,9 @@ use crate::bindings::_helpers::compile_command::constants::{ TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, }; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; +use crate::bindings::_helpers::compile_command::runtime_identity::{ + invocation_identity, redirected_output_files, runtime_identity, +}; use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::compile_outputs::json_report::plain_json; @@ -34,10 +38,8 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"); -const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; -const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; const HIT: &str = "hit"; const CHECKED_HIT: &str = "hit"; const CHECKED_BYPASS: &str = "bypass"; @@ -68,8 +70,7 @@ pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult = - imported(py, RUNTIME_IDENTITY_MODULE, "redirected_output_files")?.call0()?; + let output_files: Bound<'_, PyList> = redirected_output_files(py)?; if with_output_files { rules.set_item("output_files", output_files)?; } @@ -210,11 +211,7 @@ pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let environment: Bound<'_, PyAny> = imported(py, "os", "environ")?; - if environment.contains(imported( - py, - OUTPUT_CAPTURE_CONSTANTS, - "INVOCATION_CONTEXT_ENV", - )?)? { + if environment.contains(INVOCATION_CONTEXT_ENV)? { return Ok(()); } let mut directory: Bound<'_, PyAny> = attempt.getattr("project_dir")?; @@ -274,19 +271,9 @@ fn reuse_check<'py>( check.set_item("selected_target", request.getattr("selected_target")?)?; check.set_item( "invocation", - call_with( - &imported(py, RUNTIME_IDENTITY_MODULE, "invocation_identity")?, - &[ - ("request", request.clone()), - ("project_dir", project_dir.clone()), - ("use_color", use_color.into_bound_py_any(py)?), - ], - )?, - )?; - check.set_item( - "runtime", - imported(py, RUNTIME_IDENTITY_MODULE, "runtime_identity")?.call0()?, + invocation_identity(request, project_dir, use_color)?, )?; + check.set_item("runtime", runtime_identity(py)?)?; check.set_item("search_path", listed(py.import("sys")?.getattr("path")?)?)?; check.set_item( "bypass_requested", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs new file mode 100644 index 0000000000..9e4f5df9cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs @@ -0,0 +1,112 @@ +//! Interpreter, native build and invocation facts that decide whether a stored compile is reusable. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_cli::replay::main::output_file_identities::output_file_identities; + +use crate::bindings::_helpers::compile_driver::python_calls::keywords; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The version of the runtime identity a stored compile records. +const REUSE_FORMAT_VERSION: &str = "1"; +/// Each `CompileReuseRequest` field the invocation identity records unchanged. +const REQUEST_CHOICES: [&str; 9] = [ + "no_sql_validation", + "defer_to", + "selected_target", + "json_output", + "manifest", + "dag_path", + "no_color", + "lineage_mode", + "cli_vars", +]; + +/// `sys.version`, `sys.executable`, `sys.platform` and `str(sys.implementation.cache_tag)`. +pub(crate) fn interpreter_values(py: Python<'_>) -> PyResult<[String; 4]> { + let sys: Bound<'_, PyModule> = py.import("sys")?; + Ok([ + sys.getattr("version")?.extract()?, + sys.getattr("executable")?.extract()?, + sys.getattr("platform")?.extract()?, + sys.getattr("implementation")? + .getattr("cache_tag")? + .str()? + .extract()?, + ]) +} + +/// The interpreter facts every interpreter-sensitive cache key stamps, in one order. +#[pyfunction] +fn interpreter_identity(py: Python<'_>) -> PyResult> { + PyTuple::new(py, interpreter_values(py)?) +} + +/// The interpreter and native build identity that produced a compile. +pub(crate) fn runtime_identity(py: Python<'_>) -> PyResult> { + let [version, executable, platform, cache_tag] = interpreter_values(py)?; + let identity: Bound<'_, PyDict> = PyDict::new(py); + identity.set_item("format", REUSE_FORMAT_VERSION)?; + identity.set_item("python", version)?; + identity.set_item("executable", executable)?; + identity.set_item("platform", platform)?; + identity.set_item("cache_tag", cache_tag)?; + identity.set_item( + "native_build", + py.import(NATIVE_MODULE)?.getattr("BUILD_IDENTITY")?.str()?, + )?; + Ok(identity) +} + +fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) +} + +/// Every command-line choice and location that can change compile output, as JSON bytes. +pub(crate) fn invocation_identity<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + use_color: bool, +) -> PyResult> { + let py: Python<'py> = request.py(); + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("project_dir", project_dir)?; + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let requested: Bound<'py, PyAny> = if requested.is_none() { + requested + } else { + requested.str()?.into_any() + }; + payload.set_item("requested_project_dir", requested)?; + for field in REQUEST_CHOICES { + payload.set_item(field, request.getattr(field)?)?; + } + payload.set_item("cwd", py.import("os")?.getattr("getcwd")?.call0()?)?; + payload.set_item("use_color", use_color)?; + payload.set_item("select", listed(request.getattr("select")?)?)?; + payload.set_item("exclude", listed(request.getattr("exclude")?)?)?; + payload.set_item("sys_path", listed(py.import("sys")?.getattr("path")?)?)?; + let options: Bound<'py, PyDict> = keywords( + py, + &[ + ("sort_keys", true.into_bound_py_any(py)?), + ("separators", (",", ":").into_bound_py_any(py)?), + ("default", py.import("builtins")?.getattr("repr")?), + ], + )?; + let text: Bound<'py, PyAny> = py + .import("json")? + .getattr("dumps")? + .call((payload,), Some(&options))?; + text.call_method1("encode", ("utf-8", "surrogateescape")) +} + +/// Device and inode pairs of stdout and stderr files, which hold output rather than inputs. +pub(crate) fn redirected_output_files(py: Python<'_>) -> PyResult> { + PyList::new(py, output_file_identities()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(interpreter_identity, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs new file mode 100644 index 0000000000..3be85bc159 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/startup_inputs.rs @@ -0,0 +1,69 @@ +//! Stamps of the interpreter and site-path inputs Python startup depends on. + +use std::path::PathBuf; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyList; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_cache::startup_inputs::main::startup_stamps::startup_stamps; +use sqlbuild_cache::startup_inputs::models::{StartupLocations, StartupStamp}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const OS_MODULE: &str = "os"; + +fn paths(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// `OSError(errno, os.strerror(errno), path)`, which Python narrows to its subclass. +fn stat_error(py: Python<'_>, path: PathBuf, error: &std::io::Error) -> PyErr { + let Some(errno) = error.raw_os_error() else { + return PyOSError::new_err(error.to_string()); + }; + let message: String = match imported(py, OS_MODULE, "strerror") + .and_then(|strerror| strerror.call1((errno,))) + .and_then(|text| text.extract()) + { + Ok(message) => message, + Err(failure) => return failure, + }; + let Ok(path) = path.into_os_string().into_pyobject(py); + PyOSError::new_err((errno, message, path.unbind())) +} + +/// `[(path, st_mtime_ns, st_size), ...]` for every startup input path. +#[pyfunction] +#[pyo3(signature = (*, directories, files, prefixes))] +fn startup_input_stamps<'py>( + directories: &Bound<'py, PyAny>, + files: &Bound<'py, PyAny>, + prefixes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = directories.py(); + let locations = StartupLocations { + directories: paths(directories)?, + files: paths(files)?, + prefixes: paths(prefixes)?, + }; + let cwd: PathBuf = imported(py, OS_MODULE, "getcwd")?.call0()?.extract()?; + let stamps: Vec = + startup_stamps(&locations, &cwd).map_err(|(path, error)| stat_error(py, path, &error))?; + let rows: Vec> = stamps + .into_iter() + .map(|stamp| { + Ok((stamp.path.into_os_string(), stamp.modified_ns, stamp.size) + .into_pyobject(py)? + .into_any()) + }) + .collect::>()?; + PyList::new(py, rows) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(startup_input_stamps, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index a71989a2db..c0ede29fd4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -48,8 +48,8 @@ const PUBLISHED_STAGING_CHANGED: &str = "changed"; const PUBLISHED_TREE: &str = "tree"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; const PLAN_ARTIFACTS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 2997c586de..1700417c93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -31,7 +31,7 @@ const CONTRACTS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SELECTION_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index e58b1509b1..fbe498fcd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -19,7 +19,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index fb915fda00..7f9878328f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::render::declaration_values::declaration_scope_res const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 28e30a54b9..e848b2a2cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -24,7 +24,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; const AUDIT_SLOT: &str = "audit"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 543432df89..e085317b33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 42557c8924..35a99e92ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -12,7 +12,7 @@ const SPEC_ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; /// `WarehouseGroup` members in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index f3b8062bda..a07e375fd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const PRESERVE_TARGET_VALUE: &str = "preserve"; /// `sorted(DEFERRED_TARGET_CONTEXT_KEYS)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs index 5b151c0955..f2b8f5325c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -9,7 +9,7 @@ use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const CONTEXT_VARIABLE_NAME: &str = "sqlbuild_collected_compile_diagnostics"; static ACTIVE_COLLECTOR: PyOnceLock> = PyOnceLock::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index b45acacb9a..0787d960c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::graph::views::project_lineage_views; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 273d43ecf8..36d097f030 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -38,7 +38,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const FILESYSTEM_MODULE: &str = "sqlbuild._native"; const SINKS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; /// Declaration file collections counted into the declaration-parse lifecycle, in field order. const DECLARATION_COLLECTIONS: [&str; 14] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 944e325e08..95088a29f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -14,7 +14,7 @@ const ATTACHMENT_MODULE: &str = "sqlbuild._native"; const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; const ATTACH_RESOURCES_MODULE: &str = "sqlbuild._native"; const INPUTS_VIEW_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs index 22dcb11e42..1ac19d99e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -7,8 +7,8 @@ use sqlbuild_discovery::yaml_entries::main::integration_loader_name::integration use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const DISCOVERY_TYPES_MODULE: &str = "sqlbuild.compiler.discovery.types"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_TYPES_MODULE: &str = "sqlbuild._native"; /// One integration: its kind constant and the runtime module building its loader functions. struct Integration { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 075ad1d7af..3f400630ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -18,9 +18,9 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const EMPTY_DAG_PATH: &str = ""; const TARGET_DIRECTORY: &str = "target"; const DEFAULT_DAG_FILE: &str = "sqlbuild_dag.json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index 063eda4ee2..4f929f4271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -22,13 +22,13 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const TIMING_TRACKER_MODULE: &str = "sqlbuild._native"; -const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild._native"; -const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const RULE_MODELS_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 2a9f54957d..5b964605ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -10,8 +10,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild._native"; +const PLANNER_MODELS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// Discovered Python node functions whose selectors need the Python graph. const PYTHON_NODE_FUNCTIONS: [&str; 4] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 7821a065e4..a2660a48e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -17,7 +17,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const EXECUTION_PLAN_MODULE: &str = "sqlbuild._native"; const MATERIALIZATIONS_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.materializations"; -const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; +const PIPELINE_MODELS_MODULE: &str = "sqlbuild._native"; const SINGLE_CONNECTION: i64 = 1; /// The deferred relation locations and the warehouse relations behind them, when deferring. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 81b9bcee82..b82f324eb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -26,7 +26,8 @@ const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; @@ -231,18 +232,15 @@ fn targeted_project<'py>( ], )?; project_targets(&adapter_name, &project)?; - let promotion_key: Bound<'py, PyAny> = imported( - py, - DISCOVERY_CONSTANTS_MODULE, - "TABLE_PROMOTION_MODE_SETTING_KEY", - )?; + let promotion_key: Bound<'py, PyAny> = + pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); let settings_file: Bound<'py, PyAny> = if local_config .getattr("setting_overrides")? .contains(promotion_key)? { pyo3::types::PyString::new(py, LOCAL_CONFIG_FILENAME).into_any() } else { - imported(py, DISCOVERY_CONSTANTS_MODULE, "PROJECT_CONFIG_FILENAME")? + pyo3::types::PyString::new(py, PROJECT_CONFIG_FILENAME).into_any() }; let conflicts: Bound<'py, PyAny> = call_with( &imported(py, PROMOTION_MODULE, "promotion_conflict_diagnostics")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index c0cbc5c347..18b224e71d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -6,7 +6,7 @@ use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; /// The adapter defaults every compiled location falls back to. pub(crate) struct TargetDefaults<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index d64171676c..3713daa5ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; @@ -45,7 +45,7 @@ const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const DEFAULTS_SECTION: &str = "defaults"; /// One optional `(database, schema)` namespace. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index f18c81c805..f47fb226b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -52,14 +52,11 @@ const R_COMPILE_MODEL_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("logical_database", FieldDefault::None), FieldSpec::new( "time_travel_retention", - FieldDefault::Call( - "sqlbuild.spec.contracts.models", - "ResolvedTimeTravelRetention", - ), + FieldDefault::Call("sqlbuild._native", "ResolvedTimeTravelRetention"), ), FieldSpec::new( "table_type", - FieldDefault::Call("sqlbuild.spec.contracts.models", "ResolvedTableType"), + FieldDefault::Call("sqlbuild._native", "ResolvedTableType"), ), ], }; @@ -87,12 +84,14 @@ const R_DECLARATION_SCOPE_RESOLVER: RecordSpec = RecordSpec { name: "native_contexts", default: FieldDefault::Required, compared: false, + shown: false, }, FieldSpec::new("resource_specific", FieldDefault::None), FieldSpec { name: "contexts_by_directory", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, ], }; @@ -224,6 +223,7 @@ const R_COMPILE_ANALYSIS_SELECTION: RecordSpec = RecordSpec { name: "on_inputs_ready", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -299,7 +299,7 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { FieldSpec::new("effective_connection", FieldDefault::EmptyDict), FieldSpec::new( "effective_settings", - FieldDefault::Call("sqlbuild.spec.contracts.models", "SettingsConfig"), + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), ), FieldSpec::new("no_sql_validation", FieldDefault::False), FieldSpec::new("effective_vars", FieldDefault::EmptyDict), @@ -307,6 +307,7 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { name: "macro_context", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec::new("loaded_macros", FieldDefault::EmptyDict), FieldSpec::new("public_enums", FieldDefault::EmptyDict), @@ -322,17 +323,19 @@ const R_COMPILE_PROJECT_INPUTS: RecordSpec = RecordSpec { FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), FieldSpec::new( "scope_index", - FieldDefault::Call("sqlbuild.compiler.scopes.models", "ScopeIndex"), + FieldDefault::Call("sqlbuild._native", "ScopeIndex"), ), FieldSpec { name: "declaration_scope", default: FieldDefault::None, compared: false, + shown: false, }, FieldSpec { name: "native_project", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -596,7 +599,7 @@ const R_DISCOVERED_SQL_HOOK_FILE: RecordSpec = RecordSpec { FieldSpec::new("description", FieldDefault::None), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -618,7 +621,7 @@ const R_DISCOVERED_ENUM_FILE: RecordSpec = RecordSpec { FieldSpec::required("declarations"), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -635,7 +638,7 @@ const R_DISCOVERED_CONSTANT_FILE: RecordSpec = RecordSpec { FieldSpec::required("declarations"), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -663,7 +666,7 @@ const R_DISCOVERED_MODEL_SCHEMA_FILE: RecordSpec = RecordSpec { FieldSpec::required("declarations"), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -739,11 +742,11 @@ const R_DISCOVERED_AUDIT_FILE: RecordSpec = RecordSpec { FieldSpec::required("blocks"), FieldSpec::new( "declaration_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "DeclarationKind", "AUDIT"), + FieldDefault::Member("sqlbuild._native", "DeclarationKind", "AUDIT"), ), FieldSpec::new( "scope_kind", - FieldDefault::Member("sqlbuild.compiler.scopes.types", "ScopeKind", "GLOBAL"), + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), ), FieldSpec::new("ownership_root", FieldDefault::None), FieldSpec::new("owning_path", FieldDefault::None), @@ -795,8 +798,818 @@ const R_TOLERANT_SCOPE_DISCOVERY: RecordSpec = RecordSpec { FieldSpec::new("config_faults", FieldDefault::EmptyTuple), ], }; +const R_SQL_HOOK_ENTRY: RecordSpec = RecordSpec { + name: "SqlHookEntry", + doc: "A model lifecycle hook that executes SQL.", + fields: &[ + FieldSpec::required("statement"), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("relative_path", FieldDefault::None), + FieldSpec::new("definition_sql", FieldDefault::None), + FieldSpec::new("kwargs", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("reads", FieldDefault::EmptyTuple), + ], +}; +const R_PYTHON_HOOK_ENTRY: RecordSpec = RecordSpec { + name: "PythonHookEntry", + doc: "A model lifecycle hook that invokes a discovered Python hook.", + fields: &[FieldSpec::required("name"), FieldSpec::required("kwargs")], +}; +const R_ENUM_DECLARATION: RecordSpec = RecordSpec { + name: "EnumDeclaration", + doc: "One validated public or model-local enum declaration.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("members"), + FieldSpec::required("scalar_type"), + FieldSpec::required("relative_path"), + FieldSpec::new("model_name", FieldDefault::None), + ], +}; +const R_CONSTANT_DECLARATION: RecordSpec = RecordSpec { + name: "ConstantDeclaration", + doc: "One validated public or model-local constant declaration.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("value"), + FieldSpec::required("relative_path"), + FieldSpec::new("model_name", FieldDefault::None), + FieldSpec::new("render_as", FieldDefault::None), + ], +}; +const R_DISCOVERED_SOURCE_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSourceFile", + doc: "A discovered source declaration file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("source_entries"), + ], +}; +const R_DISCOVERED_SQL_TEST_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSqlTestFile", + doc: "A discovered SQL-native test file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("blocks"), + FieldSpec::new( + "ownership_root", + FieldDefault::CallText("pathlib", "Path", "tests/unit"), + ), + ], +}; +const R_SQL_TEST_PARAMETER_DECLARATION: RecordSpec = RecordSpec { + name: "SqlTestParameterDeclaration", + doc: "One typed parameter declared by a SQL-native test template.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("value_type"), + FieldSpec::new("nullable", FieldDefault::False), + ], +}; +const R_DISCOVERED_SQL_SCENARIO_FILE: RecordSpec = RecordSpec { + name: "DiscoveredSqlScenarioFile", + doc: "A discovered SQL-native scenario file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::required("header_values"), + FieldSpec::required("sql_body"), + FieldSpec::required("name"), + FieldSpec::new( + "ownership_root", + FieldDefault::CallText("pathlib", "Path", "tests/scenarios"), + ), + FieldSpec::new("sql_body_span", FieldDefault::None), + ], +}; +const R_DISCOVERED_MACRO_FILE: RecordSpec = RecordSpec { + name: "DiscoveredMacroFile", + doc: "A discovered project macro file and its raw contents.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("contents"), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_MATERIALIZATION_FILE: RecordSpec = RecordSpec { + name: "DiscoveredMaterializationFile", + doc: "A discovered custom materialization Python file.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_PROVIDER_USAGE: RecordSpec = RecordSpec { + name: "DiscoveredProviderUsage", + doc: "Framework-owned metadata for provider usage by a Python callable surface.", + fields: &[ + FieldSpec::required("provider_name"), + FieldSpec::required("parameter_name"), + FieldSpec::new("annotation_class_name", FieldDefault::None), + FieldSpec::new("annotation_module", FieldDefault::None), + ], +}; +const R_DISCOVERED_LOADER_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredLoaderFunction", + doc: "A discovered project source loader function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("destination", FieldDefault::None), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new( + "connection_mode", + FieldDefault::Member("sqlbuild._native", "LoaderConnectionMode", "SQLBUILD"), + ), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredTaskFunction", + doc: "A discovered project task function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("retry", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredAssetFunction", + doc: "A discovered project asset function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("column_lineage", FieldDefault::None), + FieldSpec::new("retry", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_CHECK_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredCheckFunction", + doc: "A discovered project check function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("depends_on"), + FieldSpec::new( + "severity", + FieldDefault::Member( + "sqlbuild.python_nodes.types", + "PythonCheckSeverity", + "ERROR", + ), + ), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_HOOK_FUNCTION: RecordSpec = RecordSpec { + name: "DiscoveredHookFunction", + doc: "A discovered project model lifecycle hook function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("reads", FieldDefault::EmptyTuple), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec::new( + "scope_kind", + FieldDefault::Member("sqlbuild._native", "ScopeKind", "GLOBAL"), + ), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("declaration_root", FieldDefault::None), + ], +}; +const R_DISCOVERED_AUDIT_FACTORY: RecordSpec = RecordSpec { + name: "DiscoveredAuditFactory", + doc: "A discovered audit factory and its normalized cases.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("line"), + FieldSpec::new("cases", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_EVENT_EXPORTER_DECLARATION: RecordSpec = RecordSpec { + name: "DiscoveredEventExporterDeclaration", + doc: "A discovered exporter declaration before provider binding.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("event_kinds"), + FieldSpec::required("min_severity"), + ], +}; +const R_DISCOVERED_EVENT_EXPORTER: RecordSpec = RecordSpec { + name: "DiscoveredEventExporter", + doc: "A discovered canonical lifecycle event exporter function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new( + "event_kinds", + FieldDefault::TextFrozenset(&[ + "audit", + "invocation", + "microbatch", + "operation", + "resource", + "retry", + "run", + "statement", + ]), + ), + FieldSpec::new( + "min_severity", + FieldDefault::Member("sqlbuild._native", "EventExportSeverity", "DEBUG"), + ), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_COMMAND_OUTPUT_SINK_DECLARATION: RecordSpec = RecordSpec { + name: "DiscoveredCommandOutputSinkDeclaration", + doc: "A command-output sink declaration before provider binding.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("streams"), + ], +}; +const R_DISCOVERED_COMMAND_OUTPUT_SINK: RecordSpec = RecordSpec { + name: "DiscoveredCommandOutputSink", + doc: "A discovered command-output sink function.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::required("streams"), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_PROVIDER: RecordSpec = RecordSpec { + name: "DiscoveredProvider", + doc: "A discovered project provider class and validated settings object.", + fields: &[ + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("provider_class"), + FieldSpec::required("settings"), + ], +}; +const R_DISCOVERED_PYTHON_NODE_FUNCTIONS: RecordSpec = RecordSpec { + name: "DiscoveredPythonNodeFunctions", + doc: "Discovered project Python DAG node functions grouped by kind.", + fields: &[ + FieldSpec::new("loaders", FieldDefault::EmptyTuple), + FieldSpec::new("tasks", FieldDefault::EmptyTuple), + FieldSpec::new("assets", FieldDefault::EmptyTuple), + FieldSpec::new("checks", FieldDefault::EmptyTuple), + FieldSpec::new("audit_factories", FieldDefault::EmptyTuple), + ], +}; +const R_DISCOVERED_PROJECT_INPUTS: RecordSpec = RecordSpec { + name: "DiscoveredProjectInputs", + doc: "All raw project inputs discovered from disk before semantic resolution.", + fields: &[ + FieldSpec::required("project_config"), + FieldSpec::required("local_config"), + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("model_files", FieldDefault::EmptyTuple), + FieldSpec::new("enum_files", FieldDefault::EmptyTuple), + FieldSpec::new("constant_files", FieldDefault::EmptyTuple), + FieldSpec::new("model_schema_files", FieldDefault::EmptyTuple), + FieldSpec::new("sql_function_files", FieldDefault::EmptyTuple), + FieldSpec::new("sql_hook_files", FieldDefault::EmptyTuple), + FieldSpec::new("python_function_files", FieldDefault::EmptyTuple), + FieldSpec::new("schema_files", FieldDefault::EmptyTuple), + FieldSpec::new("source_files", FieldDefault::EmptyTuple), + FieldSpec::new("seed_files", FieldDefault::EmptyTuple), + FieldSpec::new("test_files", FieldDefault::EmptyTuple), + FieldSpec::new("scenario_files", FieldDefault::EmptyTuple), + FieldSpec::new("audit_files", FieldDefault::EmptyTuple), + FieldSpec::new("macro_files", FieldDefault::EmptyTuple), + FieldSpec::new("materialization_files", FieldDefault::EmptyTuple), + FieldSpec::new("loader_functions", FieldDefault::EmptyTuple), + FieldSpec::new("task_functions", FieldDefault::EmptyTuple), + FieldSpec::new("asset_functions", FieldDefault::EmptyTuple), + FieldSpec::new("check_functions", FieldDefault::EmptyTuple), + FieldSpec::new("audit_factories", FieldDefault::EmptyTuple), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("event_exporters", FieldDefault::EmptyTuple), + FieldSpec::new("command_output_sinks", FieldDefault::EmptyTuple), + FieldSpec::new("providers", FieldDefault::EmptyTuple), + FieldSpec::new("adapter_file", FieldDefault::None), + FieldSpec { + name: "native_session", + default: FieldDefault::None, + compared: false, + shown: false, + }, + ], +}; +const R_DISCOVERED_RUNTIME_EXTENSIONS: RecordSpec = RecordSpec { + name: "DiscoveredRuntimeExtensions", + doc: "Command-startup providers, typed sinks, and lifecycle delivery settings.", + fields: &[ + FieldSpec::new("providers", FieldDefault::EmptyTuple), + FieldSpec::new("event_exporters", FieldDefault::EmptyTuple), + FieldSpec::new("command_output_sinks", FieldDefault::EmptyTuple), + FieldSpec::new("lifecycle_shutdown_timeout_seconds", FieldDefault::None), + ], +}; +const R_COMPILER_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "CompilerDiagnostic", + doc: "One project diagnostic produced by compile-time checks.", + fields: &[ + FieldSpec::required("phase"), + FieldSpec::required("severity"), + FieldSpec::required("code"), + FieldSpec::required("message"), + FieldSpec::new("resource_type", FieldDefault::None), + FieldSpec::new("resource_name", FieldDefault::None), + FieldSpec::new("column_name", FieldDefault::None), + FieldSpec::new("path", FieldDefault::None), + FieldSpec::new("line", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("location", FieldDefault::None), + FieldSpec::new("related_locations", FieldDefault::EmptyTuple), + FieldSpec::new("help", FieldDefault::None), + FieldSpec::new("notes", FieldDefault::EmptyTuple), + FieldSpec::new("affected_rules", FieldDefault::EmptyTuple), + ], +}; +const R_INFERRED_COLUMN: RecordSpec = RecordSpec { + name: "InferredColumn", + doc: "One output column inferred from query SQL via sql_analysis parsing.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("type", FieldDefault::None), + FieldSpec::new( + "nullability", + FieldDefault::Member( + "sqlbuild.compiler.lineage.types", + "InferredNullability", + "UNKNOWN", + ), + ), + ], +}; +const R_LOADED_MACRO: RecordSpec = RecordSpec { + name: "LoadedMacro", + doc: "One loaded project macro available for compile-time SQL expansion.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("raw_source"), + FieldSpec::required("function"), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + ], +}; +const R_STATIC_MACRO_EXPORT: RecordSpec = RecordSpec { + name: "StaticMacroExport", + doc: "Value-free AST inventory for one project-owned macro export.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("parameters"), + FieldSpec::required("line"), + FieldSpec::required("source_digest"), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + ], +}; +const R_STATIC_MACRO_FAULT: RecordSpec = RecordSpec { + name: "StaticMacroFault", + doc: "One value-free static macro inventory fault.", + fields: &[ + FieldSpec::required("relative_path"), + FieldSpec::required("message"), + ], +}; +const R_STATIC_MACRO_INVENTORY: RecordSpec = RecordSpec { + name: "StaticMacroInventory", + doc: "AST macro exports and faults collected without module execution.", + fields: &[ + FieldSpec::new("exports", FieldDefault::EmptyTuple), + FieldSpec::new("faults", FieldDefault::EmptyTuple), + ], +}; +const R_MACRO_CONTEXT: RecordSpec = RecordSpec { + name: "MacroContext", + doc: "Expose target settings and caller-visible declarations to a Python SQL macro.", + fields: &[ + FieldSpec::required("adapter_name"), + FieldSpec::required("sql_analysis_enabled"), + FieldSpec::required("target_name"), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("constants", FieldDefault::EmptyDict), + FieldSpec::new("enums", FieldDefault::EmptyDict), + FieldSpec { + name: "_value_renderer", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec { + name: "_collection_rendering", + default: FieldDefault::Member("sqlbuild._native", "CollectionRendering", "VALUE_LIST"), + compared: false, + shown: false, + }, + FieldSpec { + name: "_constant_declarations", + default: FieldDefault::EmptyDict, + compared: false, + shown: false, + }, + FieldSpec { + name: "_enforce_explicit_references", + default: FieldDefault::True, + compared: false, + shown: false, + }, + ], +}; +const R_DECLARATION_RESOLUTION_CONTEXT: RecordSpec = RecordSpec { + name: "DeclarationResolutionContext", + doc: "Declarations visible during one authored SQL expansion.", + fields: &[ + FieldSpec::new("enums", FieldDefault::EmptyDict), + FieldSpec::new("constants", FieldDefault::EmptyDict), + FieldSpec::new("inaccessible_enums", FieldDefault::EmptyDict), + FieldSpec::new("inaccessible_constants", FieldDefault::EmptyDict), + FieldSpec::new("enum_visibility", FieldDefault::EmptyDict), + FieldSpec::new("constant_visibility", FieldDefault::EmptyDict), + FieldSpec::new("macros", FieldDefault::EmptyDict), + FieldSpec::new("macro_records", FieldDefault::EmptyDict), + FieldSpec::new("macro_visibility", FieldDefault::EmptyDict), + FieldSpec::new("inaccessible_macros", FieldDefault::EmptyDict), + FieldSpec::new("consumer", FieldDefault::None), + ], +}; +const R_MODEL_INPUT_BUILD_CONTEXT: RecordSpec = RecordSpec { + name: "ModelInputBuildContext", + doc: "Run-constant config and macros for building model compile inputs.", + fields: &[ + FieldSpec::required("effective_vars"), + FieldSpec::required("effective_settings"), + FieldSpec::required("target_config"), + FieldSpec::required("effective_target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("macro_context"), + FieldSpec::required("loaded_macros"), + FieldSpec::required("value_renderer"), + FieldSpec::required("collection_rendering"), + FieldSpec::required("sql_lexical_syntax"), + FieldSpec::new("public_enums", FieldDefault::EmptyDict), + FieldSpec::new("public_constants", FieldDefault::EmptyDict), + FieldSpec::new("public_model_schemas", FieldDefault::EmptyDict), + FieldSpec::new("declaration_resolver", FieldDefault::None), + ], +}; +const R_COMPILE_SQL_REFERENCE: RecordSpec = RecordSpec { + name: "CompileSqlReference", + doc: "One logical SQL reference discovered from compiled SQL text.", + fields: &[ + FieldSpec::required("ref_kind"), + FieldSpec::required("ref_name"), + FieldSpec::new("ref_package", FieldDefault::None), + FieldSpec::new("call_argument_count", FieldDefault::None), + ], +}; +const R_COMPILED_LINEAGE_SOURCE_FACT: RecordSpec = RecordSpec { + name: "CompiledLineageSourceFact", + doc: "Compact upstream column fact extracted during SQL analysis.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("column_name"), + ], +}; +const R_COMPACT_LINEAGE_FACTS: RecordSpec = RecordSpec { + name: "CompactLineageFacts", + doc: "Indexed native lineage rows with lazy object projection.", + fields: &[ + FieldSpec::required("string_pool"), + FieldSpec::required("rows"), + FieldSpec::new("resource_name_indexes", FieldDefault::EmptyDict), + FieldSpec::new("resource_names", FieldDefault::EmptyDict), + ], +}; +const R_COMPILED_OBJECT_KEY: RecordSpec = RecordSpec { + name: "CompiledObjectKey", + doc: "Stable logical identity for one compiled resource or external dependency.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("name"), + ], +}; +const R_COMPILE_MODEL_INPUT: RecordSpec = RecordSpec { + name: "CompileModelInput", + doc: "One discovered model file with its attached schema metadata, if any.", + fields: &[ + FieldSpec::required("model_file"), + FieldSpec::new( + "config", + FieldDefault::Call("sqlbuild._native", "CompileModelConfig"), + ), + FieldSpec::new("query_sql", FieldDefault::Text("")), + FieldSpec::new("macro_source_sql", FieldDefault::Text("")), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("schema_entry", FieldDefault::None), + FieldSpec::new("schema_file", FieldDefault::None), + FieldSpec::new("sql_validation_enabled", FieldDefault::False), + FieldSpec::new("rejected_sql_analysis_opt_out", FieldDefault::None), + FieldSpec::new("enum_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("constant_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("enum_columns", FieldDefault::EmptyDict), + FieldSpec::new("macro_deps", FieldDefault::EmptyTuple), + FieldSpec::new("macro_usages", FieldDefault::EmptyTuple), + FieldSpec::new("declaration_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "sql_expansion", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec { + name: "lint_expansion", + default: FieldDefault::None, + compared: false, + shown: false, + }, + ], +}; +const R_COMPILED_RELATION_LOCATION: RecordSpec = RecordSpec { + name: "CompiledRelationLocation", + doc: "Logical and physical relation location resolved during compile.", + fields: &[ + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("name"), + FieldSpec::required("qualified_name"), + FieldSpec::new("logical_schema", FieldDefault::None), + FieldSpec::new("logical_database", FieldDefault::None), + ], +}; +const R_COMPILED_MODEL: RecordSpec = RecordSpec { + name: "CompiledModel", + doc: "Planner-ready compiled model metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("deps"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("query_sql"), + FieldSpec::required("config"), + FieldSpec::required("destination"), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("schema_entry", FieldDefault::None), + FieldSpec::new("inferred_columns", FieldDefault::None), + FieldSpec::new("fast_lineage_columns", FieldDefault::None), + FieldSpec::new("fast_lineage_has_star", FieldDefault::False), + FieldSpec::new("fast_lineage_star_resolved", FieldDefault::False), + FieldSpec::new("authored_sql", FieldDefault::Text("")), + FieldSpec::new("authored_query_sql", FieldDefault::Text("")), + FieldSpec::new("output_column_locations", FieldDefault::EmptyDict), + FieldSpec::new("extract_implicit_alias_columns", FieldDefault::True), + FieldSpec::new("macro_deps", FieldDefault::EmptyTuple), + FieldSpec::new("enum_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("constant_declarations", FieldDefault::EmptyTuple), + FieldSpec::new("enum_columns", FieldDefault::EmptyDict), + FieldSpec::new("binding_diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("binding_validated", FieldDefault::False), + FieldSpec::new("dynamic_column_contract", FieldDefault::None), + FieldSpec::new("unchecked_output_columns", FieldDefault::EmptyFrozenset), + FieldSpec::new("rejected_sql_analysis_opt_out", FieldDefault::None), + ], +}; +const R_COMPILED_AUDIT: RecordSpec = RecordSpec { + name: "CompiledAudit", + doc: "Compiled audit metadata selected by scope dependencies.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("scope_deps"), + FieldSpec::required("name"), + FieldSpec::required("definition_name"), + FieldSpec::required("audit_file"), + FieldSpec::required("audit_block"), + FieldSpec::required("sql_body"), + FieldSpec::new( + "evaluation_mode", + FieldDefault::Member( + "sqlbuild.compiler.auditing.types", + "AuditEvaluationMode", + "VIOLATIONS", + ), + ), + FieldSpec::new("measurement_contract", FieldDefault::None), + FieldSpec::new("thresholds", FieldDefault::None), + FieldSpec::new("minimum_samples", FieldDefault::None), + FieldSpec::new("measure_sql", FieldDefault::None), + FieldSpec::new("evidence_sql", FieldDefault::None), + FieldSpec::new("evidence_limit", FieldDefault::None), + FieldSpec::new("references", FieldDefault::EmptyTuple), + FieldSpec::new("attached_target_kind", FieldDefault::None), + FieldSpec::new("attached_target_name", FieldDefault::None), + FieldSpec::new("attached_column_name", FieldDefault::None), + FieldSpec::new("severity", FieldDefault::None), + FieldSpec::new("run_scope", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + FieldSpec::new("description", FieldDefault::None), + ], +}; +const R_COMPILED_SQL_SCENARIO: RecordSpec = RecordSpec { + name: "CompiledSqlScenario", + doc: "Compiled SQL-native scenario metadata selected by inferred graph targets.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("scenario_file"), + FieldSpec::required("sql_body"), + FieldSpec::new("authored_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("expected_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("source_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("source_path", FieldDefault::None), + FieldSpec::new("ownership_root", FieldDefault::None), + ], +}; +const R_COMPILED_PROJECT: RecordSpec = RecordSpec { + name: "CompiledProject", + doc: "Planner-ready whole-project compile output.", + fields: &[ + FieldSpec::required("run_id"), + FieldSpec::required("effective_target_name"), + FieldSpec::required("effective_connection"), + FieldSpec::required("effective_vars"), + FieldSpec { + name: "binding_catalog", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec::new("effective_target_database", FieldDefault::None), + FieldSpec::new("effective_target_schema", FieldDefault::None), + FieldSpec::new("sql_analysis_dialect", FieldDefault::None), + FieldSpec::new( + "sql_lexical_syntax", + FieldDefault::Call("sqlbuild._native", "SqlLexicalSyntax"), + ), + FieldSpec::new("compile_cache_dir", FieldDefault::None), + FieldSpec::new( + "settings", + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), + ), + FieldSpec::new( + "scenario", + FieldDefault::Call("sqlbuild._native", "ScenarioConfig"), + ), + FieldSpec::new("models", FieldDefault::EmptyTuple), + FieldSpec::new("sources", FieldDefault::EmptyTuple), + FieldSpec::new("seeds", FieldDefault::EmptyTuple), + FieldSpec::new("functions", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + FieldSpec::new("sql_tests", FieldDefault::EmptyTuple), + FieldSpec::new("sql_scenarios", FieldDefault::EmptyTuple), + FieldSpec::new("loader_functions", FieldDefault::EmptyTuple), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("enforce_explicit_references", FieldDefault::True), + FieldSpec::new("sql_hook_files", FieldDefault::EmptyTuple), + FieldSpec::new("materialization_files", FieldDefault::EmptyTuple), + FieldSpec::new("public_enums", FieldDefault::EmptyDict), + FieldSpec::new("public_constants", FieldDefault::EmptyDict), + FieldSpec::new("loaded_macros", FieldDefault::EmptyDict), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("unmatched_literal_sql_relations", FieldDefault::EmptyTuple), + FieldSpec::new("external_sql_reference_resolver", FieldDefault::None), + FieldSpec::new( + "scope_index", + FieldDefault::Call("sqlbuild._native", "ScopeIndex"), + ), + FieldSpec { + name: "sql_expansions", + default: FieldDefault::EmptyDict, + compared: false, + shown: false, + }, + FieldSpec { + name: "native_session", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec { + name: "native_project", + default: FieldDefault::None, + compared: false, + shown: false, + }, + ], +}; +const R_COMPILED_SQL_TEST: RecordSpec = RecordSpec { + name: "CompiledSqlTest", + doc: "Compiled SQL-native unit test metadata selected by expected targets.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("scope_deps"), + FieldSpec::required("name"), + FieldSpec::required("test_file"), + FieldSpec::required("test_block"), + FieldSpec::required("sql_body"), + FieldSpec::new( + "mode", + FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + ), + FieldSpec::new( + "payload", + FieldDefault::Call("sqlbuild._native", "CompiledModelSqlTestPayload"), + ), + FieldSpec::new("source_path", FieldDefault::None), + FieldSpec::new("ownership_root", FieldDefault::None), + FieldSpec::new("block_index", FieldDefault::None), + FieldSpec::new("explicit_name", FieldDefault::None), + FieldSpec::new("parent_name", FieldDefault::None), + FieldSpec::new("case_name", FieldDefault::None), + FieldSpec::new("case_index", FieldDefault::None), + FieldSpec::new("case_fingerprint", FieldDefault::None), + FieldSpec::new("parameter_schema", FieldDefault::EmptyTuple), + FieldSpec::new("parameter_values", FieldDefault::EmptyTuple), + FieldSpec::new("expected_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("read_helper_names", FieldDefault::EmptyTuple), + FieldSpec::new("reference_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("tested_resources", FieldDefault::EmptyTuple), + ], +}; -/// Compile and discovery value records. +/// The compile and discovery value records. pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ R_RELATED_LOCATION, R_FIXTURE_COLUMN_INFERENCE, @@ -860,4 +1673,49 @@ pub(crate) const COMPILE_VALUE_RECORDS: &[RecordSpec] = &[ R_DISCOVERED_ADAPTER_FILE, R_DISCOVERY_FILE_FAULT, R_TOLERANT_SCOPE_DISCOVERY, + R_SQL_HOOK_ENTRY, + R_PYTHON_HOOK_ENTRY, + R_ENUM_DECLARATION, + R_CONSTANT_DECLARATION, + R_DISCOVERED_SOURCE_FILE, + R_DISCOVERED_SQL_TEST_FILE, + R_SQL_TEST_PARAMETER_DECLARATION, + R_DISCOVERED_SQL_SCENARIO_FILE, + R_DISCOVERED_MACRO_FILE, + R_DISCOVERED_MATERIALIZATION_FILE, + R_DISCOVERED_PROVIDER_USAGE, + R_DISCOVERED_LOADER_FUNCTION, + R_DISCOVERED_TASK_FUNCTION, + R_DISCOVERED_ASSET_FUNCTION, + R_DISCOVERED_CHECK_FUNCTION, + R_DISCOVERED_HOOK_FUNCTION, + R_DISCOVERED_AUDIT_FACTORY, + R_DISCOVERED_EVENT_EXPORTER_DECLARATION, + R_DISCOVERED_EVENT_EXPORTER, + R_DISCOVERED_COMMAND_OUTPUT_SINK_DECLARATION, + R_DISCOVERED_COMMAND_OUTPUT_SINK, + R_DISCOVERED_PROVIDER, + R_DISCOVERED_PYTHON_NODE_FUNCTIONS, + R_DISCOVERED_PROJECT_INPUTS, + R_DISCOVERED_RUNTIME_EXTENSIONS, + R_COMPILER_DIAGNOSTIC, + R_INFERRED_COLUMN, + R_LOADED_MACRO, + R_STATIC_MACRO_EXPORT, + R_STATIC_MACRO_FAULT, + R_STATIC_MACRO_INVENTORY, + R_MACRO_CONTEXT, + R_DECLARATION_RESOLUTION_CONTEXT, + R_MODEL_INPUT_BUILD_CONTEXT, + R_COMPILE_SQL_REFERENCE, + R_COMPILED_LINEAGE_SOURCE_FACT, + R_COMPACT_LINEAGE_FACTS, + R_COMPILED_OBJECT_KEY, + R_COMPILE_MODEL_INPUT, + R_COMPILED_RELATION_LOCATION, + R_COMPILED_MODEL, + R_COMPILED_AUDIT, + R_COMPILED_SQL_SCENARIO, + R_COMPILED_PROJECT, + R_COMPILED_SQL_TEST, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 5f9f2b902b..ec2abe5bcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -28,13 +28,13 @@ use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGOR const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PYTHON_RESOURCES_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index dfd411e73c..fc7c9d8483 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -22,6 +22,9 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ CompiledViewClasses, InferredRow, }; use crate::bindings::_helpers::compiled_project::model_input_view::ModelInputRecord; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + confidence_member, transform_kind_member, +}; const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; @@ -296,15 +299,11 @@ fn lineage_facts(classes: &CompiledViewClasses<'_>, rows: &[LineageRow]) -> PyRe fields.set_item("upstream_columns", PyTuple::new(py, sources)?)?; fields.set_item( "transform_kind", - classes - .compact_lineage_facts - .call_method1("transform_kind", (row.transform_code,))?, + transform_kind_member(py, usize::from(row.transform_code))?, )?; fields.set_item( "confidence", - classes - .compact_lineage_facts - .call_method1("confidence", (row.confidence_code,))?, + confidence_member(py, usize::from(row.confidence_code))?, )?; facts.push( classes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index 0c5ed72558..083f4d9d8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -8,19 +8,19 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The materializations a model's storage policies are inherited for, besides none. const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index e286a8f9a0..846deceaae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs new file mode 100644 index 0000000000..b9f26149d2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -0,0 +1,780 @@ +//! The compiler, rule, lint and config string enums hosted natively. + +use crate::bindings::_helpers::records::models::EnumSpec; + +const E_CONTRACT_FINDING_KIND: EnumSpec = EnumSpec { + name: "ContractFindingKind", + doc: "Stable physical-to-authored contract difference categories.", + members: &[ + ("MISSING_DECLARATION", "missing_declaration"), + ("MISSING_PHYSICAL_COLUMN", "missing_physical_column"), + ("TYPE_MISMATCH", "type_mismatch"), + ("RELATION_MISSING", "relation_missing"), + ("INSPECTION_UNAVAILABLE", "inspection_unavailable"), + ("OWNERSHIP_CONFLICT", "ownership_conflict"), + ], +}; +const E_COMPILER_STAGE: EnumSpec = EnumSpec { + name: "CompilerStage", + doc: "Frontier objects handed from one compiler stage to the next, in pipeline order.", + members: &[ + ("DISCOVERED_PROJECT_INPUTS", "discovered_project_inputs"), + ("COMPILE_PROJECT_INPUTS", "compile_project_inputs"), + ("COMPILED_PROJECT", "compiled_project"), + ], +}; +const E_HOOK_READ_TYPE: EnumSpec = EnumSpec { + name: "HookReadType", + doc: "Kind of model hook that declares or writes a read.", + members: &[ + ("PYTHON", "Python"), + ("SQL", "SQL"), + ("INLINE_SQL", "inline SQL"), + ], +}; +const E_LINT_SEVERITY: EnumSpec = EnumSpec { + name: "LintSeverity", + doc: "Severity assigned to one lint violation.", + members: &[("FAULT", "fault"), ("WARNING", "warning")], +}; +const E_DIAGNOSTIC_PHASE: EnumSpec = EnumSpec { + name: "DiagnosticPhase", + doc: "Phase that produced a compiler diagnostic.", + members: &[ + ("COMPILE", "compile"), + ("CONTRACT", "contract"), + ("RULE", "rule"), + ("PLAN", "plan"), + ("BUILD", "build"), + ("AUDIT", "audit"), + ("TEST", "test"), + ("CONNECTION", "connection"), + ], +}; +const E_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { + name: "DiagnosticSeverity", + doc: "Severity for a compiler diagnostic.", + members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], +}; +const E_COLUMN_TRANSFORM_KIND: EnumSpec = EnumSpec { + name: "ColumnTransformKind", + doc: "High-level transform classification for one output column.", + members: &[ + ("DIRECT", "direct"), + ("CAST", "cast"), + ("EXPRESSION", "expression"), + ("AGGREGATION", "aggregation"), + ("STAR", "star"), + ("CONSTANT", "constant"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { + name: "ColumnLineageConfidence", + doc: "Coarse confidence that a lineage edge is fully understood.", + members: &[ + ("HIGH", "high"), + ("MEDIUM", "medium"), + ("LOW", "low"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLUMN_LINEAGE_MODE: EnumSpec = EnumSpec { + name: "ColumnLineageMode", + doc: "Column lineage analyzer mode.", + members: &[("RICH", "rich"), ("FAST", "fast")], +}; +const E_POLYGLOT_ANALYSIS_DIALECT: EnumSpec = EnumSpec { + name: "PolyglotAnalysisDialect", + doc: "Dialect names accepted by Polyglot analyze_query.", + members: &[ + ("GENERIC", "generic"), + ("POSTGRESQL", "postgresql"), + ("MYSQL", "mysql"), + ("BIGQUERY", "bigquery"), + ("SNOWFLAKE", "snowflake"), + ("DUCKDB", "duckdb"), + ("SQLITE", "sqlite"), + ("HIVE", "hive"), + ("SPARK", "spark"), + ("TRINO", "trino"), + ("PRESTO", "presto"), + ("REDSHIFT", "redshift"), + ("TSQL", "tsql"), + ("ORACLE", "oracle"), + ("CLICKHOUSE", "clickhouse"), + ("DATABRICKS", "databricks"), + ("ATHENA", "athena"), + ("TERADATA", "teradata"), + ("DORIS", "doris"), + ("STARROCKS", "starrocks"), + ("MATERIALIZE", "materialize"), + ("RISINGWAVE", "risingwave"), + ("SINGLESTORE", "singlestore"), + ("COCKROACHDB", "cockroachdb"), + ("TIDB", "tidb"), + ("DRUID", "druid"), + ("SOLR", "solr"), + ("TABLEAU", "tableau"), + ("DUNE", "dune"), + ("FABRIC", "fabric"), + ("DRILL", "drill"), + ("DREMIO", "dremio"), + ("EXASOL", "exasol"), + ("DATAFUSION", "datafusion"), + ], +}; +const E_MIGRATION_DISCOVERY: EnumSpec = EnumSpec { + name: "MigrationDiscovery", + doc: "How a migration was requested.", + members: &[("MANUAL", "manual"), ("AUTOMATIC", "automatic")], +}; +const E_MIGRATION_COMPATIBILITY: EnumSpec = EnumSpec { + name: "MigrationCompatibility", + doc: "Outcome of checking the origin relation against the destination model.", + members: &[ + ("COMPATIBLE", "compatible"), + ("INCOMPATIBLE", "incompatible"), + ("NOT_CHECKED", "not_checked"), + ], +}; +const E_OLD_NAME_VIEW_ACTION: EnumSpec = EnumSpec { + name: "OldNameViewAction", + doc: "What a build does at a migrated model's old name.", + members: &[ + ("ARCHIVE_AND_VIEW", "archive_and_view"), + ("VIEW_ONLY", "view_only"), + ("LIVE", "live"), + ("NONE", "none"), + ], +}; +const E_SELECTOR_KIND: EnumSpec = EnumSpec { + name: "SelectorKind", + doc: "", + members: &[ + ("NAME", "name"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("TASK", "task"), + ("ASSET", "asset"), + ("LOADER", "loader"), + ("CHECK", "check"), + ("TAG", "tag"), + ("PATH", "path"), + ("TEST", "test"), + ], +}; +const E_PYTHON_RUN_PHASE: EnumSpec = EnumSpec { + name: "PythonRunPhase", + doc: "Lifecycle-aware Python execution phase.", + members: &[ + ("PRE_SQL_INGRESS", "pre_sql_ingress"), + ("READ_SIDE", "read_side"), + ], +}; +const E_PYTHON_IDENTITY_STATUS: EnumSpec = EnumSpec { + name: "PythonIdentityStatus", + doc: "Read-only identity diff status for Python plan display.", + members: &[ + ("NEW", "new"), + ("CHANGED", "changed"), + ("UNCHANGED", "unchanged"), + ("UNKNOWN", "unknown"), + ], +}; +const E_COLLECTION_RENDERING: EnumSpec = EnumSpec { + name: "CollectionRendering", + doc: "Physical rendering choices for homogeneous SQL collections.", + members: &[("VALUE_LIST", "value_list"), ("ARRAY", "array")], +}; +const E_RESOURCE_KIND: EnumSpec = EnumSpec { + name: "ResourceKind", + doc: "Kinds of authored resources represented in a scope index.", + members: &[ + ("MODEL", "model"), + ("TEST", "test"), + ("SCENARIO", "scenario"), + ("FUNCTION", "function"), + ("SOURCE", "source"), + ("SEED", "seed"), + ], +}; +const E_DECLARATION_KIND: EnumSpec = EnumSpec { + name: "DeclarationKind", + doc: "Independent public declaration namespaces.", + members: &[ + ("MACRO", "macro"), + ("ENUM", "enum"), + ("CONSTANT", "constant"), + ("AUDIT", "audit"), + ("SINGULAR_AUDIT", "singular_audit"), + ("SCHEMA", "schema"), + ("SQL_HOOK", "sql_hook"), + ("PYTHON_HOOK", "python_hook"), + ], +}; +const E_SCOPE_KIND: EnumSpec = EnumSpec { + name: "ScopeKind", + doc: "Compiler-enforced declaration visibility tiers.", + members: &[ + ("GLOBAL", "global"), + ("INHERITED", "inherited"), + ("LOCAL", "local"), + ("PRIVATE", "private"), + ], +}; +const E_OWNERSHIP_ROOT_KIND: EnumSpec = EnumSpec { + name: "OwnershipRootKind", + doc: "How an ownership root participates in authored resource scope.", + members: &[("RESOURCE", "resource"), ("GLOBAL", "global")], +}; +const E_USAGE_KIND: EnumSpec = EnumSpec { + name: "UsageKind", + doc: "Why one indexed identity consumes a declaration.", + members: &[ + ("RUNTIME", "runtime"), + ("GENERATED", "generated"), + ("DECLARATION_DEPENDENCY", "declaration_dependency"), + ], +}; +const E_GRANT_KIND: EnumSpec = EnumSpec { + name: "GrantKind", + doc: "Compiler relationships that grant additional public vocabulary.", + members: &[ + ("EXPECTED_MODEL", "expected_model"), + ("TESTED_MACRO", "tested_macro"), + ], +}; +const E_VISIBILITY_REASON: EnumSpec = EnumSpec { + name: "VisibilityReason", + doc: "Positive explanations emitted by the canonical resolver.", + members: &[ + ("GLOBAL", "global"), + ("INHERITED_ANCESTOR", "inherited_ancestor"), + ("LOCAL_OWNER", "local_owner"), + ("PRIVATE_OWNER", "private_owner"), + ("EXPECTED_MODEL", "expected_model"), + ("TESTED_MACRO", "tested_macro"), + ], +}; +const E_INACCESSIBLE_REASON: EnumSpec = EnumSpec { + name: "InaccessibleReason", + doc: "Stable explanations for declarations outside effective scope.", + members: &[ + ("LOCAL_BOUNDARY", "local_boundary"), + ("SIBLING_SCOPE", "sibling_scope"), + ("DESCENDANT_SCOPE", "descendant_scope"), + ("UNRELATED_SCOPE", "unrelated_scope"), + ("PRIVATE_OWNER", "private_owner"), + ("UNSUPPORTED_RESOURCE_KIND", "unsupported_resource_kind"), + ], +}; +const E_SCOPE_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { + name: "ScopeDiagnosticSeverity", + doc: "Severity of a stable scope diagnostic.", + members: &[("ERROR", "error"), ("WARNING", "warning")], +}; +const E_SCOPE_DIAGNOSTIC_CODE: EnumSpec = EnumSpec { + name: "ScopeDiagnosticCode", + doc: "Stable diagnostic identities shared by future scope consumers.", + members: &[ + ("UNKNOWN_TARGET", "S001"), + ("UNQUALIFIED_TARGET", "S002"), + ("DUPLICATE_DECLARATION", "S003"), + ("INVALID_DECLARATION_NAME", "S004"), + ("RESERVED_DECLARATION_NAME", "S005"), + ("INACCESSIBLE_DECLARATION", "S006"), + ("LOCAL_NEEDED_BY_DESCENDANT", "S007"), + ("OVER_BROAD_INHERITED", "S008"), + ("REQUIRES_GLOBAL_PLACEMENT", "S009"), + ("UNUSED_DECLARATION", "S010"), + ("INVALID_PROSPECTIVE_PATH", "S011"), + ("NESTED_DECLARATION_ROOT", "S012"), + ("INCOMPLETE_USAGE", "S013"), + ("INVALID_CURSOR", "S014"), + ("MACRO_MODULE_IMPORT", "S015"), + ("INVALID_MACRO_DEPENDENCY", "S016"), + ("MACRO_DEPENDENCY_CYCLE", "S017"), + ("DUPLICATE_RESOURCE", "S018"), + ("RESOURCE_PARSE_ERROR", "S019"), + ("DECLARATION_PARSE_ERROR", "S020"), + ("MACRO_PARSE_ERROR", "S021"), + ("RELATIONSHIP_PARSE_ERROR", "S022"), + ("CONFIG_PARSE_ERROR", "S023"), + ("OVER_BROAD_GLOBAL", "S024"), + ], +}; +const E_EVENT_EXPORT_SEVERITY: EnumSpec = EnumSpec { + name: "EventExportSeverity", + doc: "Severity used to filter exported lifecycle events.", + members: &[ + ("DEBUG", "debug"), + ("INFO", "info"), + ("WARNING", "warning"), + ("ERROR", "error"), + ], +}; +const E_WAREHOUSE_GROUP: EnumSpec = EnumSpec { + name: "WarehouseGroup", + doc: "Command group that selects a target's default warehouse.", + members: &[("BUILD", "build"), ("QUERY", "query")], +}; +const E_SOURCE_WRITE_STRATEGY: EnumSpec = EnumSpec { + name: "SourceWriteStrategy", + doc: "", + members: &[ + ("APPEND", "append"), + ("DELETE_INSERT", "delete_insert"), + ("MERGE", "merge"), + ("TABLE", "table"), + ], +}; +const E_FUTURE_CURSOR_ACTION: EnumSpec = EnumSpec { + name: "FutureCursorAction", + doc: "Response when an effective cursor lies beyond the future horizon.", + members: &[("CAP", "cap"), ("ERROR", "error")], +}; +const E_MICROBATCH_LIMIT_ACTION: EnumSpec = EnumSpec { + name: "MicrobatchLimitAction", + doc: "Response when one model plans more than the configured batch limit.", + members: &[ + ("CAP_FROM_END", "cap_from_end"), + ("CAP_FROM_START", "cap_from_start"), + ("ERROR", "error"), + ("WARN", "warn"), + ], +}; +const E_COLUMN_CONTRACT_MODE: EnumSpec = EnumSpec { + name: "ColumnContractMode", + doc: "Default contract behavior for model column declarations.", + members: &[("IMPLICIT", "implicit"), ("EXPLICIT", "explicit")], +}; +const E_SOURCE_FRESHNESS_STRATEGY: EnumSpec = EnumSpec { + name: "SourceFreshnessStrategy", + doc: "", + members: &[("ADAPTER", "adapter"), ("COLUMN", "column"), ("SQL", "sql")], +}; +const E_SOURCE_FRESHNESS_VALUE_KIND: EnumSpec = EnumSpec { + name: "SourceFreshnessValueKind", + doc: "", + members: &[ + ("TIMESTAMP", "timestamp"), + ("INTEGER", "integer"), + ("STRING", "string"), + ], +}; +const E_TIME_TRAVEL_RETENTION_SOURCE: EnumSpec = EnumSpec { + name: "TimeTravelRetentionSource", + doc: "Authored layer that supplied the effective retention policy.", + members: &[ + ("TARGET", "target"), + ("MATERIALIZATION", "materialization"), + ("MODEL", "model"), + ], +}; +const E_TABLE_TYPE: EnumSpec = EnumSpec { + name: "TableType", + doc: "Snowflake managed table type.", + members: &[("PERMANENT", "permanent"), ("TRANSIENT", "transient")], +}; +const E_TABLE_TYPE_SOURCE: EnumSpec = EnumSpec { + name: "TableTypeSource", + doc: "Authored layer that supplied the effective table type.", + members: &[ + ("DEFAULT", "default"), + ("TARGET", "target"), + ("MATERIALIZATION", "materialization"), + ("MODEL", "model"), + ], +}; +const E_TABLE_TYPE_DOWNGRADE_POLICY: EnumSpec = EnumSpec { + name: "TableTypeDowngradePolicy", + doc: "Permanent-to-transient conversion policy.", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("ALLOW", "allow"), + ], +}; +const E_MISSING_MIGRATION_ORIGIN_POLICY: EnumSpec = EnumSpec { + name: "MissingMigrationOriginPolicy", + doc: "Policy for a declared migrate_from whose origin does not exist in a target.", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("ALLOW", "allow"), + ], +}; +const E_RETENTION_DECREASE_POLICY: EnumSpec = EnumSpec { + name: "RetentionDecreasePolicy", + doc: "Policy for lowering live time-travel retention.", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("ALLOW", "allow"), + ], +}; +const E_SQL_VALUE_KIND: EnumSpec = EnumSpec { + name: "SqlValueKind", + doc: "Closed logical kinds accepted by the typed SQL value boundary.", + members: &[ + ("STRING", "string"), + ("INTEGER", "integer"), + ("BOOLEAN", "boolean"), + ("FLOAT", "float"), + ("DECIMAL", "decimal"), + ("NULL", "null"), + ("LIST", "list"), + ("SET", "set"), + ("OBJECT", "object"), + ], +}; +const E_STATE_SQL_VALUE_TYPE: EnumSpec = EnumSpec { + name: "StateSqlValueType", + doc: "Closed physical value kinds used by warehouse state columns.", + members: &[ + ("STRING", "string"), + ("INTEGER", "integer"), + ("BOOLEAN", "boolean"), + ("TIMESTAMP", "timestamp"), + ("TEXT_TIMESTAMP", "text_timestamp"), + ("DATE", "date"), + ("JSON", "json"), + ], +}; +const E_MIGRATION_DECISION: EnumSpec = EnumSpec { + name: "MigrationDecision", + doc: "Per-run decision for one declared or discovered model migration.", + members: &[ + ("DONE", "done"), + ("MIGRATE", "migrate"), + ("REDO", "redo"), + ("SUPERSEDED_REPLACE", "superseded_replace"), + ("FORCED_REPLACE", "forced_replace"), + ("CONFLICT", "conflict"), + ("ORIGIN_MISSING", "origin_missing"), + ("RENAMED", "renamed"), + ], +}; +const E_MIGRATION_PROMOTION: EnumSpec = EnumSpec { + name: "MigrationPromotion", + doc: "How a completed migration stage replaces or creates the destination.", + members: &[ + ("SWAP", "swap"), + ("RENAME", "rename"), + ("TRANSACTIONAL_RENAME", "transactional_rename"), + ], +}; +const E_COLUMN_MIGRATION_DECISION: EnumSpec = EnumSpec { + name: "ColumnMigrationDecision", + doc: "Per-run decision for one declared or detected column rename.", + members: &[ + ("RENAME", "rename"), + ("RECORD", "record"), + ("DONE", "done"), + ("CONFLICT", "conflict"), + ("SOURCE_MISSING", "source_missing"), + ("STILL_PRODUCED", "still_produced"), + ("UNSUPPORTED", "unsupported"), + ], +}; +const E_OLD_NAME_VIEW_EVENT_TYPE: EnumSpec = EnumSpec { + name: "OldNameViewEventType", + doc: "Immutable facts about what happened at a migrated relation's old name.", + members: &[ + ("REQUIRED", "required"), + ("ORIGIN_ARCHIVED", "origin_archived"), + ("VIEW_CREATED", "view_created"), + ("VIEW_DROPPED", "view_dropped"), + ], +}; +const E_OLD_NAME_VIEW_DROP_REASON: EnumSpec = EnumSpec { + name: "OldNameViewDropReason", + doc: "Why a compatibility view at an old name was dropped.", + members: &[ + ("EXPIRED", "expired"), + ("EARLY", "early"), + ("MISSING", "missing"), + ], +}; +const E_OLD_NAME_VIEW_STATUS: EnumSpec = EnumSpec { + name: "OldNameViewStatus", + doc: "Projected state of the old-name steps of one recorded move.", + members: &[ + ("PENDING_ARCHIVE", "pending_archive"), + ("PENDING_VIEW", "pending_view"), + ("LIVE", "live"), + ("EXPIRED", "expired"), + ("DROPPED", "dropped"), + ], +}; +const E_BACKFILL_ACTION: EnumSpec = EnumSpec { + name: "BackfillAction", + doc: "", + members: &[ + ("FULL", "full"), + ("BOUNDED", "bounded"), + ("FORWARD_ONLY", "forward"), + ], +}; +const E_ON_SCHEMA_CHANGE: EnumSpec = EnumSpec { + name: "OnSchemaChange", + doc: "", + members: &[ + ("IGNORE", "ignore"), + ("FAIL", "fail"), + ("APPEND_NEW_COLUMNS", "append_new_columns"), + ("SYNC_ALL_COLUMNS", "sync_all_columns"), + ], +}; +const E_CONTRACT_POLICY: EnumSpec = EnumSpec { + name: "ContractPolicy", + doc: "", + members: &[("NONE", "none"), ("ENFORCED", "enforced")], +}; +const E_RETENTION_DIRECTION: EnumSpec = EnumSpec { + name: "RetentionDirection", + doc: "", + members: &[ + ("INCREASE", "increase"), + ("DECREASE", "decrease"), + ("MIXED", "mixed"), + ("MATCH", "match"), + ("APPLY_AFTER_CREATE", "apply_after_create"), + ], +}; +const E_RETENTION_PLAN_PHASE: EnumSpec = EnumSpec { + name: "RetentionPlanPhase", + doc: "", + members: &[ + ("PRE", "pre"), + ("POST", "post"), + ("AFTER_CREATE", "after_create"), + ("NONE", "none"), + ], +}; +const E_PLAN_ACTION: EnumSpec = EnumSpec { + name: "PlanAction", + doc: "", + members: &[ + ("CREATE_VIEW", "create_view"), + ("CREATE_TABLE", "create_table"), + ("INCREMENTAL_APPEND", "incremental_append"), + ("INCREMENTAL_DELETE_INSERT", "incremental_delete_insert"), + ("INCREMENTAL_MERGE", "incremental_merge"), + ("SNAPSHOT", "snapshot"), + ("LOAD_SEED", "load_seed"), + ("SKIP", "skip"), + ("CUSTOM", "custom"), + ], +}; +const E_PLAN_REASON: EnumSpec = EnumSpec { + name: "PlanReason", + doc: "", + members: &[ + ("FIRST_RUN", "first_run"), + ("RENAMED", "renamed"), + ("FULL_REFRESH", "full_refresh"), + ("QUERY_CHANGED", "query_changed"), + ("FUNCTION_CHANGED", "function_changed"), + ("CONFIG_CHANGED", "config_changed"), + ("SCHEMA_CHANGED", "schema_changed"), + ("NORMAL_INCREMENTAL", "normal_incremental"), + ("NO_CHANGE", "no_change"), + ("DISABLED", "disabled"), + ("SOURCE_FRESHNESS_ERROR", "source_freshness_error"), + ("EXTERNAL_UPSTREAM_FAILED", "external_upstream_failed"), + ], +}; +const E_INCREMENTAL_STRATEGY: EnumSpec = EnumSpec { + name: "IncrementalStrategy", + doc: "", + members: &[ + ("APPEND", "append"), + ("DELETE_INSERT", "delete_insert"), + ("MERGE", "merge"), + ], +}; +const E_MICROBATCH_STRATEGY: EnumSpec = EnumSpec { + name: "MicrobatchStrategy", + doc: "How an ordinary microbatch range is selected.", + members: &[ + ("ROLLING_WINDOW", "rolling_window"), + ("WATERMARK", "watermark"), + ], +}; +const E_INCREMENTAL_MODE: EnumSpec = EnumSpec { + name: "IncrementalMode", + doc: "", + members: &[("FULL", "full"), ("MICROBATCH", "microbatch")], +}; +const E_SNAPSHOT_STRATEGY: EnumSpec = EnumSpec { + name: "SnapshotStrategy", + doc: "", + members: &[("TIMESTAMP", "timestamp"), ("CHECK", "check")], +}; +const E_HISTORICAL_INPUT: EnumSpec = EnumSpec { + name: "HistoricalInput", + doc: "", + members: &[("SNAPSHOT", "snapshot"), ("CHANGES", "changes")], +}; +const E_INITIAL_VALID_FROM: EnumSpec = EnumSpec { + name: "InitialValidFrom", + doc: "", + members: &[ + ("UPDATED_AT", "updated_at"), + ("OBSERVED_AT", "observed_at"), + ("EXECUTION_TIME", "execution_time"), + ], +}; +const E_SNAPSHOT_SCHEMA_CHANGE_POLICY: EnumSpec = EnumSpec { + name: "SnapshotSchemaChangePolicy", + doc: "", + members: &[ + ("DENY", "deny"), + ("REQUIRE_CONFIRMATION", "require_confirmation"), + ("APPEND_NEW_COLUMNS", "append_new_columns"), + ], +}; +const E_CURSOR_TYPE: EnumSpec = EnumSpec { + name: "CursorType", + doc: "", + members: &[("TIMESTAMP", "timestamp"), ("INTEGER", "integer")], +}; +const E_CURSOR_GRAIN: EnumSpec = EnumSpec { + name: "CursorGrain", + doc: "", + members: &[ + ("SECOND", "second"), + ("MINUTE", "minute"), + ("HOUR", "hour"), + ("DAY", "day"), + ("MONTH", "month"), + ("YEAR", "year"), + ], +}; +const E_MATERIALIZATION_TYPE: EnumSpec = EnumSpec { + name: "MaterializationType", + doc: "", + members: &[ + ("VIEW", "view"), + ("TABLE", "table"), + ("INCREMENTAL", "incremental"), + ("SNAPSHOT", "snapshot"), + ("SEED", "seed"), + ("CUSTOM", "custom"), + ], +}; +const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { + name: "ScenarioArtifactKind", + doc: "", + members: &[ + ("SOURCE", "source"), + ("REF", "ref"), + ("SEED", "seed"), + ("DBT_REF", "dbt_ref"), + ("MODEL", "model"), + ], +}; +const E_SQL_REFERENCE_KIND: EnumSpec = EnumSpec { + name: "SqlReferenceKind", + doc: "", + members: &[ + ("REF", "ref"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("DBT_REF", "dbt_ref"), + ("UDF", "udf"), + ("TABLE_FUNCTION", "table_fn"), + ], +}; +const E_HARD_CODED_RELATION_OWNER_KIND: EnumSpec = EnumSpec { + name: "HardCodedRelationOwnerKind", + doc: "The kind of Python code that names a project relation in SQL.", + members: &[("NODE", "node"), ("HOOK", "hook"), ("LOADER", "loader")], +}; +const E_LOADER_CONNECTION_MODE: EnumSpec = EnumSpec { + name: "LoaderConnectionMode", + doc: "How a source loader owns the destination warehouse connection.", + members: &[("SQLBUILD", "sqlbuild"), ("EXTERNAL", "external")], +}; +const E_RULE_SUBJECT: EnumSpec = EnumSpec { + name: "RuleSubject", + doc: "Compiler-owned unit used to invoke and cache one custom rule.", + members: &[("MODEL", "model"), ("PROJECT", "project")], +}; + +/// Compiler, rule, lint and config string enums. +pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ + E_CONTRACT_FINDING_KIND, + E_COMPILER_STAGE, + E_HOOK_READ_TYPE, + E_LINT_SEVERITY, + E_DIAGNOSTIC_PHASE, + E_DIAGNOSTIC_SEVERITY, + E_COLUMN_TRANSFORM_KIND, + E_COLUMN_LINEAGE_CONFIDENCE, + E_COLUMN_LINEAGE_MODE, + E_POLYGLOT_ANALYSIS_DIALECT, + E_MIGRATION_DISCOVERY, + E_MIGRATION_COMPATIBILITY, + E_OLD_NAME_VIEW_ACTION, + E_SELECTOR_KIND, + E_PYTHON_RUN_PHASE, + E_PYTHON_IDENTITY_STATUS, + E_COLLECTION_RENDERING, + E_RESOURCE_KIND, + E_DECLARATION_KIND, + E_SCOPE_KIND, + E_OWNERSHIP_ROOT_KIND, + E_USAGE_KIND, + E_GRANT_KIND, + E_VISIBILITY_REASON, + E_INACCESSIBLE_REASON, + E_SCOPE_DIAGNOSTIC_SEVERITY, + E_SCOPE_DIAGNOSTIC_CODE, + E_EVENT_EXPORT_SEVERITY, + E_WAREHOUSE_GROUP, + E_SOURCE_WRITE_STRATEGY, + E_FUTURE_CURSOR_ACTION, + E_MICROBATCH_LIMIT_ACTION, + E_COLUMN_CONTRACT_MODE, + E_SOURCE_FRESHNESS_STRATEGY, + E_SOURCE_FRESHNESS_VALUE_KIND, + E_TIME_TRAVEL_RETENTION_SOURCE, + E_TABLE_TYPE, + E_TABLE_TYPE_SOURCE, + E_TABLE_TYPE_DOWNGRADE_POLICY, + E_MISSING_MIGRATION_ORIGIN_POLICY, + E_RETENTION_DECREASE_POLICY, + E_SQL_VALUE_KIND, + E_STATE_SQL_VALUE_TYPE, + E_MIGRATION_DECISION, + E_MIGRATION_PROMOTION, + E_COLUMN_MIGRATION_DECISION, + E_OLD_NAME_VIEW_EVENT_TYPE, + E_OLD_NAME_VIEW_DROP_REASON, + E_OLD_NAME_VIEW_STATUS, + E_BACKFILL_ACTION, + E_ON_SCHEMA_CHANGE, + E_CONTRACT_POLICY, + E_RETENTION_DIRECTION, + E_RETENTION_PLAN_PHASE, + E_PLAN_ACTION, + E_PLAN_REASON, + E_INCREMENTAL_STRATEGY, + E_MICROBATCH_STRATEGY, + E_INCREMENTAL_MODE, + E_SNAPSHOT_STRATEGY, + E_HISTORICAL_INPUT, + E_INITIAL_VALID_FROM, + E_SNAPSHOT_SCHEMA_CHANGE_POLICY, + E_CURSOR_TYPE, + E_CURSOR_GRAIN, + E_MATERIALIZATION_TYPE, + E_SCENARIO_ARTIFACT_KIND, + E_SQL_REFERENCE_KIND, + E_HARD_CODED_RELATION_OWNER_KIND, + E_LOADER_CONNECTION_MODE, + E_RULE_SUBJECT, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs new file mode 100644 index 0000000000..2493399577 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -0,0 +1,521 @@ +//! The compiler, rule, lint and config expected-error classes hosted natively. + +use crate::bindings::_helpers::records::constants::{EXCEPTION_PARENT, VALUE_ERROR_PARENT}; +use crate::bindings::_helpers::records::models::{ClassValue, ErrorSpec}; + +/// Compiler, rule, lint and config expected-error classes, each after its parent. +pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ + ErrorSpec { + name: "DiscoveryError", + parent: VALUE_ERROR_PARENT, + doc: "Base error for expected discovery failures.", + class_attributes: &[("code", ClassValue::Text("D000"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectConfigError", + parent: "DiscoveryError", + doc: "Raised when shared or local project config files are invalid.", + class_attributes: &[("code", ClassValue::Text("D001"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ModelSqlParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL model file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D002"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlTestParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL-native test file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D003"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlAuditParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL audit file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D004"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SchemaParseError", + parent: "DiscoveryError", + doc: "Raised when a schema.yml file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D005"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SourceParseError", + parent: "DiscoveryError", + doc: "Raised when a sources/*.yml file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D006"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "DiscoveryConflictError", + parent: "DiscoveryError", + doc: "Raised when discovered project inputs conflict across files.", + class_attributes: &[("code", ClassValue::Text("D007"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SeedDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when declared seed metadata does not match local seed files.", + class_attributes: &[("code", ClassValue::Text("D008"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlScenarioParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL scenario file has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D009"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "PythonNodeDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project Python nodes cannot be discovered.", + class_attributes: &[("code", ClassValue::Text("D011"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProviderDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project providers cannot be discovered.", + class_attributes: &[("code", ClassValue::Text("D012"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "DeclarationParseError", + parent: "DiscoveryError", + doc: "Raised when an authored public SQL declaration is invalid.", + class_attributes: &[("code", ClassValue::Text("D013"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlHookParseError", + parent: "DiscoveryError", + doc: "Raised when an authored SQL hook resource has an invalid shape.", + class_attributes: &[("code", ClassValue::Text("D014"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "EventExporterDiscoveryError", + parent: "DiscoveryError", + doc: "Raised when project lifecycle event exporters are invalid.", + class_attributes: &[("code", ClassValue::Text("D015"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectPythonPathError", + parent: "DiscoveryError", + doc: "Raised when project-owned Python lives outside a supported extension root.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectPathError", + parent: "DiscoveryError", + doc: "Raised when an authored project path cannot be read as valid UTF-8.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ResourceIdentityError", + parent: "DiscoveryError", + doc: "Raised when an authored resource identity is not canonical snake_case.", + class_attributes: &[("code", ClassValue::Text("D016"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "MeasurementAuditError", + parent: "builtins.ValueError", + doc: "Raised when a measurement audit contract or value is invalid.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ContractAdoptionError", + parent: "builtins.RuntimeError", + doc: "Raised when safe contract source editing cannot continue.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "FingerprintInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when fingerprint storage inputs or rows are invalid.", + class_attributes: &[("code", ClassValue::Text("F000"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "QueryFingerprintError", + parent: "builtins.ValueError", + doc: "Raised when SQL cannot be tokenized into a change fingerprint.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeStageMismatchError", + parent: "builtins.RuntimeError", + doc: "Raised when native state the compiler replays is inconsistent with its inputs.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnknownLineageDialectError", + parent: "builtins.ValueError", + doc: "Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RefactorValueError", + parent: "builtins.ValueError", + doc: "An unhandled value failure the Python planner also leaves to the caller.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RefactorIOError", + parent: "builtins.OSError", + doc: "A file read or write failure the Python planner also leaves to the caller.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ScopeError", + parent: VALUE_ERROR_PARENT, + doc: "Base error for invalid scope-domain input.", + class_attributes: &[("code", ClassValue::Text("S000"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "InvalidScopePathError", + parent: "ScopeError", + doc: "Raised when a project-relative scope path is invalid.", + class_attributes: &[("code", ClassValue::Text("S011"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "InvalidQualifiedIdentityError", + parent: "ScopeError", + doc: "Raised when a qualified scope identity cannot be parsed.", + class_attributes: &[("code", ClassValue::Text("S002"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnindexableScopeError", + parent: "ScopeError", + doc: "Raised when the native scope index cannot place a discovered path or resource kind.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ScopeValidationError", + parent: "ScopeError", + doc: "Aggregate error raised only when strict scope validation is requested.", + class_attributes: &[], + required: &["diagnostics"], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "UnsupportedLexicalSyntaxError", + parent: "builtins.ValueError", + doc: "Raised when an adapter declares lexical rules the native SQL scanners do not support.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "LintError", + parent: EXCEPTION_PARENT, + doc: "A lint or format run could not proceed.", + class_attributes: &[("code", ClassValue::Text("L001"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeLintError", + parent: "LintError", + doc: "The SQLBuild-owned native lint engine rejected a request or response.", + class_attributes: &[("code", ClassValue::Text("L003"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ProjectCompileError", + parent: "LintError", + doc: "The project could not be compiled, so its SQL cannot be linted.", + class_attributes: &[("code", ClassValue::Text("L005"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "LintStoppedError", + parent: "builtins.Exception", + doc: "A speculative lint run was stopped because its result is no longer needed.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SpecConfigError", + parent: "builtins.RuntimeError", + doc: "Raised when project specification configuration is invalid.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "ConfigValueTypeError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when an authored config value has the wrong runtime type.", + class_attributes: &[], + required: &["key", "expected", "actual_type"], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropError", + parent: VALUE_ERROR_PARENT, + doc: "Base class for expected dbt interop errors.", + class_attributes: &[("code", ClassValue::Text("C240"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropConfigError", + parent: "DbtInteropError", + doc: "Raised when dbt project configuration is missing or invalid.", + class_attributes: &[("code", ClassValue::Text("C240"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropRuntimeError", + parent: "DbtInteropError", + doc: "Raised when dbt command execution fails.", + class_attributes: &[("code", ClassValue::Text("C241"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "DbtInteropArgumentError", + parent: "DbtInteropError", + doc: "Raised when `sqb dbt` arguments cannot be routed safely.", + class_attributes: &[("code", ClassValue::Text("C230"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "PlannerInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when planner inputs cannot be resolved safely.", + class_attributes: &[("code", ClassValue::Text("S000"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "NativeSqlTestPlanningError", + parent: "PlannerInputError", + doc: "Raised when the native SQL-test planning boundary returns invalid data.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "FutureCursorSafetyError", + parent: "PlannerInputError", + doc: "Raised when an effective cursor exceeds the configured future limit.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "MaximumAutomaticStartError", + parent: "PlannerInputError", + doc: "Raised when automatic cursor recovery would exceed its configured horizon.", + class_attributes: &[], + required: &[], + optional: &["evidence"], + fixed: &[], + }, + ErrorSpec { + name: "CompileInputError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when discovered inputs cannot be attached into a compile view.", + class_attributes: &[("code", ClassValue::Text("P001"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "MacroArgumentError", + parent: "CompileInputError", + doc: "Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "SqlTestReferenceError", + parent: "CompileInputError", + doc: "Raised when a SQL-test CTE calls a reference the test query cannot resolve.", + class_attributes: &[("code", ClassValue::Text("P013"))], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RulesError", + parent: "builtins.RuntimeError", + doc: "Raised for invalid compiler rules or execution.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RulesConfigError", + parent: "RulesError", + doc: "Raised when rules configuration breaks a project policy; help shows the setting to change.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RuleUsageError", + parent: "RulesError", + doc: "Raised when a rule or rule test uses the public API incorrectly.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "NonHermeticRuleError", + parent: "RulesError", + doc: "Raised when a custom rule observes state the Rules cache cannot track.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "RuleAssertionError", + parent: "RulesError", + doc: "Raised when a custom-rule harness case produces an unexpected result.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "FactDigestError", + parent: "RulesError", + doc: "Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "OpaqueModuleStateError", + parent: "RulesError", + doc: "Raised when custom-rule module state holds a value whose mutations cannot be observed.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, + ErrorSpec { + name: "HostCancelledError", + parent: "RulesError", + doc: "Raised in a custom-rule host that stops because another host already failed.", + class_attributes: &[], + required: &[], + optional: &[], + fixed: &[], + }, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 048b70cc3d..b34dea553e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -1,9 +1,6 @@ -//! Layouts of the compiler, rule, lint and config value records hosted natively. +//! Layouts of the compiler, lint and config value records hosted natively. -use crate::bindings::_helpers::records::constants::{EXCEPTION_PARENT, VALUE_ERROR_PARENT}; -use crate::bindings::_helpers::records::models::{ - ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, -}; +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; const R_NATIVE_MODEL_ANALYSIS_REQUEST: RecordSpec = RecordSpec { name: "NativeModelAnalysisRequest", @@ -422,6 +419,7 @@ const R_LINT_CONFIG: RecordSpec = RecordSpec { name: "relation_keys", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, ], }; @@ -436,6 +434,7 @@ const R_LINT_RUN_RESULT: RecordSpec = RecordSpec { name: "source_texts", default: FieldDefault::EmptyDict, compared: false, + shown: false, }, FieldSpec::new("unexpandable", FieldDefault::EmptyDict), ], @@ -575,124 +574,6 @@ const R_CLONE_PIPELINE_RESULT: RecordSpec = RecordSpec { FieldSpec::new("origin_source_entries", FieldDefault::EmptyTuple), ], }; -const R_PARSED_SELECTOR: RecordSpec = RecordSpec { - name: "ParsedSelector", - doc: "One parsed selector token before graph resolution.", - fields: &[ - FieldSpec::required("kind"), - FieldSpec::required("value"), - FieldSpec::new("upstream", FieldDefault::False), - FieldSpec::new("downstream", FieldDefault::False), - ], -}; -const R_PATH_SELECTOR: RecordSpec = RecordSpec { - name: "PathSelector", - doc: "A directed path selector between two model names with optional endpoint expansion.", - fields: &[ - FieldSpec::required("start_name"), - FieldSpec::required("end_name"), - FieldSpec::new("upstream", FieldDefault::False), - FieldSpec::new("downstream", FieldDefault::False), - ], -}; -const R_SQL_TEST_SELECTION: RecordSpec = RecordSpec { - name: "SqlTestSelection", - doc: "Unit tests selected or excluded by name, beside the tests of the selected resources.", - fields: &[ - FieldSpec::new("names", FieldDefault::EmptyFrozenset), - FieldSpec::new("excluded_names", FieldDefault::EmptyFrozenset), - FieldSpec::new("models_selected", FieldDefault::True), - ], -}; -const R_PLANNER_SCOPE: RecordSpec = RecordSpec { - name: "PlannerScope", - doc: "Resolved graph scope for one planner invocation.", - fields: &[ - FieldSpec::required("upstream_deps"), - FieldSpec::required("downstream_deps"), - FieldSpec::required("all_keys"), - FieldSpec::required("models_by_name"), - FieldSpec::required("selected_keys"), - FieldSpec::required("execution_order"), - FieldSpec::new("user_selected_keys", FieldDefault::EmptyFrozenset), - FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), - FieldSpec::new( - "sql_test_selection", - FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), - ), - ], -}; -const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { - name: "PlannerRelationsContext", - doc: "Resolved relation and source inputs for plan entry construction.", - fields: &[ - FieldSpec::required("model_locations"), - FieldSpec::required("seed_locations"), - FieldSpec::required("function_locations"), - FieldSpec::required("source_map"), - FieldSpec::required("source_read_map"), - FieldSpec::required("python_source_read_map"), - FieldSpec::required("source_warehouse_columns"), - FieldSpec::required("star_exclude_keyword"), - FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), - ], -}; -const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { - name: "SqlTestAssertionStep", - doc: "One zero-row assertion step in a SQL-native unit test.", - fields: &[ - FieldSpec::required("name"), - FieldSpec::required("resolved_sql"), - FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), - FieldSpec::new("comparison_body_sql", FieldDefault::None), - ], -}; -const R_PLANNER_SELECTION: RecordSpec = RecordSpec { - name: "PlannerSelection", - doc: "User-facing selection inputs for one planner invocation.", - fields: &[ - FieldSpec::new("select", FieldDefault::EmptyTuple), - FieldSpec::new("exclude", FieldDefault::EmptyTuple), - FieldSpec::new("selected_keys", FieldDefault::None), - FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), - FieldSpec::new( - "sql_test_selection", - FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), - ), - ], -}; -const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { - name: "PlannerOverrides", - doc: "Explicit user overrides that force or bound planner decisions.", - fields: &[ - FieldSpec::new("cursor_overrides", FieldDefault::None), - FieldSpec::new("full_refresh", FieldDefault::False), - FieldSpec::new("reload_sources", FieldDefault::False), - FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), - FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), - FieldSpec::new("max_microbatches", FieldDefault::None), - ], -}; -const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { - name: "DeferralInputs", - doc: "Deferred relation and source-deferral inputs for planning.", - fields: &[ - FieldSpec::new("deferred_locations", FieldDefault::None), - FieldSpec::new("deferred_relations", FieldDefault::None), - FieldSpec::new("defer_sources_to", FieldDefault::None), - FieldSpec::new("source_deferral_enabled", FieldDefault::True), - ], -}; -const R_PLANNER_POLICIES: RecordSpec = RecordSpec { - name: "PlannerPolicies", - doc: "Behavior policies selected by the caller for one planner invocation.", - fields: &[ - FieldSpec::new("auto_load_sources", FieldDefault::False), - FieldSpec::new("selection_diagnostics", FieldDefault::False), - FieldSpec::new("plan_sql_tests", FieldDefault::True), - FieldSpec::new("record_migration_fingerprints", FieldDefault::True), - ], -}; const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { name: "DiscoveredPythonLoaderMetadata", doc: "Loader-specific metadata carried by an internal Python loader node.", @@ -705,11 +586,7 @@ const R_DISCOVERED_PYTHON_LOADER_METADATA: RecordSpec = RecordSpec { FieldSpec::new("contract", FieldDefault::None), FieldSpec::new( "connection_mode", - FieldDefault::Member( - "sqlbuild.compiler.discovery.types", - "LoaderConnectionMode", - "SQLBUILD", - ), + FieldDefault::Member("sqlbuild._native", "LoaderConnectionMode", "SQLBUILD"), ), ], }; @@ -767,20 +644,6 @@ const R_LITERAL_SQL_RELATION: RecordSpec = RecordSpec { FieldSpec::required("relation"), ], }; -const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { - name: "DirectSourceFreshnessPlanningResult", - doc: "Direct planning-time source freshness observations and comparisons.", - fields: &[ - FieldSpec::new("observed_records", FieldDefault::EmptyTuple), - FieldSpec::new("previous_records", FieldDefault::EmptyTuple), - FieldSpec::new("changed_identities", FieldDefault::EmptyFrozenset), - FieldSpec::new("unchanged_identities", FieldDefault::EmptyFrozenset), - FieldSpec::new("unknown_source_names", FieldDefault::EmptyTuple), - FieldSpec::new("unknown_sources", FieldDefault::EmptyDict), - FieldSpec::new("age_statuses", FieldDefault::EmptyDict), - FieldSpec::new("propagation", FieldDefault::None), - ], -}; const R_SQL_BINDING_DIAGNOSTIC: RecordSpec = RecordSpec { name: "SqlBindingDiagnostic", doc: "One native binding error or non-blocking runtime-conversion warning.", @@ -813,6 +676,7 @@ const R_SQL_SCHEMA_VALIDATION_REQUEST: RecordSpec = RecordSpec { name: "catalog", default: FieldDefault::None, compared: false, + shown: false, }, ], }; @@ -916,346 +780,210 @@ const R_DBT_LS_RESULT: RecordSpec = RecordSpec { doc: "Parsed result from one dbt ls invocation.", fields: &[FieldSpec::required("nodes"), FieldSpec::required("command")], }; -const R_RULE_GUIDANCE: RecordSpec = RecordSpec { - name: "RuleGuidance", - doc: "Structured authoring guidance shared by diagnostics and generated documentation.", +const R_MEASUREMENT_THRESHOLD_BOUND: RecordSpec = RecordSpec { + name: "MeasurementThresholdBound", + doc: "One directional threshold bound for a measurement audit.", fields: &[ - FieldSpec::required("good_example"), - FieldSpec::required("anti_tautology"), - FieldSpec::required("mutation_check"), + FieldSpec::required("operator"), + FieldSpec::new("limit", FieldDefault::None), + FieldSpec::new("lower", FieldDefault::None), + FieldSpec::new("upper", FieldDefault::None), ], }; -const R_RULE_EXEMPTION: RecordSpec = RecordSpec { - name: "RuleExemption", - doc: "", +const R_MEASUREMENT_THRESHOLDS: RecordSpec = RecordSpec { + name: "MeasurementThresholds", + doc: "Warning and error policy for a measurement audit.", fields: &[ - FieldSpec::required("rule"), - FieldSpec::required("path"), - FieldSpec::required("reason"), + FieldSpec::new("warn", FieldDefault::None), + FieldSpec::new("error", FieldDefault::None), ], }; -const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { - name: "GraphEdgeExclusion", - doc: "One exact, temporary internal-layer dependency exception.", +const R_MEASUREMENT_CONTRACT: RecordSpec = RecordSpec { + name: "MeasurementContract", + doc: "Columns and sample unit returned by a measurement query.", fields: &[ - FieldSpec::required("consumer"), - FieldSpec::required("dependency"), - FieldSpec::required("reason"), + FieldSpec::required("value_column"), + FieldSpec::new("sample_count_column", FieldDefault::None), + FieldSpec::new("sample_unit", FieldDefault::None), ], }; -const R_RULE_IGNORE: RecordSpec = RecordSpec { - name: "RuleIgnore", - doc: "", +const R_AUDIT_IDENTITY: RecordSpec = RecordSpec { + name: "AuditIdentity", + doc: "Stable identity values for one planned audit binding.", fields: &[ - FieldSpec::required("rules"), - FieldSpec::required("paths"), - FieldSpec::required("reason"), - FieldSpec::new("selectors", FieldDefault::EmptyTuple), + FieldSpec::required("binding_key"), + FieldSpec::required("audit_name"), + FieldSpec::required("definition_fingerprint"), + FieldSpec::required("execution_fingerprint"), + FieldSpec::required("severity"), + FieldSpec::required("run_scope_phase"), + FieldSpec::required("attachment_kind"), + FieldSpec::new("attached_target_name", FieldDefault::None), + FieldSpec::new("attached_column_name", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), ], }; -const R_SELECT_STAR_ALLOW: RecordSpec = RecordSpec { - name: "SelectStarAllow", - doc: "", - fields: &[FieldSpec::required("paths"), FieldSpec::required("reason")], +const R_AUDIT_GATE_IDENTITY: RecordSpec = RecordSpec { + name: "AuditGateIdentity", + doc: "Aggregate identity for a model's runtime audit gate.", + fields: &[ + FieldSpec::required("binding_set_hash"), + FieldSpec::required("blocking_set_hash"), + FieldSpec::required("audits"), + ], }; -const R_THRESHOLD_OVERRIDE: RecordSpec = RecordSpec { - name: "ThresholdOverride", - doc: "", +const R_SQL_LOGICAL_TYPE: RecordSpec = RecordSpec { + name: "SqlLogicalType", + doc: "Logical type inferred for a normalized SQL value.", fields: &[ - FieldSpec::required("paths"), - FieldSpec::required("thresholds"), - FieldSpec::required("reason"), + FieldSpec::required("kind"), + FieldSpec::new("element_type", FieldDefault::None), ], }; -const R_RULES_CACHE_CONFIG: RecordSpec = RecordSpec { - name: "RulesCacheConfig", - doc: "", - fields: &[FieldSpec::new("enabled", FieldDefault::True)], +const R_SQL_VALUE: RecordSpec = RecordSpec { + name: "SqlValue", + doc: "One validated, canonical SQL value.", + fields: &[ + FieldSpec::required("logical_type"), + FieldSpec::required("value"), + ], }; -const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { - name: "SqlTestRulesConfig", - doc: "", - fields: &[FieldSpec::new( - "pipeline_directory", - FieldDefault::Text("pipelines"), - )], +const R_SELECTED_DBT_PROFILE_OUTPUT: RecordSpec = RecordSpec { + name: "SelectedDbtProfileOutput", + doc: "Selected dbt profile target output before rendering.", + fields: &[ + FieldSpec::required("profile_name"), + FieldSpec::required("target_name"), + FieldSpec::required("output"), + ], }; -const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { - name: "PreparedSqlLintResult", - doc: "Reusable expansion context and completed preliminary SQL checks.", +const R_RESOLVED_DBT_PROFILE_OUTPUT: RecordSpec = RecordSpec { + name: "ResolvedDbtProfileOutput", + doc: "Rendered dbt profile target output.", fields: &[ - FieldSpec::required("result"), - FieldSpec::required("context"), + FieldSpec::required("project_dir"), + FieldSpec::required("profiles_dir"), + FieldSpec::required("profile_name"), + FieldSpec::required("target_name"), + FieldSpec::required("output"), + ], +}; +const R_DISCOVERED_PYTHON_NODE: RecordSpec = RecordSpec { + name: "DiscoveredPythonNode", + doc: "Shared internal view of a discovered Python DAG node.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("file_path"), + FieldSpec::required("relative_path"), + FieldSpec::required("name"), + FieldSpec::required("function"), + FieldSpec::new("depends_on", FieldDefault::EmptyTuple), + FieldSpec::new("sql_deps", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("group", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::None), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec::new("loader", FieldDefault::None), + FieldSpec::new("task", FieldDefault::None), + FieldSpec::new("asset", FieldDefault::None), + FieldSpec::new("check", FieldDefault::None), + FieldSpec::new("identity", FieldDefault::None), ], }; -const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { - name: "SqlExpansionReuse", - doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", +const R_PYTHON_NODE_GRAPH: RecordSpec = RecordSpec { + name: "PythonNodeGraph", + doc: "Internal graph inventory for discovered executable Python nodes.", fields: &[ - FieldSpec::required("discovered_inputs"), - FieldSpec::required("declaration_scope"), + FieldSpec::required("nodes"), + FieldSpec::required("dependency_edges"), + FieldSpec::required("upstream_deps"), + FieldSpec::required("downstream_deps"), + FieldSpec::required("tag_index"), + FieldSpec::required("path_index"), + FieldSpec::required("nodes_by_name"), + FieldSpec::required("nodes_by_typed_selector"), ], }; -const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { - name: "PreparedSqlLint", - doc: "Process-local SQL checks started after compiler expansion has completed.", +const R_PYTHON_SQL_RUN_LIFECYCLE_PLAN: RecordSpec = RecordSpec { + name: "PythonSqlRunLifecyclePlan", + doc: "Lifecycle-aware run classification for SQL resources and Python nodes.", fields: &[ - FieldSpec::required("codes"), - FieldSpec::required("future"), - FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::required("ingress_python_node_names"), + FieldSpec::required("ingress_loader_names"), + FieldSpec::required("read_side_python_node_names"), ], }; -const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { - name: "SqlRulesEvaluation", - doc: "SQL rule findings of one run, their cache accounting and elapsed time.", +const R_PYTHON_IDENTITY_DEPENDENCY: RecordSpec = RecordSpec { + name: "PythonIdentityDependency", + doc: "One first-party Python source object included in a Python node identity.", fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("cache_hits"), - FieldSpec::required("cache_misses"), - FieldSpec::new("sql_ms", FieldDefault::Int(0)), + FieldSpec::required("kind"), + FieldSpec::required("module"), + FieldSpec::required("qualname"), + FieldSpec::required("source_path"), + FieldSpec::required("source_hash"), + FieldSpec::required("source_text"), ], }; -const R_RULE_RUN_PLAN: RecordSpec = RecordSpec { - name: "RuleRunPlan", - doc: "The catalogue, selected rules, project slice and settings one rules run evaluates.", +const R_PYTHON_NODE_IDENTITY: RecordSpec = RecordSpec { + name: "PythonNodeIdentity", + doc: "Read-only identity metadata for a SQLBuild Python node.", + fields: &[ + FieldSpec::required("node_type"), + FieldSpec::required("node_name"), + FieldSpec::required("source_path"), + FieldSpec::required("source_hash"), + FieldSpec::required("definition_hash"), + FieldSpec::required("version_hash"), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("definition_json", FieldDefault::Text("{}")), + FieldSpec::new("metadata_json", FieldDefault::Text("{}")), + ], +}; +const R_SQL_LEXICAL_SYNTAX: RecordSpec = RecordSpec { + name: "SqlLexicalSyntax", + doc: "Dialect rules for where SQL quoted text and comments end.", + fields: &[ + FieldSpec::new("backslash_escape_quotes", FieldDefault::EmptyFrozenset), + FieldSpec::new("escape_string_prefix", FieldDefault::False), + FieldSpec::new("raw_string_prefix", FieldDefault::False), + FieldSpec::new("triple_quoted_strings", FieldDefault::False), + FieldSpec::new("nested_block_comments", FieldDefault::False), + FieldSpec::new( + "line_comment_prefixes", + FieldDefault::TextFrozenset(&["--"]), + ), + ], +}; +const R_PROJECT_GRAPH: RecordSpec = RecordSpec { + name: "ProjectGraph", + doc: "Static compiled project graph held natively; dict views are built only when read.", fields: &[ - FieldSpec::required("effective_config"), - FieldSpec::required("project_dir"), - FieldSpec::required("catalogue"), - FieldSpec::required("custom_payloads"), - FieldSpec::required("selected"), - FieldSpec::required("skipped_type_proof_rules"), - FieldSpec::required("model_paths"), FieldSpec::required("project"), - FieldSpec::required("config"), - ], -}; -const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { - name: "RulesRunResult", - doc: "Unified findings and phase measurements for configured rules.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("evaluated_models"), - FieldSpec::required("built_in_ms"), - FieldSpec::required("custom_ms"), - FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), - FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), - FieldSpec::new("cache_hits", FieldDefault::Int(0)), - FieldSpec::new("cache_misses", FieldDefault::Int(0)), - FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), - ], -}; -const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { - name: "NativeRulesEvaluation", - doc: "The built-in rules response, decoded natively.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("selected_codes"), - FieldSpec::required("evaluated_models"), - FieldSpec::required("cache_hits"), - FieldSpec::required("cache_misses"), - FieldSpec::required("built_in_ms"), - FieldSpec::required("reused"), - ], -}; -const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { - name: "CustomRulesOutcome", - doc: "Custom-rule findings in canonical order and their cache accounting.", - fields: &[ - FieldSpec::required("findings"), - FieldSpec::required("cache_hits"), - FieldSpec::required("cache_misses"), - FieldSpec::required("custom_ms"), - FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), - ], -}; -const E_CONTRACT_FINDING_KIND: EnumSpec = EnumSpec { - name: "ContractFindingKind", - doc: "Stable physical-to-authored contract difference categories.", - members: &[ - ("MISSING_DECLARATION", "missing_declaration"), - ("MISSING_PHYSICAL_COLUMN", "missing_physical_column"), - ("TYPE_MISMATCH", "type_mismatch"), - ("RELATION_MISSING", "relation_missing"), - ("INSPECTION_UNAVAILABLE", "inspection_unavailable"), - ("OWNERSHIP_CONFLICT", "ownership_conflict"), - ], -}; -const E_COMPILER_STAGE: EnumSpec = EnumSpec { - name: "CompilerStage", - doc: "Frontier objects handed from one compiler stage to the next, in pipeline order.", - members: &[ - ("DISCOVERED_PROJECT_INPUTS", "discovered_project_inputs"), - ("COMPILE_PROJECT_INPUTS", "compile_project_inputs"), - ("COMPILED_PROJECT", "compiled_project"), - ], -}; -const E_HOOK_READ_TYPE: EnumSpec = EnumSpec { - name: "HookReadType", - doc: "Kind of model hook that declares or writes a read.", - members: &[ - ("PYTHON", "Python"), - ("SQL", "SQL"), - ("INLINE_SQL", "inline SQL"), - ], -}; -const E_LINT_SEVERITY: EnumSpec = EnumSpec { - name: "LintSeverity", - doc: "Severity assigned to one lint violation.", - members: &[("FAULT", "fault"), ("WARNING", "warning")], -}; -const E_DIAGNOSTIC_PHASE: EnumSpec = EnumSpec { - name: "DiagnosticPhase", - doc: "Phase that produced a compiler diagnostic.", - members: &[ - ("COMPILE", "compile"), - ("CONTRACT", "contract"), - ("RULE", "rule"), - ("PLAN", "plan"), - ("BUILD", "build"), - ("AUDIT", "audit"), - ("TEST", "test"), - ("CONNECTION", "connection"), - ], -}; -const E_DIAGNOSTIC_SEVERITY: EnumSpec = EnumSpec { - name: "DiagnosticSeverity", - doc: "Severity for a compiler diagnostic.", - members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], -}; -const E_COLUMN_TRANSFORM_KIND: EnumSpec = EnumSpec { - name: "ColumnTransformKind", - doc: "High-level transform classification for one output column.", - members: &[ - ("DIRECT", "direct"), - ("CAST", "cast"), - ("EXPRESSION", "expression"), - ("AGGREGATION", "aggregation"), - ("STAR", "star"), - ("CONSTANT", "constant"), - ("UNKNOWN", "unknown"), - ], -}; -const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { - name: "ColumnLineageConfidence", - doc: "Coarse confidence that a lineage edge is fully understood.", - members: &[ - ("HIGH", "high"), - ("MEDIUM", "medium"), - ("LOW", "low"), - ("UNKNOWN", "unknown"), - ], -}; -const E_COLUMN_LINEAGE_MODE: EnumSpec = EnumSpec { - name: "ColumnLineageMode", - doc: "Column lineage analyzer mode.", - members: &[("RICH", "rich"), ("FAST", "fast")], -}; -const E_POLYGLOT_ANALYSIS_DIALECT: EnumSpec = EnumSpec { - name: "PolyglotAnalysisDialect", - doc: "Dialect names accepted by Polyglot analyze_query.", - members: &[ - ("GENERIC", "generic"), - ("POSTGRESQL", "postgresql"), - ("MYSQL", "mysql"), - ("BIGQUERY", "bigquery"), - ("SNOWFLAKE", "snowflake"), - ("DUCKDB", "duckdb"), - ("SQLITE", "sqlite"), - ("HIVE", "hive"), - ("SPARK", "spark"), - ("TRINO", "trino"), - ("PRESTO", "presto"), - ("REDSHIFT", "redshift"), - ("TSQL", "tsql"), - ("ORACLE", "oracle"), - ("CLICKHOUSE", "clickhouse"), - ("DATABRICKS", "databricks"), - ("ATHENA", "athena"), - ("TERADATA", "teradata"), - ("DORIS", "doris"), - ("STARROCKS", "starrocks"), - ("MATERIALIZE", "materialize"), - ("RISINGWAVE", "risingwave"), - ("SINGLESTORE", "singlestore"), - ("COCKROACHDB", "cockroachdb"), - ("TIDB", "tidb"), - ("DRUID", "druid"), - ("SOLR", "solr"), - ("TABLEAU", "tableau"), - ("DUNE", "dune"), - ("FABRIC", "fabric"), - ("DRILL", "drill"), - ("DREMIO", "dremio"), - ("EXASOL", "exasol"), - ("DATAFUSION", "datafusion"), - ], -}; -const E_MIGRATION_DISCOVERY: EnumSpec = EnumSpec { - name: "MigrationDiscovery", - doc: "How a migration was requested.", - members: &[("MANUAL", "manual"), ("AUTOMATIC", "automatic")], -}; -const E_MIGRATION_COMPATIBILITY: EnumSpec = EnumSpec { - name: "MigrationCompatibility", - doc: "Outcome of checking the origin relation against the destination model.", - members: &[ - ("COMPATIBLE", "compatible"), - ("INCOMPATIBLE", "incompatible"), - ("NOT_CHECKED", "not_checked"), - ], -}; -const E_OLD_NAME_VIEW_ACTION: EnumSpec = EnumSpec { - name: "OldNameViewAction", - doc: "What a build does at a migrated model's old name.", - members: &[ - ("ARCHIVE_AND_VIEW", "archive_and_view"), - ("VIEW_ONLY", "view_only"), - ("LIVE", "live"), - ("NONE", "none"), - ], -}; -const E_SELECTOR_KIND: EnumSpec = EnumSpec { - name: "SelectorKind", - doc: "", - members: &[ - ("NAME", "name"), - ("SEED", "seed"), - ("SOURCE", "source"), - ("TASK", "task"), - ("ASSET", "asset"), - ("LOADER", "loader"), - ("CHECK", "check"), - ("TAG", "tag"), - ("PATH", "path"), - ("TEST", "test"), - ], -}; -const E_PYTHON_RUN_PHASE: EnumSpec = EnumSpec { - name: "PythonRunPhase", - doc: "Lifecycle-aware Python execution phase.", - members: &[ - ("PRE_SQL_INGRESS", "pre_sql_ingress"), - ("READ_SIDE", "read_side"), - ], -}; -const E_PYTHON_IDENTITY_STATUS: EnumSpec = EnumSpec { - name: "PythonIdentityStatus", - doc: "Read-only identity diff status for Python plan display.", - members: &[ - ("NEW", "new"), - ("CHANGED", "changed"), - ("UNCHANGED", "unchanged"), - ("UNKNOWN", "unknown"), - ], -}; -const E_COLLECTION_RENDERING: EnumSpec = EnumSpec { - name: "CollectionRendering", - doc: "Physical rendering choices for homogeneous SQL collections.", - members: &[("VALUE_LIST", "value_list"), ("ARRAY", "array")], + FieldSpec { + name: "native", + default: FieldDefault::Required, + compared: false, + shown: false, + }, + ], +}; +const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { + name: "CompilePipelineResult", + doc: "Complete output from the compile-and-plan pipeline.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("plan_output"), + FieldSpec::new("custom_materializations", FieldDefault::EmptyDict), + FieldSpec::new("python_node_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("python_plan_entries", FieldDefault::EmptyTuple), + FieldSpec::new("compile_seconds", FieldDefault::None), + FieldSpec::new("planning_seconds", FieldDefault::None), + ], }; -/// Compiler, rule, lint and config value records. +/// The compiler, lint and config value records. pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_NATIVE_MODEL_ANALYSIS_REQUEST, R_NATIVE_MODEL_ANALYSES, @@ -1304,16 +1032,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_CLONE_PIPELINE_OPTIONS, R_CLONE_PIPELINE_CONNECTION, R_CLONE_PIPELINE_RESULT, - R_PARSED_SELECTOR, - R_PATH_SELECTOR, - R_SQL_TEST_SELECTION, - R_PLANNER_SCOPE, - R_PLANNER_RELATIONS_CONTEXT, - R_SQL_TEST_ASSERTION_STEP, - R_PLANNER_SELECTION, - R_PLANNER_OVERRIDES, - R_DEFERRAL_INPUTS, - R_PLANNER_POLICIES, R_DISCOVERED_PYTHON_LOADER_METADATA, R_DISCOVERED_PYTHON_TASK_METADATA, R_DISCOVERED_PYTHON_ASSET_METADATA, @@ -1321,7 +1039,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_PYTHON_NODE_DEPENDENCY_EDGE, R_PYTHON_SQL_RUN_SELECTION, R_LITERAL_SQL_RELATION, - R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT, R_SQL_BINDING_DIAGNOSTIC, R_SQL_BINDING_RESULT, R_SQL_SCHEMA_VALIDATION_REQUEST, @@ -1333,379 +1050,21 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_DBT_COMMAND_RESULT, R_DBT_NODE_EXECUTION_RESULT, R_DBT_LS_RESULT, - R_RULE_GUIDANCE, - R_RULE_EXEMPTION, - R_GRAPH_EDGE_EXCLUSION, - R_RULE_IGNORE, - R_SELECT_STAR_ALLOW, - R_THRESHOLD_OVERRIDE, - R_RULES_CACHE_CONFIG, - R_SQL_TEST_RULES_CONFIG, - R_PREPARED_SQL_LINT_RESULT, - R_SQL_EXPANSION_REUSE, - R_PREPARED_SQL_LINT, - R_SQL_RULES_EVALUATION, - R_RULE_RUN_PLAN, - R_RULES_RUN_RESULT, - R_NATIVE_RULES_EVALUATION, - R_CUSTOM_RULES_OUTCOME, -]; -/// Compiler, rule, lint and config string enums. -pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ - E_CONTRACT_FINDING_KIND, - E_COMPILER_STAGE, - E_HOOK_READ_TYPE, - E_LINT_SEVERITY, - E_DIAGNOSTIC_PHASE, - E_DIAGNOSTIC_SEVERITY, - E_COLUMN_TRANSFORM_KIND, - E_COLUMN_LINEAGE_CONFIDENCE, - E_COLUMN_LINEAGE_MODE, - E_POLYGLOT_ANALYSIS_DIALECT, - E_MIGRATION_DISCOVERY, - E_MIGRATION_COMPATIBILITY, - E_OLD_NAME_VIEW_ACTION, - E_SELECTOR_KIND, - E_PYTHON_RUN_PHASE, - E_PYTHON_IDENTITY_STATUS, - E_COLLECTION_RENDERING, -]; -/// Compiler, rule, lint and config expected-error classes, each after its parent. -pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ - ErrorSpec { - name: "DiscoveryError", - parent: VALUE_ERROR_PARENT, - doc: "Base error for expected discovery failures.", - class_attributes: &[("code", ClassValue::Text("D000"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectConfigError", - parent: "DiscoveryError", - doc: "Raised when shared or local project config files are invalid.", - class_attributes: &[("code", ClassValue::Text("D001"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ModelSqlParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL model file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D002"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlTestParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL-native test file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D003"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlAuditParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL audit file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D004"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SchemaParseError", - parent: "DiscoveryError", - doc: "Raised when a schema.yml file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D005"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SourceParseError", - parent: "DiscoveryError", - doc: "Raised when a sources/*.yml file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D006"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DiscoveryConflictError", - parent: "DiscoveryError", - doc: "Raised when discovered project inputs conflict across files.", - class_attributes: &[("code", ClassValue::Text("D007"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SeedDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when declared seed metadata does not match local seed files.", - class_attributes: &[("code", ClassValue::Text("D008"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlScenarioParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL scenario file has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D009"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "PythonNodeDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when project Python nodes cannot be discovered.", - class_attributes: &[("code", ClassValue::Text("D011"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProviderDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when project providers cannot be discovered.", - class_attributes: &[("code", ClassValue::Text("D012"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DeclarationParseError", - parent: "DiscoveryError", - doc: "Raised when an authored public SQL declaration is invalid.", - class_attributes: &[("code", ClassValue::Text("D013"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SqlHookParseError", - parent: "DiscoveryError", - doc: "Raised when an authored SQL hook resource has an invalid shape.", - class_attributes: &[("code", ClassValue::Text("D014"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "EventExporterDiscoveryError", - parent: "DiscoveryError", - doc: "Raised when project lifecycle event exporters are invalid.", - class_attributes: &[("code", ClassValue::Text("D015"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectPythonPathError", - parent: "DiscoveryError", - doc: "Raised when project-owned Python lives outside a supported extension root.", - class_attributes: &[("code", ClassValue::Text("D016"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectPathError", - parent: "DiscoveryError", - doc: "Raised when an authored project path cannot be read as valid UTF-8.", - class_attributes: &[("code", ClassValue::Text("D016"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ResourceIdentityError", - parent: "DiscoveryError", - doc: "Raised when an authored resource identity is not canonical snake_case.", - class_attributes: &[("code", ClassValue::Text("D016"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "MeasurementAuditError", - parent: "builtins.ValueError", - doc: "Raised when a measurement audit contract or value is invalid.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ContractAdoptionError", - parent: "builtins.RuntimeError", - doc: "Raised when safe contract source editing cannot continue.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "FingerprintInputError", - parent: VALUE_ERROR_PARENT, - doc: "Raised when fingerprint storage inputs or rows are invalid.", - class_attributes: &[("code", ClassValue::Text("F000"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "QueryFingerprintError", - parent: "builtins.ValueError", - doc: "Raised when SQL cannot be tokenized into a change fingerprint.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "NativeStageMismatchError", - parent: "builtins.RuntimeError", - doc: "Raised when native state the compiler replays is inconsistent with its inputs.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "UnknownLineageDialectError", - parent: "builtins.ValueError", - doc: "Raised, as Python's parser raised `ValueError`, for a dialect Polyglot does not know.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "RefactorValueError", - parent: "builtins.ValueError", - doc: "An unhandled value failure the Python planner also leaves to the caller.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "RefactorIOError", - parent: "builtins.OSError", - doc: "A file read or write failure the Python planner also leaves to the caller.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ScopeError", - parent: VALUE_ERROR_PARENT, - doc: "Base error for invalid scope-domain input.", - class_attributes: &[("code", ClassValue::Text("S000"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "InvalidScopePathError", - parent: "ScopeError", - doc: "Raised when a project-relative scope path is invalid.", - class_attributes: &[("code", ClassValue::Text("S011"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "InvalidQualifiedIdentityError", - parent: "ScopeError", - doc: "Raised when a qualified scope identity cannot be parsed.", - class_attributes: &[("code", ClassValue::Text("S002"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "UnindexableScopeError", - parent: "ScopeError", - doc: "Raised when the native scope index cannot place a discovered path or resource kind.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ScopeValidationError", - parent: "ScopeError", - doc: "Aggregate error raised only when strict scope validation is requested.", - class_attributes: &[], - required: &["diagnostics"], - fixed: &[], - }, - ErrorSpec { - name: "UnsupportedLexicalSyntaxError", - parent: "builtins.ValueError", - doc: "Raised when an adapter declares lexical rules the native SQL scanners do not support.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "LintError", - parent: EXCEPTION_PARENT, - doc: "A lint or format run could not proceed.", - class_attributes: &[("code", ClassValue::Text("L001"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "NativeLintError", - parent: "LintError", - doc: "The SQLBuild-owned native lint engine rejected a request or response.", - class_attributes: &[("code", ClassValue::Text("L003"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ProjectCompileError", - parent: "LintError", - doc: "The project could not be compiled, so its SQL cannot be linted.", - class_attributes: &[("code", ClassValue::Text("L005"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "LintStoppedError", - parent: "builtins.Exception", - doc: "A speculative lint run was stopped because its result is no longer needed.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "SpecConfigError", - parent: "builtins.RuntimeError", - doc: "Raised when project specification configuration is invalid.", - class_attributes: &[], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "ConfigValueTypeError", - parent: VALUE_ERROR_PARENT, - doc: "Raised when an authored config value has the wrong runtime type.", - class_attributes: &[], - required: &["key", "expected", "actual_type"], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropError", - parent: VALUE_ERROR_PARENT, - doc: "Base class for expected dbt interop errors.", - class_attributes: &[("code", ClassValue::Text("C240"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropConfigError", - parent: "DbtInteropError", - doc: "Raised when dbt project configuration is missing or invalid.", - class_attributes: &[("code", ClassValue::Text("C240"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropRuntimeError", - parent: "DbtInteropError", - doc: "Raised when dbt command execution fails.", - class_attributes: &[("code", ClassValue::Text("C241"))], - required: &[], - fixed: &[], - }, - ErrorSpec { - name: "DbtInteropArgumentError", - parent: "DbtInteropError", - doc: "Raised when `sqb dbt` arguments cannot be routed safely.", - class_attributes: &[("code", ClassValue::Text("C230"))], - required: &[], - fixed: &[], - }, + R_MEASUREMENT_THRESHOLD_BOUND, + R_MEASUREMENT_THRESHOLDS, + R_MEASUREMENT_CONTRACT, + R_AUDIT_IDENTITY, + R_AUDIT_GATE_IDENTITY, + R_SQL_LOGICAL_TYPE, + R_SQL_VALUE, + R_SELECTED_DBT_PROFILE_OUTPUT, + R_RESOLVED_DBT_PROFILE_OUTPUT, + R_DISCOVERED_PYTHON_NODE, + R_PYTHON_NODE_GRAPH, + R_PYTHON_SQL_RUN_LIFECYCLE_PLAN, + R_PYTHON_IDENTITY_DEPENDENCY, + R_PYTHON_NODE_IDENTITY, + R_SQL_LEXICAL_SYNTAX, + R_PROJECT_GRAPH, + R_COMPILE_PIPELINE_RESULT, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs index 1c9fbfd410..a9e9ea9d42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/shape_classes.rs @@ -1,41 +1,42 @@ -//! Resolve a shape class by name: the native class once it is hosted natively, else the Python one. +//! Resolve a shape class by its Python module and name: the native class once it is hosted natively. +use pyo3::exceptions::PyLookupError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cache::hosted_shapes::main::hosted_native_name::hosted_native_name; + use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS; -use crate::bindings::_helpers::compiler_records::constants::{ - COMPILER_ENUMS, COMPILER_ERRORS, COMPILER_RECORDS, -}; use crate::bindings::_helpers::records::constants::NATIVE_MODULE; -/// Whether `class` is one of the compiler shapes hosted natively. -fn hosted(class: &str) -> bool { - COMPILER_RECORDS.iter().any(|spec| spec.name == class) - || COMPILE_VALUE_RECORDS.iter().any(|spec| spec.name == class) - || COMPILER_ENUMS.iter().any(|spec| spec.name == class) - || COMPILER_ERRORS.iter().any(|spec| spec.name == class) +/// The native class name hosting `module:class`, failing loudly when the name is ambiguous. +fn hosted_name(module: &str, class: &str) -> PyResult> { + hosted_native_name(module, class).map_err(|_| { + PyLookupError::new_err(format!( + "{module}:{class} names more than one natively hosted shape" + )) + }) } -/// `class` from the native module when hosted natively, else from the imported `module`. +/// `class` of the Python module object `module`, or the native class hosting it. pub(crate) fn shape_class<'py, T>( module: &Bound<'py, T>, class: &str, ) -> PyResult> { - if hosted(class) { - return imported(module.py(), NATIVE_MODULE, class); + let module_path: String = module.as_any().getattr("__name__")?.extract()?; + if let Some(native) = hosted_name(&module_path, class)? { + return imported(module.py(), NATIVE_MODULE, native); } module.as_any().getattr(class) } -/// `class` from the native module when hosted natively, else from the module at `module_path`. +/// `class` of the module at `module_path`, or the native class hosting it. pub(crate) fn shape_class_at<'py>( py: Python<'py>, module_path: &str, class: &str, ) -> PyResult> { - if hosted(class) { - return imported(py, NATIVE_MODULE, class); + if let Some(native) = hosted_name(module_path, class)? { + return imported(py, NATIVE_MODULE, native); } imported(py, module_path, class) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs new file mode 100644 index 0000000000..4b9a90b52a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -0,0 +1,780 @@ +//! Layouts of the project configuration and contract value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_CLONE_POLICY: RecordSpec = RecordSpec { + name: "ClonePolicy", + doc: "Environment clone policy.", + fields: &[ + FieldSpec::new("allow_as_clone_origin", FieldDefault::False), + FieldSpec::new("allow_as_clone_destination", FieldDefault::False), + ], +}; +const R_LOCAL_CLONE_POLICY: RecordSpec = RecordSpec { + name: "LocalClonePolicy", + doc: "Local environment clone policy overrides.", + fields: &[ + FieldSpec::new("allow_as_clone_origin", FieldDefault::None), + FieldSpec::new("allow_as_clone_destination", FieldDefault::None), + ], +}; +const R_AUTHORED_TIME_TRAVEL_RETENTION: RecordSpec = RecordSpec { + name: "AuthoredTimeTravelRetention", + doc: "One authored time-travel retention policy.", + fields: &[ + FieldSpec::new("desired_days", FieldDefault::None), + FieldSpec::new("unmanaged", FieldDefault::False), + ], +}; +const R_MATERIALIZATION_RETENTION_DEFAULTS: RecordSpec = RecordSpec { + name: "MaterializationRetentionDefaults", + doc: "Retention defaults for one built-in materialization.", + fields: &[ + FieldSpec::new("time_travel_retention", FieldDefault::None), + FieldSpec::new("table_type", FieldDefault::None), + ], +}; +const R_MATERIALIZATION_DEFAULTS_CONFIG: RecordSpec = RecordSpec { + name: "MaterializationDefaultsConfig", + doc: "Project retention defaults by materialization type.", + fields: &[ + FieldSpec::new( + "table", + FieldDefault::Call("sqlbuild._native", "MaterializationRetentionDefaults"), + ), + FieldSpec::new( + "incremental", + FieldDefault::Call("sqlbuild._native", "MaterializationRetentionDefaults"), + ), + FieldSpec::new( + "snapshot", + FieldDefault::Call("sqlbuild._native", "MaterializationRetentionDefaults"), + ), + ], +}; +const R_RESOLVED_TIME_TRAVEL_RETENTION: RecordSpec = RecordSpec { + name: "ResolvedTimeTravelRetention", + doc: "Effective retention policy attached to a compiled model.", + fields: &[ + FieldSpec::new("desired_days", FieldDefault::None), + FieldSpec::new("unmanaged", FieldDefault::True), + FieldSpec::new("source", FieldDefault::None), + ], +}; +const R_RESOLVED_TABLE_TYPE: RecordSpec = RecordSpec { + name: "ResolvedTableType", + doc: "Effective Snowflake table type attached to a compiled model.", + fields: &[ + FieldSpec::new( + "value", + FieldDefault::Member("sqlbuild._native", "TableType", "TRANSIENT"), + ), + FieldSpec::new( + "source", + FieldDefault::Member("sqlbuild._native", "TableTypeSource", "DEFAULT"), + ), + FieldSpec::new("declared", FieldDefault::False), + ], +}; +const R_EXECUTION_LIMITS_CONFIG: RecordSpec = RecordSpec { + name: "ExecutionLimitsConfig", + doc: "Optional target-scoped build execution limits.", + fields: &[ + FieldSpec::new("max_models", FieldDefault::None), + FieldSpec::new("max_duration", FieldDefault::None), + FieldSpec::new("max_duration_seconds", FieldDefault::None), + FieldSpec::new("remediation", FieldDefault::None), + ], +}; +const R_TARGET_DIFF_CONFIG: RecordSpec = RecordSpec { + name: "TargetDiffConfig", + doc: "Optional target-scoped data-diff limits.", + fields: &[FieldSpec::new("max_full_rows", FieldDefault::None)], +}; +const R_TARGET_WAREHOUSES_CONFIG: RecordSpec = RecordSpec { + name: "TargetWarehousesConfig", + doc: "Optional per-command-group default warehouses for one target.", + fields: &[ + FieldSpec::new("build", FieldDefault::None), + FieldSpec::new("query", FieldDefault::None), + ], +}; +const R_TARGET_CONFIG: RecordSpec = RecordSpec { + name: "TargetConfig", + doc: "One named target configuration.", + fields: &[ + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connection_name", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("loader_schema", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("defer_clone_from", FieldDefault::None), + FieldSpec::new( + "clone", + FieldDefault::Call("sqlbuild._native", "ClonePolicy"), + ), + FieldSpec::new("compile_cache", FieldDefault::None), + FieldSpec::new("time_travel_retention", FieldDefault::None), + FieldSpec::new( + "time_travel_retention_by_materialization", + FieldDefault::EmptyDict, + ), + FieldSpec::new("owns_time_travel_retention_namespace", FieldDefault::False), + FieldSpec::new("default_table_type", FieldDefault::None), + FieldSpec::new( + "table_type_downgrade", + FieldDefault::Member( + "sqlbuild._native", + "TableTypeDowngradePolicy", + "REQUIRE_CONFIRMATION", + ), + ), + FieldSpec::new( + "time_travel_retention_decrease", + FieldDefault::Member("sqlbuild._native", "RetentionDecreasePolicy", "DENY"), + ), + FieldSpec::new( + "missing_migration_origin", + FieldDefault::Member("sqlbuild._native", "MissingMigrationOriginPolicy", "ALLOW"), + ), + FieldSpec::new( + "execution_limits", + FieldDefault::Call("sqlbuild._native", "ExecutionLimitsConfig"), + ), + FieldSpec::new( + "diff", + FieldDefault::Call("sqlbuild._native", "TargetDiffConfig"), + ), + FieldSpec::new( + "warehouses", + FieldDefault::Call("sqlbuild._native", "TargetWarehousesConfig"), + ), + ], +}; +const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { + name: "LocalTargetConfig", + doc: "Local developer overrides for one named target.", + fields: &[ + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connection_name", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("loader_schema", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("defer_clone_from", FieldDefault::None), + FieldSpec::new( + "clone", + FieldDefault::Call("sqlbuild._native", "LocalClonePolicy"), + ), + FieldSpec::new("compile_cache", FieldDefault::None), + FieldSpec::new("time_travel_retention", FieldDefault::None), + FieldSpec::new( + "time_travel_retention_by_materialization", + FieldDefault::None, + ), + FieldSpec::new("owns_time_travel_retention_namespace", FieldDefault::None), + FieldSpec::new("default_table_type", FieldDefault::None), + FieldSpec::new("table_type_downgrade", FieldDefault::None), + FieldSpec::new("time_travel_retention_decrease", FieldDefault::None), + FieldSpec::new("missing_migration_origin", FieldDefault::None), + FieldSpec::new( + "execution_limits", + FieldDefault::Call("sqlbuild._native", "ExecutionLimitsConfig"), + ), + FieldSpec::new( + "diff", + FieldDefault::Call("sqlbuild._native", "TargetDiffConfig"), + ), + FieldSpec::new( + "warehouses", + FieldDefault::Call("sqlbuild._native", "TargetWarehousesConfig"), + ), + ], +}; +const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { + name: "SettingsConfig", + doc: "Global feature toggles.", + fields: &[ + FieldSpec::new("sql_analysis", FieldDefault::True), + FieldSpec::new("require_sql_analysis", FieldDefault::False), + FieldSpec::new("query_change_tracking", FieldDefault::True), + FieldSpec::new( + "column_contract_mode", + FieldDefault::Member("sqlbuild._native", "ColumnContractMode", "IMPLICIT"), + ), + FieldSpec::new("concurrency", FieldDefault::Int(1)), + FieldSpec::new("auto_load_sources", FieldDefault::True), + FieldSpec::new("microbatch_concurrency", FieldDefault::False), + FieldSpec::new( + "microbatch_unaccounted_partition_policy", + FieldDefault::Text("synthesize"), + ), + FieldSpec::new("table_promotion_mode", FieldDefault::None), + FieldSpec::new("default_audit_severity", FieldDefault::None), + FieldSpec::new("default_audit_run_scope", FieldDefault::None), + ], +}; +const R_SCOPES_CONFIG: RecordSpec = RecordSpec { + name: "ScopesConfig", + doc: "Declaration scope policy configuration.", + fields: &[FieldSpec::new("enforce_placement", FieldDefault::True)], +}; +const R_REFERENCES_CONFIG: RecordSpec = RecordSpec { + name: "ReferencesConfig", + doc: "Explicit-reference enforcement policy configuration.", + fields: &[FieldSpec::new("enforce_explicit", FieldDefault::True)], +}; +const R_COST_CONFIG: RecordSpec = RecordSpec { + name: "CostConfig", + doc: "Snowflake compute estimate configuration.", + fields: &[ + FieldSpec::new( + "usd_per_credit", + FieldDefault::CallText("decimal", "Decimal", "3.00"), + ), + FieldSpec::new("usd_per_credit_is_default", FieldDefault::True), + ], +}; +const R_CONSTANTS_CONFIG: RecordSpec = RecordSpec { + name: "ConstantsConfig", + doc: "Project-wide constant rendering configuration.", + fields: &[FieldSpec::new( + "collection_rendering", + FieldDefault::Member("sqlbuild._native", "CollectionRendering", "VALUE_LIST"), + )], +}; +const R_FUTURE_CURSORS_CONFIG: RecordSpec = RecordSpec { + name: "FutureCursorsConfig", + doc: "Safety policy for automatically discovered future cursor watermarks.", + fields: &[ + FieldSpec::new("max_distance", FieldDefault::None), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "FutureCursorAction", "ERROR"), + ), + ], +}; +const R_START_CURSORS_CONFIG: RecordSpec = RecordSpec { + name: "StartCursorsConfig", + doc: "Safety policy for automatically discovered incremental starts.", + fields: &[ + FieldSpec::new("max_ahead", FieldDefault::None), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "FutureCursorAction", "ERROR"), + ), + ], +}; +const R_CURSORS_CONFIG: RecordSpec = RecordSpec { + name: "CursorsConfig", + doc: "Project-wide cursor safety configuration.", + fields: &[ + FieldSpec::new( + "future", + FieldDefault::Call("sqlbuild._native", "FutureCursorsConfig"), + ), + FieldSpec::new( + "start", + FieldDefault::Call("sqlbuild._native", "StartCursorsConfig"), + ), + ], +}; +const R_MICROBATCH_LIMITS_CONFIG: RecordSpec = RecordSpec { + name: "MicrobatchLimitsConfig", + doc: "Per-model microbatch planning safety limit.", + fields: &[ + FieldSpec::new("max_batches", FieldDefault::None), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "MicrobatchLimitAction", "ERROR"), + ), + ], +}; +const R_MICROBATCHES_CONFIG: RecordSpec = RecordSpec { + name: "MicrobatchesConfig", + doc: "Project-wide microbatch configuration.", + fields: &[FieldSpec::new( + "limits", + FieldDefault::Call("sqlbuild._native", "MicrobatchLimitsConfig"), + )], +}; +const R_DEFAULTS_CONFIG: RecordSpec = RecordSpec { + name: "DefaultsConfig", + doc: "Project-wide resource defaults.", + fields: &[ + FieldSpec::new("materialized", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("seed_database", FieldDefault::None), + FieldSpec::new("seed_schema", FieldDefault::None), + FieldSpec::new("function_database", FieldDefault::None), + FieldSpec::new("function_schema", FieldDefault::None), + FieldSpec::new("incremental_strategy", FieldDefault::None), + FieldSpec::new("incremental_mode", FieldDefault::None), + FieldSpec::new("microbatch_strategy", FieldDefault::None), + FieldSpec::new("cursor_watermark_mode", FieldDefault::None), + FieldSpec::new("merge_exclude_columns", FieldDefault::EmptyTuple), + FieldSpec::new("full_refresh", FieldDefault::None), + FieldSpec::new("append_cursor_inclusive", FieldDefault::None), + FieldSpec::new("cursor_start", FieldDefault::None), + FieldSpec::new("cursor_end", FieldDefault::None), + FieldSpec::new("cursor_start_max_ahead", FieldDefault::None), + FieldSpec::new("cursor_start_max_action", FieldDefault::None), + FieldSpec::new("cursor_future_max_distance", FieldDefault::None), + FieldSpec::new("cursor_future_action", FieldDefault::None), + FieldSpec::new("lookback", FieldDefault::None), + FieldSpec::new("batch_size", FieldDefault::None), + FieldSpec::new("batch_concurrency", FieldDefault::None), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("unaccounted_partition_policy", FieldDefault::None), + FieldSpec::new("replay_on_change", FieldDefault::None), + FieldSpec::new("row_diff_exclude_columns", FieldDefault::EmptyTuple), + FieldSpec::new("row_diff_tolerances", FieldDefault::EmptyDict), + FieldSpec::new("row_diff_sample_rows", FieldDefault::None), + FieldSpec::new("row_diff_sample_seed", FieldDefault::None), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new("pre_hooks", FieldDefault::None), + FieldSpec::new("post_hooks", FieldDefault::None), + ], +}; +const R_DIFF_CONFIG: RecordSpec = RecordSpec { + name: "DiffConfig", + doc: "Project-wide data-diff configuration.", + fields: &[FieldSpec::new( + "query_artifact_ttl", + FieldDefault::Text("24h"), + )], +}; +const R_JANITOR_CONFIG: RecordSpec = RecordSpec { + name: "JanitorConfig", + doc: "Janitor command defaults.", + fields: &[ + FieldSpec::new("enabled", FieldDefault::False), + FieldSpec::new("retention_days", FieldDefault::None), + FieldSpec::new("archive_retention_days", FieldDefault::Int(14)), + FieldSpec::new("direct_state_history_versions", FieldDefault::Int(20)), + FieldSpec::new("delete_tracked_only", FieldDefault::True), + FieldSpec::new("exclude_patterns", FieldDefault::EmptyTuple), + ], +}; +const R_MIGRATIONS_CONFIG: RecordSpec = RecordSpec { + name: "MigrationsConfig", + doc: "Model migration defaults.", + fields: &[FieldSpec::new("old_name_views", FieldDefault::Text("30d"))], +}; +const R_SNAPSHOTS_CONFIG: RecordSpec = RecordSpec { + name: "SnapshotsConfig", + doc: "Snapshot materialization safety defaults.", + fields: &[ + FieldSpec::new("current_state_full_refresh", FieldDefault::Text("deny")), + FieldSpec::new( + "historical_full_refresh", + FieldDefault::Text("require_confirmation"), + ), + FieldSpec::new("schema_change", FieldDefault::Text("append_new_columns")), + FieldSpec::new( + "wildcard_check_schema_change", + FieldDefault::Text("require_confirmation"), + ), + ], +}; +const R_SCENARIO_SNAPSHOT_LIMITS_CONFIG: RecordSpec = RecordSpec { + name: "ScenarioSnapshotLimitsConfig", + doc: "Scenario snapshot capture safety limits.", + fields: &[ + FieldSpec::new("max_rows_per_relation", FieldDefault::None), + FieldSpec::new("max_total_rows", FieldDefault::None), + FieldSpec::new("max_bytes_per_relation", FieldDefault::None), + FieldSpec::new("max_total_bytes", FieldDefault::None), + ], +}; +const R_SCENARIO_CONFIG: RecordSpec = RecordSpec { + name: "ScenarioConfig", + doc: "Scenario command configuration.", + fields: &[ + FieldSpec::new("run_namespace", FieldDefault::None), + FieldSpec::new("local_type_overrides", FieldDefault::EmptyDict), + FieldSpec::new( + "snapshot_limits", + FieldDefault::Call("sqlbuild._native", "ScenarioSnapshotLimitsConfig"), + ), + ], +}; +const R_DBT_CONFIG: RecordSpec = RecordSpec { + name: "DbtConfig", + doc: "dbt interop configuration.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("profiles_dir", FieldDefault::None), + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("target_path", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + ], +}; +const R_LOCAL_DBT_CONFIG: RecordSpec = RecordSpec { + name: "LocalDbtConfig", + doc: "Local dbt interop configuration overrides.", + fields: &[ + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("vars", FieldDefault::EmptyDict), + ], +}; +const R_LIFECYCLE_EVENT_SINK_FILTER_CONFIG: RecordSpec = RecordSpec { + name: "LifecycleEventSinkFilterConfig", + doc: "Destination-neutral runtime narrowing for lifecycle-event sinks.", + fields: &[ + FieldSpec::new("event_kinds", FieldDefault::None), + FieldSpec::new("min_severity", FieldDefault::None), + ], +}; +const R_LIFECYCLE_EVENT_SINKS_CONFIG: RecordSpec = RecordSpec { + name: "LifecycleEventSinksConfig", + doc: "Global and named lifecycle-event sink runtime filters and delivery settings.", + fields: &[ + FieldSpec::new( + "defaults", + FieldDefault::Call("sqlbuild._native", "LifecycleEventSinkFilterConfig"), + ), + FieldSpec::new("named", FieldDefault::EmptyDict), + FieldSpec::new("shutdown_timeout", FieldDefault::None), + FieldSpec::new("shutdown_timeout_seconds", FieldDefault::None), + ], +}; +const R_SINKS_CONFIG: RecordSpec = RecordSpec { + name: "SinksConfig", + doc: "Typed destination sink configuration.", + fields: &[FieldSpec::new( + "lifecycle", + FieldDefault::Call("sqlbuild._native", "LifecycleEventSinksConfig"), + )], +}; +const R_PROJECT_CONFIG: RecordSpec = RecordSpec { + name: "ProjectConfig", + doc: "Shared project configuration loaded from sqlbuild_project.toml.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("adapter"), + FieldSpec::new("default_target", FieldDefault::None), + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connections", FieldDefault::EmptyDict), + FieldSpec::new( + "settings", + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), + ), + FieldSpec::new( + "scopes", + FieldDefault::Call("sqlbuild._native", "ScopesConfig"), + ), + FieldSpec::new( + "references", + FieldDefault::Call("sqlbuild._native", "ReferencesConfig"), + ), + FieldSpec::new("cost", FieldDefault::Call("sqlbuild._native", "CostConfig")), + FieldSpec::new( + "constants", + FieldDefault::Call("sqlbuild._native", "ConstantsConfig"), + ), + FieldSpec::new( + "cursors", + FieldDefault::Call("sqlbuild._native", "CursorsConfig"), + ), + FieldSpec::new( + "microbatches", + FieldDefault::Call("sqlbuild._native", "MicrobatchesConfig"), + ), + FieldSpec::new( + "defaults", + FieldDefault::Call("sqlbuild._native", "DefaultsConfig"), + ), + FieldSpec::new("diff", FieldDefault::Call("sqlbuild._native", "DiffConfig")), + FieldSpec::new( + "materialization_defaults", + FieldDefault::Call("sqlbuild._native", "MaterializationDefaultsConfig"), + ), + FieldSpec::new("path_defaults", FieldDefault::EmptyDict), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new("targets", FieldDefault::EmptyDict), + FieldSpec::new( + "janitor", + FieldDefault::Call("sqlbuild._native", "JanitorConfig"), + ), + FieldSpec::new( + "migrations", + FieldDefault::Call("sqlbuild._native", "MigrationsConfig"), + ), + FieldSpec::new( + "snapshots", + FieldDefault::Call("sqlbuild._native", "SnapshotsConfig"), + ), + FieldSpec::new( + "scenario", + FieldDefault::Call("sqlbuild._native", "ScenarioConfig"), + ), + FieldSpec::new("dbt", FieldDefault::Call("sqlbuild._native", "DbtConfig")), + FieldSpec::new( + "sinks", + FieldDefault::Call("sqlbuild._native", "SinksConfig"), + ), + ], +}; +const R_LOCAL_CONFIG: RecordSpec = RecordSpec { + name: "LocalConfig", + doc: "Local developer overrides from sqlbuild_local.toml.", + fields: &[ + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("adapter", FieldDefault::None), + FieldSpec::new("connection", FieldDefault::EmptyDict), + FieldSpec::new("connections", FieldDefault::EmptyDict), + FieldSpec::new("targets", FieldDefault::EmptyDict), + FieldSpec::new( + "settings", + FieldDefault::Call("sqlbuild._native", "SettingsConfig"), + ), + FieldSpec::new("setting_overrides", FieldDefault::EmptyFrozenset), + FieldSpec::new("vars", FieldDefault::EmptyDict), + FieldSpec::new( + "dbt", + FieldDefault::Call("sqlbuild._native", "LocalDbtConfig"), + ), + FieldSpec::new( + "scenario", + FieldDefault::Call("sqlbuild._native", "ScenarioConfig"), + ), + ], +}; +const R_SOURCE_LOCATION: RecordSpec = RecordSpec { + name: "SourceLocation", + doc: "An authored source location for compiler diagnostics.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::new("end_line", FieldDefault::None), + FieldSpec::new("end_column", FieldDefault::None), + ], +}; +const R_SEED_CSV_SETTINGS: RecordSpec = RecordSpec { + name: "SeedCsvSettings", + doc: "CSV reader settings for a seed file.", + fields: &[ + FieldSpec::new("delimiter", FieldDefault::None), + FieldSpec::new("quotechar", FieldDefault::None), + FieldSpec::new("doublequote", FieldDefault::None), + FieldSpec::new("escapechar", FieldDefault::None), + FieldSpec::new("skipinitialspace", FieldDefault::None), + FieldSpec::new("lineterminator", FieldDefault::None), + FieldSpec::new("encoding", FieldDefault::None), + FieldSpec::new("na_values", FieldDefault::None), + FieldSpec::new("keep_default_na", FieldDefault::None), + ], +}; +const R_SCHEMA_AUDIT_INSTANCE: RecordSpec = RecordSpec { + name: "SchemaAuditInstance", + doc: "One audit instance attached to model, column, or seed metadata.", + fields: &[ + FieldSpec::required("definition_name"), + FieldSpec::new("arguments", FieldDefault::EmptyDict), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("severity", FieldDefault::None), + FieldSpec::new("run_scope", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + FieldSpec::new("thresholds", FieldDefault::None), + FieldSpec::new("minimum_samples", FieldDefault::None), + FieldSpec::new("evidence_limit", FieldDefault::None), + FieldSpec { + name: "location", + default: FieldDefault::None, + compared: false, + shown: false, + }, + ], +}; +const R_SCHEMA_COLUMN: RecordSpec = RecordSpec { + name: "SchemaColumn", + doc: "One declared model, seed, or source column entry.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("type", FieldDefault::None), + FieldSpec::new("nullable", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + FieldSpec::new("location", FieldDefault::None), + FieldSpec::new("migrate_from", FieldDefault::None), + ], +}; +const R_SCHEMA_DYNAMIC_COLUMN_FAMILY: RecordSpec = RecordSpec { + name: "SchemaDynamicColumnFamily", + doc: "One declared family of columns generated by a runtime-dynamic pivot.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("pivot_column"), + FieldSpec::required("value_column"), + FieldSpec::required("aggregate"), + FieldSpec::required("type"), + FieldSpec::new("name_pattern", FieldDefault::None), + ], +}; +const R_SCHEMA_MODEL_ENTRY: RecordSpec = RecordSpec { + name: "SchemaModelEntry", + doc: "One model metadata entry normalized from MODEL(...).", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("model_schema", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("type_enforcement", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("dynamic_columns", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + ], +}; +const R_SCHEMA_SEED_ENTRY: RecordSpec = RecordSpec { + name: "SchemaSeedEntry", + doc: "One seed metadata entry from seed YAML.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new( + "csv_settings", + FieldDefault::Call("sqlbuild._native", "SeedCsvSettings"), + ), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("tags", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + ], +}; +const R_INTEGRATION_LOADER_CONFIG: RecordSpec = RecordSpec { + name: "IntegrationLoaderConfig", + doc: "Typed configuration for one declarative integration loader.", + fields: &[FieldSpec::required("kind"), FieldSpec::required("config")], +}; +const R_SOURCE_FRESHNESS_AGE_POLICY: RecordSpec = RecordSpec { + name: "SourceFreshnessAgePolicy", + doc: "Age-based source freshness warning/error thresholds.", + fields: &[ + FieldSpec::new("warn_after", FieldDefault::None), + FieldSpec::new("error_after", FieldDefault::None), + ], +}; +const R_SOURCE_FRESHNESS_CONFIG: RecordSpec = RecordSpec { + name: "SourceFreshnessConfig", + doc: "Configured source freshness observation for planning.", + fields: &[ + FieldSpec::required("strategy"), + FieldSpec::new("value_kind", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("query", FieldDefault::None), + FieldSpec::new("filter", FieldDefault::None), + FieldSpec::new("lag_tolerance", FieldDefault::None), + FieldSpec::new("age_policy", FieldDefault::None), + ], +}; +const R_SOURCE_COLUMN_ENTRY: RecordSpec = RecordSpec { + name: "SourceColumnEntry", + doc: "One source column entry from sources/*.yml.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("type", FieldDefault::None), + FieldSpec::new("nullable", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + ], +}; +const R_SOURCE_ENTRY: RecordSpec = RecordSpec { + name: "SourceEntry", + doc: "One source declaration from sources/*.yml.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("table", FieldDefault::None), + FieldSpec::new("loader", FieldDefault::None), + FieldSpec::new("managed", FieldDefault::False), + FieldSpec::new("integration_loader", FieldDefault::None), + FieldSpec::new("freshness", FieldDefault::None), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("load_batch_size", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("expression", FieldDefault::None), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("type_enforcement", FieldDefault::None), + FieldSpec::new("contract", FieldDefault::None), + FieldSpec::new("meta", FieldDefault::EmptyDict), + FieldSpec::new("columns", FieldDefault::EmptyTuple), + FieldSpec::new("audits", FieldDefault::EmptyTuple), + ], +}; +const R_LOADER_DESTINATION_PARTS: RecordSpec = RecordSpec { + name: "LoaderDestinationParts", + doc: "Parsed warehouse relation parts for a loader destination.", + fields: &[ + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("table"), + ], +}; + +/// The project configuration and contract value records. +pub(crate) const CONTRACT_RECORDS: &[RecordSpec] = &[ + R_CLONE_POLICY, + R_LOCAL_CLONE_POLICY, + R_AUTHORED_TIME_TRAVEL_RETENTION, + R_MATERIALIZATION_RETENTION_DEFAULTS, + R_MATERIALIZATION_DEFAULTS_CONFIG, + R_RESOLVED_TIME_TRAVEL_RETENTION, + R_RESOLVED_TABLE_TYPE, + R_EXECUTION_LIMITS_CONFIG, + R_TARGET_DIFF_CONFIG, + R_TARGET_WAREHOUSES_CONFIG, + R_TARGET_CONFIG, + R_LOCAL_TARGET_CONFIG, + R_SETTINGS_CONFIG, + R_SCOPES_CONFIG, + R_REFERENCES_CONFIG, + R_COST_CONFIG, + R_CONSTANTS_CONFIG, + R_FUTURE_CURSORS_CONFIG, + R_START_CURSORS_CONFIG, + R_CURSORS_CONFIG, + R_MICROBATCH_LIMITS_CONFIG, + R_MICROBATCHES_CONFIG, + R_DEFAULTS_CONFIG, + R_DIFF_CONFIG, + R_JANITOR_CONFIG, + R_MIGRATIONS_CONFIG, + R_SNAPSHOTS_CONFIG, + R_SCENARIO_SNAPSHOT_LIMITS_CONFIG, + R_SCENARIO_CONFIG, + R_DBT_CONFIG, + R_LOCAL_DBT_CONFIG, + R_LIFECYCLE_EVENT_SINK_FILTER_CONFIG, + R_LIFECYCLE_EVENT_SINKS_CONFIG, + R_SINKS_CONFIG, + R_PROJECT_CONFIG, + R_LOCAL_CONFIG, + R_SOURCE_LOCATION, + R_SEED_CSV_SETTINGS, + R_SCHEMA_AUDIT_INSTANCE, + R_SCHEMA_COLUMN, + R_SCHEMA_DYNAMIC_COLUMN_FAMILY, + R_SCHEMA_MODEL_ENTRY, + R_SCHEMA_SEED_ENTRY, + R_INTEGRATION_LOADER_CONFIG, + R_SOURCE_FRESHNESS_AGE_POLICY, + R_SOURCE_FRESHNESS_CONFIG, + R_SOURCE_COLUMN_ENTRY, + R_SOURCE_ENTRY, + R_LOADER_DESTINATION_PARTS, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 291232f396..5de21e3b7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -16,10 +16,10 @@ use crate::bindings::_helpers::sqltext::model_headers::{ColumnLocation, location const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; -const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SPEC_MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const IMPLICIT_COLUMN_CONTRACTS: &str = "implicit"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs index 8f4c9e76a3..b2a1074e97 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -16,29 +16,27 @@ use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; -const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const PROJECT_CONFIG_ERROR: &str = "ProjectConfigError"; const FILE_NAMES: [&str; 4] = [ - "sqlbuild_project.toml", + PROJECT_CONFIG_FILENAME, "sqlbuild_project.yml", "sqlbuild_local.toml", "sqlbuild_local.yml", ]; /// The module of each `StrEnum` a loaded config holds. const ENUM_MODULES: [(&str, &str); 10] = [ - ("ColumnContractMode", "sqlbuild.spec.contracts.types"), - ("FutureCursorAction", "sqlbuild.spec.contracts.types"), - ("MicrobatchLimitAction", "sqlbuild.spec.contracts.types"), - ("TableType", "sqlbuild.spec.contracts.types"), - ("TableTypeDowngradePolicy", "sqlbuild.spec.contracts.types"), - ("RetentionDecreasePolicy", "sqlbuild.spec.contracts.types"), - ( - "MissingMigrationOriginPolicy", - "sqlbuild.spec.contracts.types", - ), - ("EventExportSeverity", "sqlbuild.spec.contracts.types"), + ("ColumnContractMode", "sqlbuild._native"), + ("FutureCursorAction", "sqlbuild._native"), + ("MicrobatchLimitAction", "sqlbuild._native"), + ("TableType", "sqlbuild._native"), + ("TableTypeDowngradePolicy", "sqlbuild._native"), + ("RetentionDecreasePolicy", "sqlbuild._native"), + ("MissingMigrationOriginPolicy", "sqlbuild._native"), + ("EventExportSeverity", "sqlbuild._native"), ("AuditSeverity", "sqlbuild.compiler.auditing.types"), ("CollectionRendering", "sqlbuild._native"), ]; @@ -131,7 +129,7 @@ fn enum_member<'py>(py: Python<'py>, class: &str, value: &str) -> PyResult, value: &Bound<'_, PyAny>, declared_type: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs index 142313c100..1e961676eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::cost_command::refusal::cost_error; const DATETIME_MODULE: &str = "datetime"; -const DURATION_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const DURATION_MODELS: &str = "sqlbuild._native"; const MINUTE_TO_DAY_DURATION_UNITS: [&str; 3] = ["m", "h", "d"]; fn utc(py: Python<'_>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 80c115906e..6d4b909315 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -1,4 +1,4 @@ -//! One call's cursor values read from and built as `sqlbuild.cursor_algebra.models` objects. +//! One call's cursor values read from and built as the native cursor value records. use std::cell::RefCell; @@ -19,10 +19,11 @@ use crate::bindings::_helpers::cursor::values::{ timedelta_micros, }; -pub(crate) const MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; -const GRAIN_NAMES: [(&str, Grain); 6] = [ +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; +/// Cursor grain names, finest first. +pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ ("second", Grain::Second), ("minute", Grain::Minute), ("hour", Grain::Hour), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs index 7bd4590a5c..1fbc70322f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs @@ -19,7 +19,7 @@ const MONTHS_PER_YEAR: i64 = 12; /// `(years, months, days, hours, minutes, seconds)` of a duration like `1y6mo` or `30d`. #[pyfunction] #[pyo3(signature = (value))] -fn parse_cursor_duration<'py>( +pub(crate) fn parse_cursor_duration<'py>( py: Python<'py>, value: &str, ) -> PyResult>> { @@ -86,7 +86,7 @@ pub(crate) fn shifted_moment<'py>( /// `moment` moved forward (`direction` 1) or back (`direction` -1) by `duration`. #[pyfunction] #[pyo3(signature = (*, moment, duration, direction))] -fn shift_by_cursor_duration<'py>( +pub(crate) fn shift_by_cursor_duration<'py>( moment: &Bound<'py, PyAny>, duration: &Bound<'py, PyAny>, direction: i64, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs index 12aa7365ea..f522059e2b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/intervals.rs @@ -46,7 +46,7 @@ fn python_intervals<'py>( /// Raise `CursorAlgebraError` unless the ends form a valid aligned interval. #[pyfunction(name = "validate_cursor_interval")] #[pyo3(signature = (*, start, end, grain))] -fn interval_validation( +pub(crate) fn interval_validation( start: &Bound<'_, PyAny>, end: &Bound<'_, PyAny>, grain: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs index b90720c229..c86bdc51d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod artifact; +pub(crate) mod projection; pub(crate) mod registration; pub(crate) mod rows; pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs new file mode 100644 index 0000000000..d697b98371 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/projection.rs @@ -0,0 +1,300 @@ +//! Dagster and Rivers asset and check plans over a DAG artifact, holding the exact translator inputs. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyMapping, PySet, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::dag_artifact::main::dag_asset_plans::dag_asset_plans; +use sqlbuild_analysis::dag_artifact::main::dag_asset_spec_kinds::dag_asset_spec_kinds; +use sqlbuild_analysis::dag_artifact::main::dag_check_plans::dag_check_plans; +use sqlbuild_analysis::dag_artifact::main::dag_selected_asset_nodes::dag_selected_asset_nodes; +use sqlbuild_analysis::dag_artifact::models::{ + AssetNodePlan, AssetProjectionPolicy, AssetSpecKindPolicy, CheckKindFilter, CheckPlan, + DagProjectionCheck, DagProjectionGraph, DagProjectionNode, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::dag_artifact::selection::selected_node_ids; + +const DAGSTER_CONSTANTS: &str = "sqlbuild.integrations.dagster.constants"; +const RIVERS_CONSTANTS: &str = "sqlbuild.integrations.rivers.constants"; +const PROJECT_NAME_KEY: &str = "project_name"; +const DEFAULT_MATERIALIZATION: &str = "table"; + +/// The artifact's nodes, edges and checks as Python objects alongside their projection facts. +struct ArtifactGraph<'py> { + project_name: Bound<'py, PyAny>, + nodes: Vec>, + checks: Vec>, + graph: DagProjectionGraph, +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn listed<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} + +fn projection_node(node: &Bound<'_, PyAny>) -> PyResult { + Ok(DagProjectionNode { + id: text(&node.get_item("id")?)?, + kind: text(&node.call_method1("get", ("kind",))?)?, + }) +} + +fn projection_check(check: &Bound<'_, PyAny>) -> PyResult { + let empty: Bound<'_, PyTuple> = PyTuple::empty(check.py()); + Ok(DagProjectionCheck { + kind: text(&check.call_method1("get", ("kind",))?)?, + checked_asset_ids: listed(&check.call_method1("get", ("checked_asset_ids", empty))?)? + .iter() + .map(text) + .collect::>()?, + }) +} + +/// The artifact read the way the integrations index it: `nodes`, `edges` and `checks` by key. +fn artifact_graph<'py>(dag: &Bound<'py, PyAny>, with_checks: bool) -> PyResult> { + let nodes: Vec> = listed(&dag.get_item("nodes")?)?; + let checks: Vec> = if with_checks { + listed(&dag.get_item("checks")?)? + } else { + Vec::new() + }; + let edges: Vec<(String, String)> = if with_checks { + Vec::new() + } else { + listed(&dag.get_item("edges")?)? + .iter() + .map(|edge| { + Ok(( + text(&edge.get_item("from_id")?)?, + text(&edge.get_item("to_id")?)?, + )) + }) + .collect::>()? + }; + Ok(ArtifactGraph { + project_name: dag.call_method1("get", (PROJECT_NAME_KEY,))?, + graph: DagProjectionGraph { + nodes: nodes.iter().map(projection_node).collect::>()?, + edges, + checks: checks + .iter() + .map(projection_check) + .collect::>()?, + }, + nodes, + checks, + }) +} + +/// `{**node, "project_name": project_name}`. +fn with_project_name<'py>( + node: &Bound<'py, PyAny>, + project_name: &Bound<'py, PyAny>, +) -> PyResult> { + let translated: Bound<'py, PyDict> = PyDict::new(node.py()); + translated.update(node.downcast::()?)?; + translated.set_item(PROJECT_NAME_KEY, project_name)?; + Ok(translated.into_any()) +} + +/// One `{**node, "project_name": ...}` per artifact node, in artifact order. +fn translated_nodes<'py>(artifact: &ArtifactGraph<'py>) -> PyResult>> { + artifact + .nodes + .iter() + .map(|node| with_project_name(node, &artifact.project_name)) + .collect() +} + +fn kind_set(py: Python<'_>, (module, name): (&str, &str)) -> PyResult> { + listed(&imported(py, module, name)?)? + .iter() + .map(text) + .collect() +} + +fn constant_text(py: Python<'_>, (module, name): (&str, &str)) -> PyResult { + text(&imported(py, module, name)?) +} + +fn picked<'py>(objects: &[Bound<'py, PyAny>], indexes: &[usize]) -> Vec> { + indexes + .iter() + .map(|index| objects[*index].clone()) + .collect() +} + +/// `str(node.get("materialization_type") or "table")`. +fn materialization_type(node: &Bound<'_, PyAny>) -> PyResult { + let value: Bound<'_, PyAny> = node.call_method1("get", ("materialization_type",))?; + if value.is_truthy()? { + text(&value) + } else { + Ok(DEFAULT_MATERIALIZATION.to_owned()) + } +} + +/// Dagster asset specs to build: each translated asset node, its upstream nodes and its kinds. +#[pyfunction] +#[pyo3(signature = (dag, /))] +fn dagster_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = dag.py(); + let artifact: ArtifactGraph<'py> = artifact_graph(dag, false)?; + let upstream_nodes: Vec> = translated_nodes(&artifact)?; + let policy: AssetProjectionPolicy = AssetProjectionPolicy { + asset_kinds: kind_set(py, (DAGSTER_CONSTANTS, "DAGSTER_ASSET_NODE_KINDS"))?, + upstream_asset_kinds_only: false, + }; + let kinds: AssetSpecKindPolicy = AssetSpecKindPolicy { + model_kind: constant_text(py, (DAGSTER_CONSTANTS, "MODEL_NODE_KIND"))?, + view_materialization: constant_text(py, (DAGSTER_CONSTANTS, "VIEW_MATERIALIZATION_TYPE"))?, + direct_kinds: kind_set(py, (DAGSTER_CONSTANTS, "DAGSTER_DIRECT_KIND_NODE_KINDS"))?, + }; + let plans: Vec> = dag_asset_plans(&artifact.graph, &policy) + .iter() + .map(|plan: &AssetNodePlan| { + let node: &Bound<'py, PyAny> = &artifact.nodes[plan.node]; + let spec_kinds: Vec = dag_asset_spec_kinds( + ( + &artifact.graph.nodes[plan.node].kind, + &materialization_type(node)?, + ), + &kinds, + ); + PyTuple::new( + py, + [ + with_project_name(node, &artifact.project_name)?, + PyList::new(py, picked(&upstream_nodes, &plan.upstream))?.into_any(), + PySet::new(py, spec_kinds)?.into_any(), + ], + ) + }) + .collect::>()?; + PyList::new(py, plans) +} + +/// Rivers asset definitions to build: each translated asset node and its upstream asset nodes. +#[pyfunction] +#[pyo3(signature = (dag, /))] +fn rivers_asset_plans<'py>(dag: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = dag.py(); + let artifact: ArtifactGraph<'py> = artifact_graph(dag, false)?; + let policy: AssetProjectionPolicy = AssetProjectionPolicy { + asset_kinds: kind_set(py, (RIVERS_CONSTANTS, "RIVERS_ASSET_NODE_KINDS"))?, + upstream_asset_kinds_only: true, + }; + let plans: Vec> = dag_asset_plans(&artifact.graph, &policy) + .iter() + .map(|plan: &AssetNodePlan| { + PyTuple::new( + py, + [ + with_project_name(&artifact.nodes[plan.node], &artifact.project_name)?, + PyList::new(py, picked(&artifact.nodes, &plan.upstream))?.into_any(), + ], + ) + }) + .collect::>()?; + PyList::new(py, plans) +} + +/// `str(check.get("path"))`, or `None` without a path. +fn check_description<'py>(check: &Bound<'py, PyAny>) -> PyResult> { + let path: Bound<'py, PyAny> = check.call_method1("get", ("path",))?; + if path.is_none() { + Ok(path) + } else { + Ok(path.str()?.into_any()) + } +} + +/// Dagster check specs to build: check, kind exposed, translated checked nodes and description. +#[pyfunction] +#[pyo3(signature = (dag, /, *, include_scenarios, only_scenarios))] +fn dagster_check_plans<'py>( + dag: &Bound<'py, PyAny>, + include_scenarios: bool, + only_scenarios: bool, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let artifact: ArtifactGraph<'py> = artifact_graph(dag, true)?; + let checked_nodes: Vec> = translated_nodes(&artifact)?; + let scenario: BTreeSet = BTreeSet::from([constant_text( + py, + (DAGSTER_CONSTANTS, "SCENARIO_CHECK_KIND"), + )?]); + let filter: CheckKindFilter = CheckKindFilter { + excluded: if include_scenarios || only_scenarios { + BTreeSet::new() + } else { + scenario.clone() + }, + required: only_scenarios.then_some(scenario), + }; + let plans: Vec> = dag_check_plans(&artifact.graph, &filter) + .iter() + .map(|plan: &CheckPlan| { + let check: &Bound<'py, PyAny> = &artifact.checks[plan.check]; + PyTuple::new( + py, + [ + check.clone(), + plan.included.into_pyobject(py)?.to_owned().into_any(), + PyList::new(py, picked(&checked_nodes, &plan.nodes))?.into_any(), + check_description(check)?, + ], + ) + }) + .collect::>()?; + PyList::new(py, plans) +} + +/// The translated nodes Dagster owns as assets among those `select` minus `exclude` matches. +#[pyfunction] +#[pyo3(signature = (dag, /, *, select, exclude = None))] +fn dagster_selected_asset_nodes<'py>( + dag: &Bound<'py, PyAny>, + select: &str, + exclude: Option<&str>, +) -> PyResult> { + let py: Python<'py> = dag.py(); + let selected: BTreeSet = selected_node_ids(dag, (select, exclude))?; + let nodes: Vec> = + listed(&dag.call_method1("get", ("nodes", PyTuple::empty(py)))?)?; + let project_name: Bound<'py, PyAny> = dag.call_method1("get", (PROJECT_NAME_KEY,))?; + let graph: DagProjectionGraph = DagProjectionGraph { + nodes: nodes.iter().map(selection_node).collect::>()?, + ..DagProjectionGraph::default() + }; + let asset_kinds: BTreeSet = + kind_set(py, (DAGSTER_CONSTANTS, "DAGSTER_ASSET_NODE_KINDS"))?; + let translated: Vec> = + dag_selected_asset_nodes(&graph, &selected, &asset_kinds) + .iter() + .map(|index| with_project_name(&nodes[*index], &project_name)) + .collect::>()?; + PyList::new(py, translated) +} + +/// A node read the way selection reads it: `str(node.get("id"))` and `str(node.get("kind"))`. +fn selection_node(node: &Bound<'_, PyAny>) -> PyResult { + Ok(DagProjectionNode { + id: text(&node.call_method1("get", ("id",))?)?, + kind: text(&node.call_method1("get", ("kind",))?)?, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(dagster_asset_plans, module)?)?; + module.add_function(wrap_pyfunction!(dagster_check_plans, module)?)?; + module.add_function(wrap_pyfunction!(dagster_selected_asset_nodes, module)?)?; + module.add_function(wrap_pyfunction!(rivers_asset_plans, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs index 55b6276cfe..ff1a3af1a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/registration.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::dag_artifact::{artifact, selection}; +use crate::bindings::_helpers::dag_artifact::{artifact, projection, selection}; use crate::bindings::_helpers::dagster_invocation; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { artifact::register(module)?; + projection::register(module)?; selection::register(module)?; dagster_invocation::registration::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs index 70099991bc..8fcf88a7ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/selection.rs @@ -1,10 +1,12 @@ -//! Dagster asset selection: SQLBuild selectors resolved over a DAG artifact mapping. +//! DAG artifact loading and SQLBuild selectors resolved over a DAG artifact mapping. + +use std::collections::BTreeSet; use pyo3::exceptions::PyOSError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::dag_artifact::errors::DagSelectionError; use sqlbuild_analysis::dag_artifact::main::resolve_dag_selection::resolve_dag_selection as resolve; @@ -106,18 +108,12 @@ fn selection_error(py: Python<'_>, error: DagSelectionError, raw: &str) -> PyErr } /// Ids of DAG artifact nodes `select` matches minus those `exclude` matches. -#[pyfunction] -#[pyo3(signature = (*, dag, select, exclude = None))] -fn resolve_dag_selection<'py>( - dag: &Bound<'py, PyAny>, - select: &str, - exclude: Option<&str>, -) -> PyResult> { - let py: Python<'py> = dag.py(); +pub(crate) fn selected_node_ids( + dag: &Bound<'_, PyAny>, + (select, exclude): (&str, Option<&str>), +) -> PyResult> { let graph: DagSelectionGraph = selection_graph(dag)?; - let selected = - resolve(&graph, (select, exclude)).map_err(|error| selection_error(py, error, select))?; - PyFrozenSet::new(py, selected) + resolve(&graph, (select, exclude)).map_err(|error| selection_error(dag.py(), error, select)) } fn raised(error_class: &Bound<'_, PyAny>, message: String, cause: Option) -> PyErr { @@ -212,7 +208,6 @@ fn load_sqlbuild_dag<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_dag_selection, module)?)?; module.add_function(wrap_pyfunction!(load_sqlbuild_dag, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs index 9d1391a8de..6ad661843c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod arguments; pub(crate) mod failures; pub(crate) mod registration; pub(crate) mod results; +pub(crate) mod translator; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs index 58ff98354a..0700489b8f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::dagster_invocation::{arguments, failures, results}; +use crate::bindings::_helpers::dagster_invocation::{arguments, failures, results, translator}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(arguments::dagster_cli_arguments, module)?)?; @@ -25,5 +25,30 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!(failures::dagster_failure, module)?)?; - module.add_function(wrap_pyfunction!(failures::dagster_live_failure, module)?) + module.add_function(wrap_pyfunction!(failures::dagster_live_failure, module)?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_asset_key_path, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_group_name, + module + )?)?; + module.add_function(wrap_pyfunction!(translator::dagster_default_tags, module)?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_metadata, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_description, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_check_name, + module + )?)?; + module.add_function(wrap_pyfunction!( + translator::dagster_default_check_metadata, + module + )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs new file mode 100644 index 0000000000..671f34c8e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/translator.rs @@ -0,0 +1,238 @@ +//! The default `SqlBuildDagsterTranslator` mappings its overridable methods call. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_core::dagster_invocation::constants::{DEFAULT_ASSET_KIND, DEFAULT_ASSET_NAME}; +use sqlbuild_core::dagster_invocation::main::asset_description::asset_description; +use sqlbuild_core::dagster_invocation::main::dagster_name_segment::dagster_name_segment; +use sqlbuild_core::dagster_invocation::main::dagster_tag_key::dagster_tag_key; +use sqlbuild_core::dagster_invocation::main::fallback_asset_description::fallback_asset_description; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::dagster_invocation::access::{CONSTANTS_MODULE, get, value_or}; + +const ABC_MODULE: &str = "collections.abc"; +const BUILTINS_MODULE: &str = "builtins"; +const KIND_TAG: &str = "sqlbuild/kind"; +const DEFAULT_GROUP: &str = "sqlbuild"; +const DEFAULT_CHECK_PART: &str = "check"; +const CHECK_PART_SEPARATOR: &str = "__"; +/// Node fields copied into asset metadata when present, in order. +const METADATA_KEYS: [&str; 12] = [ + "path", + "target", + "description", + "sql", + "columns", + "column_lineage", + "group", + "language", + "materialization_type", + "return_kind", + "loader", + "meta", +]; + +/// `str(value)`. +fn text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string_lossy().into_owned()) +} + +/// `get(key) or default` read lazily: `default` is not consulted when the value is truthy. +fn text_or_default<'py>(mapping: &Bound<'py, PyAny>, key: &str, default: &str) -> PyResult { + let value: Bound<'py, PyAny> = get(mapping, key)?; + if value.is_truthy()? { + return text(&value); + } + Ok(default.to_owned()) +} + +fn abc_instance(value: &Bound<'_, PyAny>, class: &str) -> PyResult { + value.is_instance(&imported(value.py(), ABC_MODULE, class)?) +} + +/// A source node's non-empty, non-text `meta.dagster.asset_key` sequence. +fn authored_asset_key<'py>(node: &Bound<'py, PyAny>) -> PyResult>> { + let source_kind: String = + imported(node.py(), CONSTANTS_MODULE, "SOURCE_NODE_KIND")?.extract()?; + if text(&get(node, "kind")?)? != source_kind { + return Ok(None); + } + let meta: Bound<'py, PyAny> = get(node, "meta")?; + if !abc_instance(&meta, "Mapping")? { + return Ok(None); + } + let dagster: Bound<'py, PyAny> = get(&meta, "dagster")?; + if !abc_instance(&dagster, "Mapping")? { + return Ok(None); + } + let asset_key: Bound<'py, PyAny> = get(&dagster, "asset_key")?; + if !abc_instance(&asset_key, "Sequence")? + || asset_key.is_instance_of::() + || asset_key.is_instance_of::() + || !asset_key.is_truthy()? + { + return Ok(None); + } + Ok(Some(asset_key)) +} + +/// The asset key path of a node: its authored Dagster key, or its SQLBuild asset key. +#[pyfunction] +pub(crate) fn dagster_default_asset_key_path(node: &Bound<'_, PyAny>) -> PyResult> { + let parts: Bound<'_, PyAny> = match authored_asset_key(node)? { + Some(parts) => parts, + None => node.get_item("asset_key")?, + }; + parts.try_iter()?.map(|part| text(&part?)).collect() +} + +/// The node's group, project name or `sqlbuild`, with `-` read as `_`. +#[pyfunction] +pub(crate) fn dagster_default_group_name(node: &Bound<'_, PyAny>) -> PyResult { + let group: Bound<'_, PyAny> = get(node, "group")?; + let chosen: String = if group.is_truthy()? { + text(&group)? + } else { + text_or_default(node, "project_name", DEFAULT_GROUP)? + }; + Ok(chosen.replace('-', "_")) +} + +/// The node kind tag and one empty-valued tag per SQLBuild tag with a usable key. +#[pyfunction] +pub(crate) fn dagster_default_tags<'py>(node: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = node.py(); + let tags: Bound<'py, PyDict> = PyDict::new(py); + tags.set_item( + KIND_TAG, + text(&value_or(node, "kind", PyString::new(py, "").into_any())?)?, + )?; + for tag in value_or(node, "tags", PyTuple::empty(py).into_any())?.try_iter()? { + let key: String = dagster_tag_key(&text(&tag?)?); + if !key.is_empty() { + tags.set_item(key, "")?; + } + } + Ok(tags) +} + +/// The SQLBuild identity fields and every present node field Dagster shows as metadata. +#[pyfunction] +pub(crate) fn dagster_default_metadata<'py>( + node: &Bound<'py, PyAny>, +) -> PyResult> { + let metadata: Bound<'py, PyDict> = PyDict::new(node.py()); + metadata.set_item("sqlbuild_id", get(node, "id")?)?; + metadata.set_item("sqlbuild_name", get(node, "name")?)?; + metadata.set_item("sqlbuild_kind", get(node, "kind")?)?; + metadata.set_item("sqlbuild_selector", get(node, "name")?)?; + for key in METADATA_KEYS { + if node.contains(key)? { + metadata.set_item(key, node.get_item(key)?)?; + } + } + Ok(metadata) +} + +/// The generated description of a node without one. +fn fallback_description(node: &Bound<'_, PyAny>) -> PyResult { + let kind: String = text_or_default(node, "kind", DEFAULT_ASSET_KIND)?; + let name: String = text_or_default(node, "name", DEFAULT_ASSET_NAME)?; + let target: Bound<'_, PyAny> = get(node, "target")?; + let qualified_name: String = if abc_instance(&target, "Mapping")? { + text_or_default(&target, "qualified_name", "")? + } else { + String::new() + }; + let materialization: Bound<'_, PyAny> = get(node, "materialization_type")?; + let materialization: Option = if materialization.is_none() { + None + } else { + Some( + imported(node.py(), BUILTINS_MODULE, "format")? + .call1((materialization,))? + .str()? + .to_string_lossy() + .into_owned(), + ) + }; + Ok(fallback_asset_description( + &kind, + &name, + &qualified_name, + materialization.as_deref(), + )) +} + +/// A string field, or `None` when the field is not text. +fn text_field(node: &Bound<'_, PyAny>, key: &str) -> PyResult> { + let value: Bound<'_, PyAny> = get(node, key)?; + match value.downcast::() { + Ok(value) => Ok(Some(value.to_string_lossy().into_owned())), + Err(_) => Ok(None), + } +} + +/// The node's description, or a generated one, followed by its SQL or source file. +#[pyfunction] +pub(crate) fn dagster_default_description(node: &Bound<'_, PyAny>) -> PyResult { + let description: Bound<'_, PyAny> = get(node, "description")?; + let description: String = if description.is_truthy()? { + text(&description)? + } else { + fallback_description(node)? + }; + let sql: Option = text_field(node, "sql")?; + let path: Option = if sql + .as_deref() + .is_some_and(|sql| !python_strip(sql).is_empty()) + { + None + } else { + text_field(node, "path")? + }; + Ok(asset_description( + &description, + sql.as_deref(), + path.as_deref(), + )) +} + +/// `kind__name[__column or target]`, each part made Dagster-safe. +#[pyfunction] +pub(crate) fn dagster_default_check_name(check: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = check.py(); + let default = || PyString::new(py, DEFAULT_CHECK_PART).into_any(); + let mut parts: Vec = vec![ + text(&value_or(check, "kind", default())?)?, + text(&value_or(check, "name", default())?)?, + ]; + for attached in ["attached_column_name", "attached_target_name"] { + if !get(check, attached)?.is_none() { + parts.push(text(&check.get_item(attached)?)?); + break; + } + } + Ok(parts + .iter() + .filter(|part| !part.is_empty()) + .map(|part| dagster_name_segment(part)) + .collect::>() + .join(CHECK_PART_SEPARATOR)) +} + +/// The SQLBuild identity fields of one check. +#[pyfunction] +pub(crate) fn dagster_default_check_metadata<'py>( + check: &Bound<'py, PyAny>, +) -> PyResult> { + let metadata: Bound<'py, PyDict> = PyDict::new(check.py()); + metadata.set_item("sqlbuild_check_id", get(check, "id")?)?; + metadata.set_item("sqlbuild_check_kind", get(check, "kind")?)?; + metadata.set_item("sqlbuild_check_name", get(check, "name")?)?; + metadata.set_item("sqlbuild_check_selector", get(check, "id")?)?; + Ok(metadata) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs index b5b97e059d..6d67f1d171 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/auto_init.rs @@ -17,15 +17,12 @@ use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_CONSTANTS: &str = "sqlbuild.compiler.discovery.constants"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; fn project_config_found(directory: &Bound<'_, PyAny>) -> PyResult { - let name: Bound<'_, PyAny> = imported( - directory.py(), - DISCOVERY_CONSTANTS, - "PROJECT_CONFIG_FILENAME", - )?; + let name: Bound<'_, PyAny> = + pyo3::types::PyString::new(directory.py(), PROJECT_CONFIG_FILENAME).into_any(); directory .call_method1("__truediv__", (name,))? .call_method0("exists")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs index ae69b05319..da84c3fb43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs @@ -25,7 +25,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const PROFILE_RENDER_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.render"; -const PROJECT_FILE: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; const DEFAULT_OUTPUT_DIR: &str = "sqlbuild_project"; const PROFILES_FILE: &str = "profiles.yml"; const PROFILES_DIR_ENV_VAR: &str = "DBT_PROFILES_DIR"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs index 44d7aa4732..da8afaf6f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const DBT_REF_KIND: &str = "dbt_ref"; const RELATION_OPTIONS: (&[&str], &[&str]) = (&["manifest", "ref_kind", "ref_name", "ref_package"], &[]); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs index 79b76ab4ca..a6dc7f2cbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod pipeline_prologue; pub(crate) mod plan; pub(crate) mod plan_output; pub(crate) mod profile; +pub(crate) mod profile_coercion; pub(crate) mod profile_connection; pub(crate) mod profile_load; pub(crate) mod reference_resolver; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index e2c81bcc2a..7a5d4ff0ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -232,12 +232,7 @@ fn execution_limits<'py>( ) -> PyResult> { let py = discovered.py(); if target.is_none() { - return imported( - py, - "sqlbuild.spec.contracts.models", - "ExecutionLimitsConfig", - )? - .call0(); + return imported(py, "sqlbuild._native", "ExecutionLimitsConfig")?.call0(); } call_with( &imported(py, "sqlbuild._native", "resolve_target_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 61ba953612..946cd50284 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const HOOKS: &str = "sqlbuild.runtime.contracts.models"; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); const ATTACH_OPTIONS: (&[&str], &[&str]) = ( @@ -204,14 +204,8 @@ pub(crate) fn build_sqlbuild_plan_output<'py>( ); match result { Err(error) - if error.is_instance( - py, - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, - ) => + if error + .is_instance(py, &imported(py, "sqlbuild._native", "PlannerInputError")?) => { call_with( &imported(py, "sqlbuild._native", "build_display_only_sqlbuild_plan")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs new file mode 100644 index 0000000000..1c679f6324 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_coercion.rs @@ -0,0 +1,71 @@ +//! dbt profile `as_bool` and `as_number` Jinja filters. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyFloat, PyInt, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::dbt_profile::main::dbt_bool_text::dbt_bool_text; +use sqlbuild_config::dbt_profile::main::dbt_number_text::dbt_number_text; +use sqlbuild_config::dbt_profile::models::DbtNumberParse; + +use crate::bindings::_helpers::dbt_interop::profile_connection::profile_error; + +/// A `DbtProfileError` for a value that does not coerce to `target`. +fn coercion_error(value: &Bound<'_, PyAny>, target: &str) -> PyResult { + let rendered: String = value.repr()?.to_str()?.to_owned(); + Ok(profile_error( + value.py(), + format!("Cannot coerce {rendered} to {target}"), + )) +} + +/// A bool, or a string dbt reads as one. +#[pyfunction] +#[pyo3(signature = (value))] +fn coerce_dbt_profile_bool<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(value.clone()); + } + if let Ok(text) = value.downcast::() + && let Some(flag) = dbt_bool_text(text.to_str()?) + { + return Ok(PyBool::new(value.py(), flag).to_owned().into_any()); + } + Err(coercion_error(value, "bool")?) +} + +/// An int or float, or a string parsed as one. +#[pyfunction] +#[pyo3(signature = (value))] +fn coerce_dbt_profile_number<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_instance_of::() { + return Err(coercion_error(value, "number")?); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(value.clone()); + } + let Ok(text) = value.downcast::() else { + return Err(coercion_error(value, "number")?); + }; + let (stripped, parse): (&str, DbtNumberParse) = dbt_number_text(text.to_str()?); + let parsed: PyResult> = match parse { + DbtNumberParse::Float => py.get_type::().call1((stripped,)), + DbtNumberParse::Integer => py.get_type::().call1((stripped,)), + }; + match parsed { + Err(error) if error.is_instance_of::(py) => { + let raised: PyErr = coercion_error(value, "number")?; + raised.set_cause(py, Some(error)); + Err(raised) + } + parsed => parsed, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(coerce_dbt_profile_bool, module)?)?; + module.add_function(wrap_pyfunction!(coerce_dbt_profile_number, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs index dbc4ebfe5e..83a7dc39ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/profile_load.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::dbt_interop::profile::raised; use crate::bindings::_helpers::dbt_interop::profile_connection::profile_error; const PROFILE_LOAD_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.load"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const DBT_MODELS: &str = "sqlbuild._native"; const DBT_PROJECT_FILE: &str = "dbt_project.yml"; const PROFILES_FILE: &str = "profiles.yml"; const DEFAULT_PROFILES_DIR: &str = "~/.dbt"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs index e9ef7b9675..d7803540f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/reference_resolver.rs @@ -12,8 +12,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const VALIDATE_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs index 02bb776394..f188970794 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/registration.rs @@ -14,6 +14,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { orchestration::register(module)?; plan::register(module)?; profile::register(module)?; + super::profile_coercion::register(module)?; super::profile_connection::register(module)?; super::reference_resolver::register(module)?; super::runtime_config::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 76a82c12f0..de3f8f48f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -37,7 +37,7 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( &[], &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs new file mode 100644 index 0000000000..e8e8571bbe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs @@ -0,0 +1,90 @@ +//! The `--debug` stderr line of one Python log record. + +use pyo3::exceptions::PyOverflowError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::diagnostic_routing::constants::SQL_TEXT_FIELD; +use sqlbuild_core::diagnostic_text::main::format_diagnostic_record::format_diagnostic_record; +use sqlbuild_core::diagnostic_text::models::DiagnosticRecordText; + +/// The record field holding an operation's duration in milliseconds. +const DURATION_FIELD: &str = "sqlbuild_duration_ms"; +const MILLISECONDS_PER_SECOND: i64 = 1000; + +/// Each context field shown as a `label=value` token, in display order. +const CONTEXT_TOKENS: [(&str, &str); 7] = [ + ("sqlbuild_phase", "phase"), + ("sqlbuild_kind", "kind"), + ("sqlbuild_action_name", "action"), + ("sqlbuild_audit_name", "audit"), + ("sqlbuild_column_name", "column"), + ("sqlbuild_window", "window"), + ("sqlbuild_status", "status"), +]; + +/// `getattr(record, field, None)` when it is a `str`. +fn text_field(record: &Bound<'_, PyAny>, field: &str) -> PyResult> { + if !record.hasattr(field)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = record.getattr(field)?; + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// Python's `sqlbuild_duration_ms / 1000` for an integer field that fits a float, else `None`. +fn duration_seconds(record: &Bound<'_, PyAny>) -> PyResult> { + if !record.hasattr(DURATION_FIELD)? { + return Ok(None); + } + let value: Bound<'_, PyAny> = record.getattr(DURATION_FIELD)?; + if !value.is_instance_of::() { + return Ok(None); + } + match value.div(MILLISECONDS_PER_SECOND) { + Ok(seconds) => Ok(Some(seconds.extract()?)), + Err(error) if error.is_instance_of::(record.py()) => Ok(None), + Err(error) => Err(error), + } +} + +/// The console line of `record` without its exception text. +#[pyfunction] +#[pyo3(signature = (record, *, use_color))] +fn format_diagnostics_console_record( + record: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult { + let mut tokens: Vec<(&'static str, String)> = Vec::new(); + for (field, label) in CONTEXT_TOKENS { + if let Some(value) = text_field(record, field)? { + tokens.push((label, value)); + } + } + let subject: Option = text_field(record, "sqlbuild_subject")?; + let duration_seconds: Option = if subject.is_some() { + duration_seconds(record)? + } else { + None + }; + let text = DiagnosticRecordText { + level_name: record.getattr("levelname")?.extract()?, + logger_name: record.getattr("name")?.extract()?, + message: record.call_method0("getMessage")?.extract()?, + subject, + name: text_field(record, "sqlbuild_name")?, + event: text_field(record, "sqlbuild_event")?, + tokens, + duration_seconds, + sql: text_field(record, SQL_TEXT_FIELD)?, + }; + Ok(format_diagnostic_record(&text, CliStyle { use_color })) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_diagnostics_console_record, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs new file mode 100644 index 0000000000..efe696e343 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs @@ -0,0 +1,381 @@ +//! Structured diagnostic record fields, destination routing and compute log retention. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyStringMethods, PyTuple, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::diagnostic_routing::constants::{ + DIAGNOSTIC_PRODUCER, DIAGNOSTIC_SOURCE, FIELD_PREFIX, FORMATTER_RECORD_FIELDS, IDENTITY_FIELDS, + INVOCATION_ID_FIELD, LOGGER_PREFIX, LOGGER_ROOT_NAME, REDACTED_SQL_LOG_MESSAGE, SAFE_FIELDS, + SQL_ACTION_FIELD, SQL_DIGEST_FIELD, SQL_INTENT_FIELD, SQL_LOGGER_NAME, + SQL_PRIVATE_RECORD_FIELD, SQL_QUERY_ID_FIELD, SQL_SAFE_FIELDS, SQL_TEXT_DIAGNOSTIC_FIELD, + SQL_TEXT_FIELD, UNFORMATTABLE_MESSAGE, +}; +use sqlbuild_core::diagnostic_routing::main::compute_log_admits::compute_log_admits; +use sqlbuild_core::diagnostic_routing::main::diagnostic_severity::diagnostic_severity; +use sqlbuild_core::diagnostic_routing::main::internal_logger::internal_logger; +use sqlbuild_core::diagnostic_routing::main::route_admits::route_admits; +use sqlbuild_core::diagnostic_routing::main::sql_digest::sql_digest; +use sqlbuild_core::diagnostic_routing::models::{RecordFacts, RouteOptions}; +use sqlbuild_core::redaction::main::redact_text::redact_text; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::diagnostics::redaction::redacted_value; + +const PRIVATE_SQL_MARKER: (&str, &str) = ( + "sqlbuild.diagnostics.constants", + "SQL_PRIVATE_RECORD_MARKER", +); + +/// `context | fields | identity`, with later mappings overriding earlier keys in place. +fn merged<'py>( + py: Python<'py>, + parts: &[Option<&Bound<'py, PyAny>>], + skip: Option<&str>, +) -> PyResult> { + let merged: Bound<'py, PyDict> = PyDict::new(py); + for part in parts.iter().flatten() { + if part.is_none() { + continue; + } + for item in part.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if skip.is_some_and(|skipped| key.eq(skipped).unwrap_or(false)) { + continue; + } + merged.set_item(key, value)?; + } + } + Ok(merged) +} + +/// The record fields of the current execution identity, each named `sqlbuild_`. +fn identity_fields<'py>( + py: Python<'py>, + identity: Option<&Bound<'py, PyAny>>, +) -> PyResult>> { + let Some(identity) = identity.filter(|identity| !identity.is_none()) else { + return Ok(None); + }; + let fields: Bound<'py, PyDict> = PyDict::new(py); + for item in identity.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + fields.set_item(format!("{FIELD_PREFIX}{}", key.str()?.to_str()?), value)?; + } + Ok(Some(fields.into_any())) +} + +/// The message and record fields of one logged SQL statement, which never carry its text. +#[pyfunction] +#[pyo3(signature = (**request))] +fn sql_log_record<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult<(String, Bound<'py, PyDict>)> { + let request: Request<'_, 'py> = Request::new("sql_log_record", request)?; + let py: Python<'py> = request.py(); + let sql: String = request.get("sql")?.extract()?; + let action: Bound<'py, PyAny> = request.get("action")?; + let sql_fields: Bound<'py, PyDict> = PyDict::new(py); + sql_fields.set_item(SQL_ACTION_FIELD, &action)?; + sql_fields.set_item(SQL_DIGEST_FIELD, sql_digest(&sql))?; + for (field, name) in [ + (SQL_INTENT_FIELD, "intent"), + (SQL_QUERY_ID_FIELD, "query_id"), + ] { + let value: Bound<'py, PyAny> = request.get(name)?; + if !value.is_none() { + sql_fields.set_item(field, value)?; + } + } + let identity: Option> = identity_fields(py, Some(&request.get("identity")?))?; + let fields: Bound<'py, PyDict> = merged( + py, + &[ + Some(&request.get("context")?), + Some(sql_fields.as_any()), + identity.as_ref(), + ], + Some(SQL_TEXT_FIELD), + )?; + let message: String = format!("{} SQL", action.str()?.to_str()?); + Ok((message, fields)) +} + +/// The record fields of one structured debug event. +#[pyfunction] +#[pyo3(signature = (*, context, event_fields, identity))] +fn debug_event_record_fields<'py>( + py: Python<'py>, + context: &Bound<'py, PyAny>, + event_fields: &Bound<'py, PyAny>, + identity: &Bound<'py, PyAny>, +) -> PyResult> { + let identity: Option> = identity_fields(py, Some(identity))?; + merged( + py, + &[Some(context), Some(event_fields), identity.as_ref()], + None, + ) +} + +/// Whether a logger name belongs to SQLBuild, with Python semantics for non-string names. +fn internal_name(name: &Bound<'_, PyAny>) -> PyResult { + if let Ok(text) = name.downcast::() { + return Ok(internal_logger(text.to_str()?)); + } + Ok(name.eq(LOGGER_ROOT_NAME)? + || name + .call_method1("startswith", (LOGGER_PREFIX,))? + .is_truthy()?) +} + +/// `record.__dict__`. +fn record_values<'py>(record: &Bound<'py, PyAny>) -> PyResult> { + Ok(record.getattr("__dict__")?.downcast_into::()?) +} + +/// `Ok(None)` for an `Exception`, which the caller swallows; other errors propagate. +fn swallowed<'py, T>(py: Python<'py>, result: PyResult) -> PyResult> { + match result { + Ok(value) => Ok(Some(value)), + Err(error) if error.is_instance_of::(py) => Ok(None), + Err(error) => Err(error), + } +} + +/// The redacted formatted message of a record. +fn redacted_message(record: &Bound<'_, PyAny>) -> PyResult { + Ok(redact_text( + record.call_method0("getMessage")?.extract::<&str>()?, + )) +} + +/// A redacted copy of `record.__dict__` for one destination, or `None` when it does not belong. +#[pyfunction] +#[pyo3(signature = (*, record, internal_only, allow_internal, include_sql_text))] +fn route_diagnostic_log_record<'py>( + record: &Bound<'py, PyAny>, + internal_only: bool, + allow_internal: bool, + include_sql_text: bool, +) -> PyResult>> { + let py: Python<'py> = record.py(); + let values: Bound<'py, PyDict> = record_values(record)?; + let facts = RecordFacts { + internal: internal_name(&record.getattr("name")?)?, + level: record.getattr("levelno")?.extract()?, + has_sql_digest: values.contains(SQL_DIGEST_FIELD)?, + has_sql_text: values.contains(SQL_TEXT_FIELD)?, + }; + let options = RouteOptions { + internal_only, + allow_internal, + include_sql_text, + }; + if !route_admits(options, facts) { + return Ok(None); + } + let copied: Bound<'py, PyDict> = values.copy()?; + let mut message: String = swallowed(py, redacted_message(record))? + .unwrap_or_else(|| UNFORMATTABLE_MESSAGE.to_owned()); + copied.set_item("args", PyTuple::empty(py))?; + if facts.has_sql_digest { + REDACTED_SQL_LOG_MESSAGE.clone_into(&mut message); + } + let exc_info: Bound<'py, PyAny> = record.getattr("exc_info")?; + if !exc_info.is_none() { + let formatted: PyResult = imported(py, "logging", "Formatter") + .and_then(|formatter| formatter.call0()) + .and_then(|formatter| formatter.call_method1("formatException", (exc_info,))) + .and_then(|text| text.extract::()) + .map(|text| redact_text(&text)); + if let Some(exception_text) = swallowed(py, formatted)? { + message = format!("{message}\n{exception_text}"); + } + copied.set_item("exc_info", py.None())?; + copied.set_item("exc_text", py.None())?; + } + copied.set_item("msg", message)?; + for (key, value) in copied.copy()?.iter() { + let name: String = key.extract()?; + if name.starts_with(FIELD_PREFIX) && name != SQL_TEXT_FIELD { + copied.set_item(&name, redacted_value(&name, &value)?)?; + } + } + if !include_sql_text && copied.contains(SQL_TEXT_FIELD)? { + copied.del_item(SQL_TEXT_FIELD)?; + } + Ok(Some(copied)) +} + +/// Redacted structured extras of a record, excluding standard, formatter and SQL text fields. +fn extra_fields<'py>(record: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = record.py(); + let standard: Bound<'py, PyDict> = + record_values(&imported(py, "logging", "makeLogRecord")?.call1((PyDict::new(py),))?)?; + let extras: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in record_values(record)?.iter() { + let name: String = key.extract()?; + let excluded: bool = standard.contains(&name)? + || FORMATTER_RECORD_FIELDS.contains(&name.as_str()) + || name == SQL_TEXT_FIELD; + if !excluded { + extras.set_item(&name, redacted_value(&name, &value)?)?; + } + } + Ok(extras) +} + +/// Redacted non-standard fields of a record without evaluating arbitrary formatters. +#[pyfunction] +#[pyo3(signature = (record))] +fn safe_diagnostic_extra_fields<'py>(record: &Bound<'py, PyAny>) -> PyResult> { + extra_fields(record) +} + +/// Whether a redacted structured value is a JSON scalar. +fn scalar(value: &Bound<'_, PyAny>) -> bool { + value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() +} + +/// One record compute logs retain, with its safe structured fields. +struct RetainedRecord<'a, 'py> { + record: &'a Bound<'py, PyAny>, + values: &'a Bound<'py, PyDict>, + facts: RecordFacts, + include_sql_text: bool, +} + +/// The message, logger and fields of a retained record, or an error the handler swallows. +fn retained<'py>( + retained: &RetainedRecord<'_, 'py>, + fields: Bound<'py, PyDict>, +) -> PyResult<(String, Bound<'py, PyAny>, Bound<'py, PyDict>)> { + let RetainedRecord { + record, + values, + facts, + include_sql_text, + } = *retained; + let py: Python<'py> = record.py(); + let contains_sql: bool = facts.has_sql_digest || facts.has_sql_text; + if !contains_sql { + let message: String = redacted_message(record)?; + let fields: Bound<'py, PyDict> = if facts.internal { + fields + } else { + extra_fields(record)? + }; + return Ok((message, record.getattr("name")?, fields)); + } + let sql_fields: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in fields.iter() { + if SQL_SAFE_FIELDS.contains(&key.extract::<&str>()?) { + sql_fields.set_item(key, value)?; + } + } + if include_sql_text && let Some(text) = values.get_item(SQL_TEXT_FIELD)? { + sql_fields.set_item(SQL_TEXT_DIAGNOSTIC_FIELD, text.str()?)?; + } + Ok(( + REDACTED_SQL_LOG_MESSAGE.to_owned(), + PyString::new(py, SQL_LOGGER_NAME).into_any(), + sql_fields, + )) +} + +/// The `DiagnosticLog` fields compute logs retain for a record, or `None` when it is not kept. +#[pyfunction] +#[pyo3(signature = (*, record, invocation_id, include_sql_text))] +fn compute_diagnostic_log_fields<'py>( + py: Python<'py>, + record: &Bound<'py, PyAny>, + invocation_id: &str, + include_sql_text: bool, +) -> PyResult>> { + let name: Bound<'py, PyAny> = record.getattr("name")?; + let Ok(name) = name.downcast::() else { + return Ok(None); + }; + let values: Bound<'py, PyDict> = record_values(record)?; + let level: i64 = record.getattr("levelno")?.extract()?; + let facts = RecordFacts { + internal: internal_logger(name.to_str()?), + level, + has_sql_digest: values.contains(SQL_DIGEST_FIELD)?, + has_sql_text: values.contains(SQL_TEXT_FIELD)?, + }; + if !compute_log_admits(facts, include_sql_text) { + return Ok(None); + } + let trusted_private_sql: bool = match values.get_item(SQL_PRIVATE_RECORD_FIELD)? { + Some(marker) => marker.is(&imported(py, PRIVATE_SQL_MARKER.0, PRIVATE_SQL_MARKER.1)?), + None => false, + }; + let mut identities: Vec<(&str, Option)> = Vec::new(); + for field in IDENTITY_FIELDS { + let value: Option = match values.get_item(format!("{FIELD_PREFIX}{field}"))? { + Some(value) if !value.is_none() => Some(value.str()?.to_str()?.to_owned()), + _ => None, + }; + identities.push((field, value)); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for safe_name in SAFE_FIELDS { + let raw: Bound<'py, PyAny> = values + .get_item(format!("{FIELD_PREFIX}{safe_name}"))? + .unwrap_or_else(|| py.None().into_bound(py)); + let safe_value: Bound<'py, PyAny> = redacted_value(safe_name, &raw)?; + if scalar(&safe_value) { + fields.set_item(safe_name, safe_value)?; + } + } + let record_to_retain = RetainedRecord { + record, + values: &values, + facts, + include_sql_text, + }; + let Some((message, logger, fields)) = swallowed(py, retained(&record_to_retain, fields))? + else { + return Ok(None); + }; + let contains_sql: bool = facts.has_sql_digest || facts.has_sql_text; + let diagnostic: Bound<'py, PyDict> = PyDict::new(py); + diagnostic.set_item("producer", DIAGNOSTIC_PRODUCER)?; + diagnostic.set_item("severity", diagnostic_severity(level))?; + diagnostic.set_item("logger", logger)?; + diagnostic.set_item("source", DIAGNOSTIC_SOURCE)?; + diagnostic.set_item("message", message)?; + diagnostic.set_item("fields", fields)?; + for (field, value) in identities { + let value: Option = if contains_sql && !trusted_private_sql { + None + } else { + value + }; + let value: Option = if field == INVOCATION_ID_FIELD + && (contains_sql || value.as_deref().is_none_or(str::is_empty)) + { + Some(invocation_id.to_owned()) + } else { + value + }; + diagnostic.set_item(field, value)?; + } + Ok(Some(diagnostic)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_log_record, module)?)?; + module.add_function(wrap_pyfunction!(debug_event_record_fields, module)?)?; + module.add_function(wrap_pyfunction!(route_diagnostic_log_record, module)?)?; + module.add_function(wrap_pyfunction!(safe_diagnostic_extra_fields, module)?)?; + module.add_function(wrap_pyfunction!(compute_diagnostic_log_fields, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs index 14fc125d3d..52003770f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/mod.rs @@ -1,2 +1,5 @@ +pub(crate) mod console_records; +pub(crate) mod log_records; pub(crate) mod redaction; +pub(crate) mod registration; pub(crate) mod trackers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs index 603adb3049..591f179773 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/redaction.rs @@ -32,7 +32,10 @@ fn redacted_mapping<'py>(value: &Bound<'py, PyAny>) -> PyResult(name: &str, value: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn redacted_value<'py>( + name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = value.py(); if secret_field_name(name) { return Ok(PyString::new(py, REDACTED).into_any()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs new file mode 100644 index 0000000000..f3a44ab4cd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/registration.rs @@ -0,0 +1,12 @@ +//! Register diagnostics console formatting, redaction and process trackers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::diagnostics::{console_records, log_records, redaction, trackers}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + console_records::register(module)?; + log_records::register(module)?; + redaction::register(module)?; + trackers::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index df0ee54d16..3fed5deff2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -31,7 +31,7 @@ const GENERIC_DIALECT: &str = "generic"; const COMMAND_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild._native"; const EXECUTE_QUERY_MODULE: &str = "sqlbuild.executor.diff.main._execute_query"; const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; const NAMESPACELESS_ADAPTERS: [&str; 2] = ["duckdb", "motherduck"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index 939191d8de..ab63518455 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -6,7 +6,7 @@ use pyo3::types::PyString; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 667afa6b83..6bc8b68467 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -33,11 +33,11 @@ use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const SESSION_MEMO_KEY: &str = "native_discovery_session"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs index 300f6fc990..dc303e0764 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs @@ -7,8 +7,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; /// Models and model schemas, skipping files that cannot be read. #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs index bceaa896b4..08024d074d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::contracts::sql_values::{ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index a5d2082a40..f0d83ff160 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; const NATIVE_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const HEADER_CACHE_SIZE: usize = 4096; /// `(name, line, column, end line, end column)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs index 0b3efe5ed7..10b44186a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/hook_entries.rs @@ -6,7 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const RELATIVE_PATH_FIELD: &str = "relative_path"; const SQL_HOOK_FIELDS: [&str; 6] = [ "statement", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 6f8dde6114..5257ca286a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::event_exporting::scope::current_scope; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const SHIM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.project_modules"; -pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; pub(crate) const ERRORS_MODULE: &str = "sqlbuild._native"; /// One project directory under discovery. @@ -115,7 +115,7 @@ pub(crate) fn module_functions<'py>( .collect() } -/// `sqlbuild.compiler.discovery.models.(**pairs)`. +/// `sqlbuild._native.(**pairs)`. pub(crate) fn discovered<'py>( py: Python<'py>, class_name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs index b6f1b12efc..b2fb22082c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::discovery::imported_modules::{ }; use crate::bindings::_helpers::discovery::imported_providers; -const PYTHON_NODE_ROOT: &str = "python"; +use sqlbuild_core::constants::PYTHON_NODE_ROOT; const NODE_ERROR: &str = "PythonNodeDiscoveryError"; const AUDIT_CASE: (&str, &str) = ("sqlbuild.python_nodes.models", "AuditCase"); const READ_FACTORY: (&str, &str) = ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index d9177933af..f948566f07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_discovery::models::LineColumnSpan; -const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SOURCE_LOCATION_MODULE: &str = "sqlbuild._native"; const SOURCE_LOCATION_CLASS: &str = "SourceLocation"; const FIELD_NAMES: [&str; 10] = [ "file_path", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs index 4be4207f44..cf869ed15b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs @@ -26,9 +26,9 @@ use crate::bindings::_helpers::discovery::yaml_records::discover_source_files; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const FALLBACK_PROJECT_NAME: &str = "project"; const FALLBACK_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 2d4011a816..2913333a00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -13,10 +13,10 @@ use crate::bindings::_helpers::contracts::sql_values::{SqlValueClasses, raw_valu const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild.sql_values.types"; +const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild._native"; pub(crate) type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 827da0c072..624322f883 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -30,8 +30,9 @@ use crate::bindings::_helpers::discovery::sql_test_blocks::{ use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::NativeProjectTree; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const SQL_TESTS_OWNERSHIP_ROOT: &str = "tests/unit"; +const SQL_SCENARIOS_OWNERSHIP_ROOT: &str = "tests/scenarios"; const PATHLIB_MODULE: &str = "pathlib"; const SCENARIO_KIND: &str = "sql_scenario"; const DESCRIPTION_KEY: &str = "description"; @@ -60,8 +61,8 @@ fn path_class(py: Python<'_>) -> PyResult> { } /// `Path(constants.)`. -fn ownership_root<'py>(py: Python<'py>, name: &str) -> PyResult> { - path_class(py)?.call1((imported(py, CONSTANTS_MODULE, name)?,)) +fn ownership_root<'py>(py: Python<'py>, root: &str) -> PyResult> { + path_class(py)?.call1((root,)) } /// Discover the files natively in the pass's snapshot: each relative path text and outcome. @@ -137,7 +138,7 @@ fn test_file<'py>( ), ( "ownership_root", - ownership_root(py, "SQL_TESTS_OWNERSHIP_ROOT")?, + ownership_root(py, SQL_TESTS_OWNERSHIP_ROOT)?, ), ], ) @@ -190,7 +191,7 @@ fn scenario_file<'py>( ("name", file_path.getattr("stem")?), ( "ownership_root", - ownership_root(py, "SQL_SCENARIOS_OWNERSHIP_ROOT")?, + ownership_root(py, SQL_SCENARIOS_OWNERSHIP_ROOT)?, ), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs index 0c0cfa71f0..2f34461ba9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs @@ -33,7 +33,7 @@ use crate::bindings::models::{NativeProjectTree, YamlDiscoveryRequest}; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; /// The most YAML bytes one native load reads before its files are built and released. const YAML_BATCH_BYTES: u64 = 4 * 1024 * 1024; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs index b03687364e..cd153535c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -1,8 +1,7 @@ //! Python modules, thread names and wait limits of native lifecycle-event export. pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.runtime.event_exporting.exceptions"; -pub(crate) const SEVERITY_MODULE: &str = "sqlbuild.spec.contracts.types"; -pub(crate) const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; +pub(crate) const SEVERITY_MODULE: &str = "sqlbuild._native"; pub(crate) const INVOKE_LIFECYCLE_MODULE: &str = "sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"; pub(crate) const RUN_OUTPUT_SINKS_MODULE: &str = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs index c5cf0b9ffd..8641e01e07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs @@ -1,32 +1,28 @@ //! Lifecycle export policy, severity comparison and narrowing for Python callers. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, - wrap_pyfunction, + Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python, pyfunction, wrap_pyfunction, }; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_core::event_export::main::severity_at_least::severity_at_least; use sqlbuild_core::event_export::main::stricter_severity::stricter_severity; use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity}; +use sqlbuild_core::lifecycle_events::constants::CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; use crate::bindings::_helpers::event_exporting::values::{ export_policy, severity_from, severity_object, }; -const CATALOG_MODULE: &str = "sqlbuild.runtime.observability.constants"; - /// `{event_type: (kind, severity, priority)}` for every catalogued lifecycle event, in catalogue order. #[pyfunction] fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { - let catalog: Bound<'_, PyAny> = py - .import(CATALOG_MODULE)? - .getattr("LIFECYCLE_EVENT_CATALOG")?; let policies: Bound<'_, PyDict> = PyDict::new(py); - for event_type in catalog.try_iter()? { - let event_type: String = event_type?.extract()?; - let policy: ExportPolicy = export_policy(py, &event_type)?; + for definition in catalog(CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION).unwrap_or_default() { + let event_type: &str = definition.event_type; + let policy: ExportPolicy = export_policy(py, event_type)?; policies.set_item( - &event_type, + event_type, ( policy.kind, severity_object(py, policy.severity)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs index cd2ee51981..c9a7c33915 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs @@ -9,11 +9,11 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_core::event_export::main::lifecycle_export_policy::lifecycle_export_policy; use sqlbuild_core::event_export::main::sanitized_exception_type::sanitized_exception_type; use sqlbuild_core::event_export::models::{ExportPolicy, ExportSeverity, ExporterCounts}; +use sqlbuild_core::lifecycle_events::constants::CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::event_exporting::constants::{ - CATALOG_MODULE, EXCEPTIONS_MODULE, SEVERITY_MODULE, -}; +use crate::bindings::_helpers::event_exporting::constants::{EXCEPTIONS_MODULE, SEVERITY_MODULE}; /// `BoundEventExporter` parameters, in positional order. const BOUND_EXPORTER_PARAMETERS: [&str; 5] = [ @@ -74,15 +74,13 @@ pub(crate) fn error_type_of(py: Python<'_>, error: &PyErr) -> String { /// Export dimensions of every catalogued lifecycle event type, derived once per interpreter. pub(crate) fn export_policies(py: Python<'_>) -> PyResult<&HashMap> { POLICIES.get_or_try_init(py, || { - let catalog: Bound<'_, PyAny> = imported(py, CATALOG_MODULE, "LIFECYCLE_EVENT_CATALOG")?; let mut policies: HashMap = HashMap::new(); - for entry in catalog.call_method0("items")?.try_iter()? { - let (event_type, definition): (String, Bound<'_, PyAny>) = entry?.extract()?; - let terminal: bool = definition.getattr("terminal")?.is_truthy()?; - let policy: ExportPolicy = lifecycle_export_policy(&event_type, terminal); - policies.insert(event_type, policy); + for definition in catalog(CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION).unwrap_or_default() { + let policy: ExportPolicy = + lifecycle_export_policy(definition.event_type, definition.terminal); + policies.insert(definition.event_type.to_owned(), policy); } - Ok(policies) + Ok::<_, pyo3::PyErr>(policies) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs new file mode 100644 index 0000000000..0718d4e3f7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -0,0 +1,141 @@ +//! Native constants that stay-Python callers read from `sqlbuild._native`, each defined once in Rust. + +use fensu_policy::lifecycle::constants::CUSTOM_HOST_PROTOCOL_VERSION; +use sqlbuild_analysis::contracts::constants::CONTRACT_ENFORCED; +use sqlbuild_analysis::fingerprint_storage::constants::{ + COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, COLUMN_NODE_NAME, + COLUMN_NODE_TYPE, COLUMN_RUN_ID, COLUMN_SCHEMA_FINGERPRINT, COLUMN_TARGET_DATABASE, + COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, + NODE_TYPE_MODEL, NODE_TYPE_SEED, +}; +use sqlbuild_analysis::planner_changes::constants::{ + AUDIT_GATE_METADATA_KEY, MIGRATION_FINGERPRINT_METADATA_KEY, +}; +use sqlbuild_analysis::snowflake_cost::constants::QUERY_TAG_APP; +use sqlbuild_cli::compute_logs::constants::{ + COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, +}; +use sqlbuild_cli::launcher::constants::LAUNCH_RECORD_OPTION; +use sqlbuild_core::command_output::constants::{ + COMMAND_OUTPUT_LOSS_RECORD_TYPE, COMMAND_OUTPUT_RECORD_TYPE, CURRENT_SCHEMA_VERSION, + INVOCATION_CONTEXT_ENV, +}; +use sqlbuild_core::constants::{ + COLUMN_MIGRATION_TABLE_NAME, CTE_AST_KIND, FINGERPRINT_TABLE_NAME, MIGRATION_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, + SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, + TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, +}; +use sqlbuild_core::diagnostic_routing::constants::{ + SQL_DIGEST_FIELD, SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD, +}; +use sqlbuild_rules::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; +use sqlbuild_rules::custom_host_pool::constants::CUSTOM_RULE_PROJECT_SUBJECT; +use sqlbuild_state::execution_history::constants::{ + DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT, SCHEMA_VERSION, +}; + +use crate::bindings::_helpers::attachments::cursor_intrinsics::{ + MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, +}; +use crate::bindings::_helpers::compile_command::constants::{ + REUSE_DISABLE_ENV_VAR, TRACKED_ENVIRONMENT_PREFIXES, UNTRACKED_ENVIRONMENT_NAMES, +}; +use crate::bindings::_helpers::frontier::stage_capture::CAPTURE_DIR_ENV_VAR; +use crate::bindings::_helpers::result_output::event_writer::RESULT_PATH_ENV; +use crate::bindings::_helpers::rules::constants::{ + CUSTOM_HOST_LAUNCH_MODULE, CUSTOM_HOST_RUNTIME_VERSION, +}; +use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; +use crate::bindings::_helpers::state_rows::constants::MIGRATION_WRITE_ATTEMPTS; + +/// Text constants exported under their Python names. +pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ + ("MICROBATCH_START_SENTINEL", MICROBATCH_START_SENTINEL), + ("MICROBATCH_END_SENTINEL", MICROBATCH_END_SENTINEL), + ( + "MIGRATION_FINGERPRINT_METADATA_KEY", + MIGRATION_FINGERPRINT_METADATA_KEY, + ), + ("SNAPSHOT_DEFAULT_VALID_FROM_COLUMN", VALID_FROM_DEFAULT), + ("SNAPSHOT_DEFAULT_VALID_TO_COLUMN", VALID_TO_DEFAULT), + ("PROJECT_CONFIG_FILENAME", PROJECT_CONFIG_FILENAME), + ("PYTHON_INIT_MODULE_STEM", PYTHON_INIT_MODULE_STEM), + ("PYTHON_NODE_ROOT", PYTHON_NODE_ROOT), + ("SETTINGS_SECTION", SETTINGS_SECTION), + ( + "TABLE_PROMOTION_MODE_SETTING_KEY", + TABLE_PROMOTION_MODE_SETTING_KEY, + ), + ("STAGED_PROMOTION_MODE", STAGED_PROMOTION_MODE), + ("STAGE_CAPTURE_DIR_ENV_VAR", CAPTURE_DIR_ENV_VAR), + ("COMPILE_REUSE_DISABLE_ENV_VAR", REUSE_DISABLE_ENV_VAR), + ("SQLBUILD_QUERY_TAG_APP", QUERY_TAG_APP), + ("COMMAND_OUTPUT_RECORD_TYPE", COMMAND_OUTPUT_RECORD_TYPE), + ( + "COMMAND_OUTPUT_LOSS_RECORD_TYPE", + COMMAND_OUTPUT_LOSS_RECORD_TYPE, + ), + ("INVOCATION_CONTEXT_ENV", INVOCATION_CONTEXT_ENV), + ("INTEGRATION_RESULT_PATH_ENV", RESULT_PATH_ENV), + ("FINGERPRINT_TABLE_NAME", FINGERPRINT_TABLE_NAME), + ("SOURCE_FRESHNESS_TABLE_NAME", SOURCE_FRESHNESS_TABLE_NAME), + ("MIGRATION_TABLE_NAME", MIGRATION_TABLE_NAME), + ("COLUMN_MIGRATION_TABLE_NAME", COLUMN_MIGRATION_TABLE_NAME), + ("OLD_NAME_VIEW_TABLE_NAME", OLD_NAME_VIEW_TABLE_NAME), + ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), + ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), + ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), + ("COLUMN_TARGET_SCHEMA", COLUMN_TARGET_SCHEMA), + ("COLUMN_TARGET_NAME", COLUMN_TARGET_NAME), + ("COLUMN_RUN_ID", COLUMN_RUN_ID), + ("COLUMN_DEFINITION_HASH", COLUMN_DEFINITION_HASH), + ("COLUMN_VERSION_HASH", COLUMN_VERSION_HASH), + ("COLUMN_SCHEMA_FINGERPRINT", COLUMN_SCHEMA_FINGERPRINT), + ("COLUMN_DEFINITION_B64", COLUMN_DEFINITION_B64), + ("COLUMN_METADATA_JSON_B64", COLUMN_METADATA_JSON_B64), + ("COLUMN_TIMESTAMP", COLUMN_TIMESTAMP), + ("NODE_TYPE_MODEL", NODE_TYPE_MODEL), + ("NODE_TYPE_SEED", NODE_TYPE_SEED), + ("AUDIT_GATE_METADATA_KEY", AUDIT_GATE_METADATA_KEY), + ("CUSTOM_HOST_RUNTIME_VERSION", CUSTOM_HOST_RUNTIME_VERSION), + ("CUSTOM_HOST_LAUNCH_MODULE", CUSTOM_HOST_LAUNCH_MODULE), + ("CUSTOM_RULE_PROJECT_SUBJECT", CUSTOM_RULE_PROJECT_SUBJECT), + ("CONTRACT_ENFORCED", CONTRACT_ENFORCED), + ("LAUNCH_RECORD_OPTION", LAUNCH_RECORD_OPTION), + ("ZERO_DAY_CURSOR_DURATION", ZERO_DAY_DURATION), + ("AST_CTE_KIND", CTE_AST_KIND), + ("SQL_TEXT_FIELD", SQL_TEXT_FIELD), + ("SQL_DIGEST_FIELD", SQL_DIGEST_FIELD), + ("SQL_PRIVATE_RECORD_FIELD", SQL_PRIVATE_RECORD_FIELD), + ("MIN_CUSTOM_RULE_TEST_CASES", MIN_CUSTOM_RULE_TEST_CASES_KEY), +]; +/// Integer constants exported under their Python names. +pub(crate) const EXPORTED_INTEGERS: &[(&str, i64)] = &[ + ("DEFAULT_PAGE_LIMIT", DEFAULT_PAGE_LIMIT), + ("MAX_PAGE_LIMIT", MAX_PAGE_LIMIT), + ("CURRENT_EVENT_LOG_SCHEMA_VERSION", SCHEMA_VERSION), + ("CURRENT_RUN_STORAGE_SCHEMA_VERSION", SCHEMA_VERSION), + ("COMPUTE_LOG_FORMAT_VERSION", COMPUTE_LOG_FORMAT_VERSION), + ("DEFAULT_RETENTION_COUNT", DEFAULT_RETENTION_COUNT as i64), + ("MAX_READ_BYTES", MAX_READ_BYTES as i64), + ( + "CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION", + CURRENT_SCHEMA_VERSION, + ), + ("MIGRATION_WRITE_ATTEMPTS", MIGRATION_WRITE_ATTEMPTS), + ( + "CUSTOM_HOST_PROTOCOL_VERSION", + CUSTOM_HOST_PROTOCOL_VERSION as i64, + ), +]; +/// Text tuples exported under their Python names. +pub(crate) const EXPORTED_TEXT_TUPLES: &[(&str, &[&str])] = &[( + "STORE_TRACKED_ENVIRONMENT_PREFIXES", + &TRACKED_ENVIRONMENT_PREFIXES, +)]; +/// Text sets exported as frozensets under their Python names. +pub(crate) const EXPORTED_TEXT_SETS: &[(&str, &[&str])] = &[( + "STORE_UNTRACKED_ENVIRONMENT_NAMES", + &UNTRACKED_ENVIRONMENT_NAMES, +)]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs new file mode 100644 index 0000000000..e9c88e14e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod constants; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs new file mode 100644 index 0000000000..bd788edf76 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs @@ -0,0 +1,32 @@ +//! Register the exported native constants and the cursor grain order on `sqlbuild._native`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; + +use crate::bindings::_helpers::cursor::conversion::GRAIN_NAMES; +use crate::bindings::_helpers::exported_constants::constants::{ + EXPORTED_INTEGERS, EXPORTED_TEXT_SETS, EXPORTED_TEXT_TUPLES, EXPORTED_TEXTS, +}; + +/// Registers after the native enums, since `GRAIN_ORDER` is keyed by `CursorGrain` members. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + for (name, value) in EXPORTED_TEXTS { + module.add(*name, *value)?; + } + for (name, value) in EXPORTED_INTEGERS { + module.add(*name, *value)?; + } + for (name, values) in EXPORTED_TEXT_TUPLES { + module.add(*name, PyTuple::new(py, values.iter())?)?; + } + for (name, values) in EXPORTED_TEXT_SETS { + module.add(*name, PyFrozenSet::new(py, values.iter())?)?; + } + let grain = module.getattr("CursorGrain")?; + let order = PyDict::new(py); + for (name, rank) in GRAIN_NAMES { + order.set_item(grain.call1((name,))?, rank as usize)?; + } + module.add("GRAIN_ORDER", order) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs index 232b5681e6..b764698f81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/latest_rows.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::fingerprint_storage::main::latest_index::latest_index; use sqlbuild_analysis::fingerprint_storage::main::selected_positions::selected_positions; use sqlbuild_analysis::fingerprint_storage::models::{FingerprintIdentity, LatestIndex}; -const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; const BINASCII_MODULE: &str = "binascii"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs index 89b33f008c..b4b8774651 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs @@ -20,8 +20,8 @@ use crate::bindings::_helpers::fingerprints::storage_sql::{ fingerprint_insert_batches, fingerprint_insert_sql, }; -const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -const MODELS_MODULE: &str = "sqlbuild.compiler.fingerprints.models"; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +const MODELS_MODULE: &str = "sqlbuild._native"; const ADAPTER_TYPES_MODULE: &str = "sqlbuild.adapter.contract.types"; const TIME_MODULE: &str = "time"; const LOGGING_MODULE: &str = "logging"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index e74a2b9a2c..a46e43cc78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -18,8 +18,8 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const PLANNER_ERRORS_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index bfaff6a081..cb2627250b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -29,7 +29,7 @@ const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const UPSTREAM_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; -const STATE_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME as STATE_TABLE_NAME; const STALE_REQUIRES_STATE_MESSAGE: &str = "freshness --fail-on-stale requires --state"; const STALE_REQUIRES_STATE_CODE: &str = "C238"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs index 8d6c6cb4b1..98a0933a17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod freshness_command; pub(crate) mod freshness_observe; pub(crate) mod observation; pub(crate) mod planning; +pub(crate) mod publication; pub(crate) mod registration; pub(crate) mod state_io; pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index 458b0a11b9..531c0b7a83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -591,7 +591,7 @@ fn source_freshness_record_from_observation<'py>( /// An aware freshness `datetime` in UTC; a naive value is an observation error. #[pyfunction] #[pyo3(signature = (*, value, field_name))] -fn require_aware_utc_datetime<'py>( +pub(crate) fn require_aware_utc_datetime<'py>( value: &Bound<'py, PyAny>, field_name: &str, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index c2e5ca131a..637249be3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -25,7 +25,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_RECORDS: &str = "sqlbuild._native"; const ADAPTER_STRATEGY: &str = "adapter"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs new file mode 100644 index 0000000000..8ee5b46af2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -0,0 +1,319 @@ +//! Append observed source freshness to state once the models it affected have built. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// Module of the execution status enum. +const SCHEDULING_TYPES_MODULE: &str = "sqlbuild.executor.scheduling.types"; +/// Module of the run-scope context manager. +const RUN_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.run_scope"; +/// Module of the operation lifecycle context manager. +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +/// Module of the operation attributes record. +const OBSERVABILITY_MODELS_MODULE: &str = "sqlbuild.runtime.observability.models"; + +/// Observed records grouped by `(database, schema)` in first-seen order. +type RecordsByLocation<'py> = Vec<( + (Bound<'py, PyAny>, Bound<'py, PyAny>), + Vec>, +)>; + +/// The names of the selected models that built successfully. +fn successful_model_names<'py>(result: &Bound<'py, PyAny>) -> PyResult>> { + let py: Python<'py> = result.py(); + let success: Bound<'py, PyAny> = + imported(py, SCHEDULING_TYPES_MODULE, "ExecutionStatus")?.getattr("SUCCESS")?; + let mut names: Vec> = Vec::new(); + for model_result in result.getattr("model_results")?.try_iter()? { + let model_result: Bound<'py, PyAny> = model_result?; + let status: Bound<'py, PyAny> = model_result.getattr("status")?; + let value: Bound<'py, PyAny> = status.getattr_opt("value")?.unwrap_or(status); + if value.eq(&success)? { + names.push(model_result.getattr("model_name")?); + } + } + Ok(names) +} + +/// `record` added under each distinct schema location of `affected`, in sorted model order. +fn collect_record<'py>( + mut grouped: RecordsByLocation<'py>, + record: &Bound<'py, PyAny>, + (affected, entries_by_name): (&Bound<'py, PyAny>, &Bound<'py, PyDict>), +) -> PyResult> { + let py: Python<'py> = record.py(); + let sorted: Bound<'py, PyAny> = imported(py, "builtins", "sorted")?.call1((affected,))?; + let mut recorded: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for model_name in sorted.try_iter()? { + let Some(entry) = entries_by_name.get_item(model_name?)? else { + continue; + }; + let destination: Bound<'py, PyAny> = entry.getattr("destination")?; + let schema: Bound<'py, PyAny> = destination.getattr("schema")?; + if schema.is_none() { + continue; + } + let location: (Bound<'py, PyAny>, Bound<'py, PyAny>) = + (destination.getattr("database")?, schema); + if contains_location(&recorded, &location)? { + continue; + } + recorded.push(location.clone()); + let mut position: Option = None; + for (index, (known, _)) in grouped.iter().enumerate() { + if same_location(known, &location)? { + position = Some(index); + break; + } + } + match position { + Some(index) => grouped[index].1.push(record.clone()), + None => grouped.push((location, vec![record.clone()])), + } + } + Ok(grouped) +} + +/// Whether two `(database, schema)` locations are equal. +fn same_location( + left: &(Bound<'_, PyAny>, Bound<'_, PyAny>), + right: &(Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult { + Ok(left.0.eq(&right.0)? && left.1.eq(&right.1)?) +} + +/// Whether `locations` holds `location`. +fn contains_location( + locations: &[(Bound<'_, PyAny>, Bound<'_, PyAny>)], + location: &(Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult { + for known in locations { + if same_location(known, location)? { + return Ok(true); + } + } + Ok(false) +} + +/// The observed records to publish, keyed by the schema of each affected model. +fn eligible_records<'py>( + plan: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = plan.py(); + let freshness: Bound<'py, PyAny> = plan.getattr("source_freshness")?; + if freshness.is_none() || freshness.getattr("propagation")?.is_none() { + return Ok(Vec::new()); + } + let entries_by_name: Bound<'py, PyDict> = PyDict::new(py); + for entry in plan.getattr("model_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + entries_by_name.set_item(entry.getattr("name")?, entry)?; + } + if entries_by_name.is_empty() { + return Ok(Vec::new()); + } + let selected: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, entries_by_name.keys())?; + let successful: Bound<'py, PyFrozenSet> = + PyFrozenSet::new(py, successful_model_names(result)?)?; + let observed: Bound<'py, PyDict> = PyDict::new(py); + for record in freshness.getattr("observed_records")?.try_iter()? { + let record: Bound<'py, PyAny> = record?; + observed.set_item(record.getattr("identity")?, record)?; + } + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("run_id", run_id)?; + let mut grouped: RecordsByLocation<'py> = Vec::new(); + let changed: Bound<'py, PyAny> = freshness + .getattr("propagation")? + .getattr("changed_source_model_names")?; + for item in changed.call_method0("items")?.try_iter()? { + let (identity, affected): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let affected: Bound<'py, PyAny> = affected.call_method1("__and__", (&selected,))?; + if !affected.is_truthy()? + || !affected + .call_method1("__le__", (&successful,))? + .is_truthy()? + { + continue; + } + let Some(record) = observed.get_item(identity)? else { + continue; + }; + let replaced: Bound<'py, PyAny> = replace.call((record,), Some(&changes))?; + grouped = collect_record(grouped, &replaced, (&affected, &entries_by_name))?; + } + Ok(grouped) +} + +/// `SourceFreshnessRenderers` bound to `adapter`. +fn renderers<'py>(adapter: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(adapter.py(), NATIVE_MODULE, "SourceFreshnessRenderers")?, + &[ + ( + "render_qualified_name", + adapter.getattr("render_qualified_name")?, + ), + ( + "render_framework_type", + adapter.getattr("render_framework_type")?, + ), + ( + "render_insert_records_sql", + adapter.getattr("render_insert_source_freshness_records_sql")?, + ), + ( + "render_create_table_sql", + adapter.getattr("render_create_source_freshness_table_sql")?, + ), + ( + "render_create_index_sqls", + adapter.getattr("render_create_source_freshness_index_sqls")?, + ), + ], + ) +} + +/// Write each location's records, one cost scope per source. +fn write_records<'py>( + adapter: &Bound<'py, PyAny>, + connection: &Bound<'py, PyAny>, + grouped: &RecordsByLocation<'py>, +) -> PyResult<()> { + let py: Python<'py> = adapter.py(); + let cost_context: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "CostContext")?; + let write: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "write_source_freshness_records")?; + for ((database, schema), records) in grouped { + let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); + for record in records { + let source_name: Bound<'py, PyAny> = record.getattr("source_name")?; + let mut position: Option = None; + for (index, (known, _)) in by_source.iter().enumerate() { + if known.eq(&source_name)? { + position = Some(index); + break; + } + } + match position { + Some(index) => by_source[index].1.push(record.clone()), + None => by_source.push((source_name, vec![record.clone()])), + } + } + for (source_name, source_records) in by_source { + let scope: Bound<'py, PyAny> = call_with( + &cost_context.getattr("resource_scope")?, + &[ + ("resource_type", PyString::new(py, "source").into_any()), + ("resource_name", source_name), + ( + "phase", + PyString::new(py, "freshness_finalization").into_any(), + ), + ], + )?; + within(&scope, |_| { + call_with( + &write, + &[ + ("connection", connection.clone()), + ("execute", adapter.getattr("execute")?), + ("database", database.clone()), + ("schema", schema.clone()), + ("records", PyTuple::new(py, &source_records)?.into_any()), + ("renderers", renderers(adapter)?), + ], + ) + })?; + } + } + Ok(()) +} + +/// Append observed source freshness only after the affected selected models succeed. +pub(crate) fn append_freshness_records<'py>( + (plan, result): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, + run_id: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = plan.py(); + let grouped: RecordsByLocation<'py> = eligible_records(plan, result, run_id)?; + let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); + if record_count == 0 { + return Ok(()); + } + let run_scope: Bound<'py, PyAny> = + imported(py, RUN_SCOPE_MODULE, "run_scope")?.call1((run_id,))?; + within(&run_scope, |_| { + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", record_count)?; + let attributes: Bound<'py, PyAny> = call_with( + &imported(py, OBSERVABILITY_MODELS_MODULE, "OperationAttributes")?, + &[ + ("phase", PyString::new(py, "publish").into_any()), + ("target_kind", PyString::new(py, "state_batch").into_any()), + ], + )?; + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", PyString::new(py, "freshness").into_any()), + ( + "operation_name", + PyString::new(py, "source_freshness_publication").into_any(), + ), + ("metadata", metadata.into_any()), + ("attributes", attributes), + ], + )?; + within(&lifecycle, |_| { + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (connection_config,))?; + let written: PyResult<()> = write_records(adapter, &connection, &grouped); + let closed: PyResult> = adapter.call_method1("close", (&connection,)); + written?; + closed?; + Ok(()) + }) + }) +} + +/// Append observed source freshness only after the affected selected models succeed. +#[pyfunction] +#[pyo3(signature = (**options))] +fn append_eligible_direct_source_freshness_records<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult<()> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "append_eligible_direct_source_freshness_records", + options, + ( + &["plan", "result", "adapter", "connection_config", "run_id"], + &[], + ), + )?; + append_freshness_records( + (&arguments.value("plan")?, &arguments.value("result")?), + &arguments.value("adapter")?, + &arguments.value("connection_config")?, + &arguments.value("run_id")?, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + append_eligible_direct_source_freshness_records, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs index 0d0d44f770..4f10818a2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/registration.rs @@ -3,7 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::freshness::{ - exception_types, freshness_command, freshness_observe, observation, planning, state_io, + exception_types, freshness_command, freshness_observe, observation, planning, publication, + state_io, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +13,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { freshness_observe::register(module)?; observation::register(module)?; planning::register(module)?; + publication::register(module)?; state_io::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index 9ebc788dde..308c987873 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -1,9 +1,10 @@ //! Source freshness state table DDL, latest-row reads and appends through adapter callbacks. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_TABLE_NAME}; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_state::freshness::constants::{REQUIRED_STATE_COLUMNS, SOURCE_FRESHNESS_COLUMN_TYPES}; use sqlbuild_state::freshness::main::read_latest_sql::read_latest_sql; use sqlbuild_state::freshness::main::relevant_sources_sql::relevant_sources_sql; @@ -12,9 +13,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::freshness::support::{ - FRESHNESS_CONSTANTS, FRESHNESS_MODELS, INPUT_ERROR, STATE_TABLE_DDL_MODULE, exception_text, - freshness_error, optional_str, presumed_utc_datetime, text, + FRESHNESS_MODELS, INPUT_ERROR, exception_text, freshness_error, optional_str, + presumed_utc_datetime, text, }; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; const NATIVE_RECORDS: &str = "sqlbuild._native"; @@ -81,23 +83,20 @@ fn create_table_sql<'py>( ) -> PyResult> { let py: Python<'py> = database.py(); let qualified_name: String = qualified_table_name(render_qualified_name, (database, schema))?; - let constants: Bound<'py, PyModule> = py.import(FRESHNESS_CONSTANTS)?; - call_with( - &imported(py, STATE_TABLE_DDL_MODULE, "render_state_table_create_sql")?, - &[ - ("qualified_name", text(py, &qualified_name)), - ("columns", constants.getattr("SOURCE_FRESHNESS_COLUMNS")?), - ( - "column_types", - constants.getattr("SOURCE_FRESHNESS_COLUMN_TYPES")?, - ), - ( - "required_columns", - PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(), - ), - ("render_framework_type", render_framework_type.clone()), - ], - ) + let columns: Bound<'py, PyAny> = + PyTuple::new(py, SOURCE_FRESHNESS_COLUMN_TYPES.map(|(column, _)| column))?.into_any(); + let column_types: Bound<'py, PyDict> = PyDict::new(py); + for (column, declared) in SOURCE_FRESHNESS_COLUMN_TYPES { + column_types.set_item(column, declared)?; + } + let column_types: Bound<'py, PyAny> = column_types.into_any(); + let required: Bound<'py, PyAny> = PyFrozenSet::new(py, REQUIRED_STATE_COLUMNS)?.into_any(); + let sql: String = state_table_create_sql( + &qualified_name, + (&columns, &column_types, &required), + render_framework_type, + )?; + Ok(text(py, &sql)) } /// The callbacks and location one latest-state read uses. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 2ce560bc3a..e448a148da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -9,18 +9,15 @@ use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; -pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; -pub(crate) const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; -pub(crate) const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; +pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild._native"; +pub(crate) const CONTRACT_MODELS: &str = "sqlbuild._native"; +pub(crate) const CONTRACT_TYPES: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; pub(crate) const ADAPTER_EXCEPTIONS: &str = "sqlbuild.adapter.contract.exceptions"; pub(crate) const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; pub(crate) const OBSERVABILITY_MODELS: &str = "sqlbuild.runtime.observability.models"; -pub(crate) const STATE_TABLE_DDL_MODULE: &str = - "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; pub(crate) const OBSERVATION_ERROR: &str = "SourceFreshnessObservationError"; pub(crate) const INPUT_ERROR: &str = "SourceFreshnessInputError"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs index 8f83ba02a0..2f12ff6312 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/encoder.rs @@ -9,10 +9,10 @@ use pyo3::types::{ PyStringMethods, PyTuple, }; +use sqlbuild_cache::hosted_shapes::constants::HOSTED_SHAPE_TYPES; use sqlbuild_cache::stage_capture::constants::{ - CANONICAL_TYPES, DECODED_SEQUENCES, HOSTED_SHAPE_TYPES, OMITTED_ATTRIBUTES, - PROJECTED_ATTRIBUTES, RESERVED_KEYS, SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, - UNORDERED_ATTRIBUTES, + CANONICAL_TYPES, DECODED_SEQUENCES, OMITTED_ATTRIBUTES, PROJECTED_ATTRIBUTES, RESERVED_KEYS, + SHARED_MARKER, SHARED_MIN_CHARS, SKIPPED_SLOTS, UNORDERED_ATTRIBUTES, }; use sqlbuild_cache::stage_capture::main::capture_sort_key::capture_sort_key; use sqlbuild_cache::stage_capture::main::compact_capture_json::compact_capture_json; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs index 8b838282a8..ab4f013520 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs @@ -6,17 +6,17 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::stage_capture::constants::OMITTED_ATTRIBUTES; use sqlbuild_cache::stage_capture::main::mask_capture_noise::mask_capture_noise as masked; +use sqlbuild_cache::stage_capture::main::next_capture_number::next_capture_number; use crate::bindings::_helpers::compile_driver::python_calls::{ call_with, imported, keywords, within, }; use crate::bindings::_helpers::frontier::encoder::{CaptureEncoder, emit_error}; -const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; +pub(crate) const CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR"; const CAPTURE_SUFFIX: &str = ".json"; const PLAN_CAPTURE_DIR_ENV_VAR: &str = "SQLBUILD_PLANNER_CAPTURE_DIR"; const PLAN_CAPTURE_NAME: &str = "plan_output"; -const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; /// The capture document of `value`, or the shared nodes written before encoding failed. fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result { @@ -32,6 +32,20 @@ fn capture_text(py: Python<'_>, value: &Bound<'_, PyAny>) -> Result) -> PyResult { + let directory: String = directory.str()?.extract()?; + let names: Vec = match std::fs::read_dir(&directory) { + Ok(entries) => entries + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(error) => return Err(pyo3::exceptions::PyOSError::new_err(error.to_string())), + }; + Ok(next_capture_number(names.iter().map(String::as_str))) +} + /// Write one frontier object when SQLBUILD_COMPILER_STAGE_CAPTURE_DIR names a directory. pub(crate) fn write_stage_capture( stage: &Bound<'_, PyAny>, @@ -43,13 +57,7 @@ pub(crate) fn write_stage_capture( if !directory.is_truthy()? { return Ok(()); } - let number: u64 = imported(py, "builtins", "next")? - .call1((imported( - py, - FRONTIER_CONSTANTS_MODULE, - "STAGE_CAPTURE_SEQUENCE", - )?,))? - .extract()?; + let number: u64 = capture_number(&directory)?; let stage_name: String = stage.getattr("value")?.str()?.extract()?; let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? .call1((directory,))? @@ -68,13 +76,7 @@ pub(crate) fn write_plan_capture(plan_output: &Bound<'_, PyAny>) -> PyResult<()> if !directory.is_truthy()? { return Ok(()); } - let number: u64 = imported(py, "builtins", "next")? - .call1((imported( - py, - FRONTIER_CONSTANTS_MODULE, - "STAGE_CAPTURE_SEQUENCE", - )?,))? - .extract()?; + let number: u64 = capture_number(&directory)?; let path: Bound<'_, PyAny> = imported(py, "pathlib", "Path")? .call1((directory,))? .call_method1( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index c02ad458ba..b8bdd2f925 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -11,9 +11,9 @@ use crate::bindings::_helpers::graph::keys::KeyCache; const GRAPH_MODELS_MODULE: &str = "sqlbuild._native"; const GRAPH_TYPES_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; /// The model config keys whose entries are hooks, in lifecycle order. const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs index e80d6eb2e2..87705c188c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// One `CompiledObjectKey` per native pair, so shared keys convert once. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs index f6b6143dff..1acdd6cbbc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs @@ -8,7 +8,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::python_plan::main::run_lifecycle::run_lifecycle; use sqlbuild_analysis::python_plan::models::{LifecycleNode, RunLifecycle}; -const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild._native"; const EXTERNAL_CONNECTION_MODE: &str = "external"; /// `build_python_sql_run_lifecycle(*, selection, python_graph)`: a `PythonSqlRunLifecyclePlan`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index 480c8e128c..c5cf6c65d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::graph::views::compiled_graph_keys; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const USER_INVENTORY_MODULE: &str = "sqlbuild.compiler.python_nodes._helpers.user_inventory"; -const NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; +const NODE_MODELS_MODULE: &str = "sqlbuild._native"; const NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; const SQL_REF_MODULE: &str = "sqlbuild.python_nodes.models"; const NAME_KEY: &str = "name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 7cb087cb02..ef06bf1fa8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; type NameRows = Vec<(String, Vec)>; /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 6cd147b959..3ea882c339 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -91,7 +91,7 @@ fn graph_views<'py>(py: Python<'py>, graph: &ProjectGraph) -> PyResult( +pub(crate) fn lineage_graph_views<'py>( py: Python<'py>, graph: PyRef<'py, NativeProjectGraph>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index 159c6369c0..ab3123db9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -5,10 +5,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; type Object<'py> = Bound<'py, PyAny>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs index 4b276a57b5..9b1c3c8348 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cli_text::models::CliStyle; -use sqlbuild_core::init_command::constants::PROJECT_FILE; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; use sqlbuild_core::init_command::main::init_project_name::init_project_name; use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs index 4a80dff91d..5aaaaab000 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs @@ -5,7 +5,8 @@ use std::io::{self, ErrorKind}; use std::path::Path; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_core::init_command::constants::{GITKEEP_FILE, PROJECT_FILE, SCAFFOLD_DIRECTORIES}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; +use sqlbuild_core::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; /// `OSError(errno, strerror, filename)`, which Python narrows to its errno subclass. fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs index c3af25ab1a..2b6a791f75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs @@ -27,8 +27,8 @@ const LOG_DEBUG_EVENT: (&str, &str) = ( ); const READ_LATEST_FINGERPRINTS: (&str, &str) = ("sqlbuild._native", "read_latest_fingerprints"); const EXECUTION_LOGGER: &str = "sqlbuild.execution"; -const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; /// The adapter and its open connection. pub(crate) struct Warehouse<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs index b891784a0e..083173283f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs @@ -15,15 +15,15 @@ use crate::bindings::_helpers::janitor::janitor_keys::{ Folded, Key, SchemaKey, optional, record, sorted_schemas, text, tuple, }; -const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +const MIGRATION_TYPES: &str = "sqlbuild._native"; const IS_COMPATIBILITY_VIEW: (&str, &str) = ( "sqlbuild.executor.old_name_views._helpers.janitor", "is_compatibility_view", ); const READ_HISTORIES: (&str, &str) = ("sqlbuild._native", "read_old_name_view_histories"); const STORED_COLUMNS: (&str, &str) = ("sqlbuild._native", "stored_old_name_view_columns"); -const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; -const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; /// Old-name view planning and the old names it owns. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index 6ddfb15348..b16318279e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -7,6 +7,7 @@ use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON; use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name; +use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name; use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_parts::query_diff_artifact_parts; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; @@ -399,3 +400,17 @@ pub(crate) fn build_janitor_archive_name( ) .map_err(|message| shared_error(archived_at.py(), message)) } + +/// The reserved relation name of one run's query-diff artifact side. +#[pyfunction] +#[pyo3(signature = (*, run_id, side))] +pub(crate) fn build_query_diff_artifact_name(run_id: &str, side: &str) -> String { + query_diff_artifact_name(run_id, side) +} + +/// The run id and folded side of a strict query-diff artifact name, or `None`. +#[pyfunction] +#[pyo3(signature = (name, /))] +pub(crate) fn parse_query_diff_artifact_name(name: &str) -> Option<(String, String)> { + query_diff_artifact_parts(name) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs index d68f7e7003..ae838bfb40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/registration.rs @@ -4,7 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::janitor::janitor_planning::{ - build_janitor_archive_name, build_janitor_plan_binding, + build_janitor_archive_name, build_janitor_plan_binding, build_query_diff_artifact_name, + parse_query_diff_artifact_name, }; use crate::bindings::_helpers::janitor::{janitor_command, janitor_plan}; @@ -12,5 +13,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { janitor_command::register(module)?; janitor_plan::register(module)?; module.add_function(wrap_pyfunction!(build_janitor_plan_binding, module)?)?; - module.add_function(wrap_pyfunction!(build_janitor_archive_name, module)?) + module.add_function(wrap_pyfunction!(build_janitor_archive_name, module)?)?; + module.add_function(wrap_pyfunction!(build_query_diff_artifact_name, module)?)?; + module.add_function(wrap_pyfunction!(parse_query_diff_artifact_name, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs new file mode 100644 index 0000000000..411f03bc91 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/catalog.rs @@ -0,0 +1,87 @@ +//! The native lifecycle event catalogue: known types, terminality and catalogued values. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyFrozenSet, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::lifecycle_events::constants::{ + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, HOOK_PHASES, + HOOK_TYPES, LIFECYCLE_EVENT_SCHEMA_VERSIONS, OPERATION_ADAPTERS, OPERATION_KINDS, + OPERATION_METADATA_FIELDS, OPERATION_NAMES, OPERATION_PHASES, OPERATION_SCOPES, + OPERATION_STRATEGIES, OPERATION_TARGET_KINDS, RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES, +}; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; +use sqlbuild_core::lifecycle_events::models::EventDefinition; + +/// The definition of `event_type` in the `schema_version` catalogue, if both are catalogued. +pub(crate) fn definition( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult> { + let (Ok(version), Ok(name)) = ( + schema_version.extract::(), + event_type.downcast::(), + ) else { + return Ok(None); + }; + let name: String = name.extract()?; + Ok(catalog(version) + .and_then(|entries| entries.into_iter().find(|entry| entry.event_type == name))) +} + +/// Whether `event_type` is catalogued for `schema_version`. +#[pyfunction] +#[pyo3(signature = (*, schema_version, event_type))] +fn lifecycle_event_catalogued( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult { + Ok(definition(schema_version, event_type)?.is_some()) +} + +/// Whether a catalogued event closes its correlated lifecycle scope; `KeyError` otherwise. +#[pyfunction] +#[pyo3(signature = (*, schema_version, event_type))] +fn lifecycle_event_terminal( + schema_version: &Bound<'_, PyAny>, + event_type: &Bound<'_, PyAny>, +) -> PyResult { + definition(schema_version, event_type)? + .map(|entry| entry.terminal) + .ok_or_else(|| PyKeyError::new_err(event_type.clone().unbind())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + module.add( + "CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION", + CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, + )?; + module.add( + "CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION", + CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, + )?; + module.add( + "LIFECYCLE_EVENT_SCHEMA_VERSIONS", + PyFrozenSet::new(py, LIFECYCLE_EVENT_SCHEMA_VERSIONS)?, + )?; + let sets: [(&str, &[&str]); 12] = [ + ("HOOK_PHASES", &HOOK_PHASES), + ("HOOK_TYPES", &HOOK_TYPES), + ("OPERATION_ADAPTERS", &OPERATION_ADAPTERS), + ("OPERATION_KINDS", &OPERATION_KINDS), + ("OPERATION_METADATA_FIELDS", &OPERATION_METADATA_FIELDS), + ("OPERATION_NAMES", &OPERATION_NAMES), + ("OPERATION_PHASES", &OPERATION_PHASES), + ("OPERATION_SCOPES", &OPERATION_SCOPES), + ("OPERATION_STRATEGIES", &OPERATION_STRATEGIES), + ("OPERATION_TARGET_KINDS", &OPERATION_TARGET_KINDS), + ("RESOURCE_SKIP_CODES", &RESOURCE_SKIP_CODES), + ("RESOURCE_SKIP_MODES", &RESOURCE_SKIP_MODES), + ]; + for (name, values) in sets { + module.add(name, PyFrozenSet::new(py, values)?)?; + } + module.add_function(wrap_pyfunction!(lifecycle_event_catalogued, module)?)?; + module.add_function(wrap_pyfunction!(lifecycle_event_terminal, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs new file mode 100644 index 0000000000..5864156618 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/envelopes.rs @@ -0,0 +1,306 @@ +//! Lifecycle event and diagnostic log envelopes, run bounds and operation field rules. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyByteArray, PyBytes, PyDict, PyDictMethods, PyInt, PyString, PyTypeMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::lifecycle_events::constants::INVOCATION_SEQUENCE_PROBLEM; +use sqlbuild_core::lifecycle_events::main::diagnostic_severity_problem::diagnostic_severity_problem; +use sqlbuild_core::lifecycle_events::main::diagnostic_version_problem::diagnostic_version_problem; +use sqlbuild_core::lifecycle_events::main::hook_problem::hook_problem; +use sqlbuild_core::lifecycle_events::main::lifecycle_version_problem::lifecycle_version_problem; +use sqlbuild_core::lifecycle_events::main::operation_metadata_field::operation_metadata_field; +use sqlbuild_core::lifecycle_events::main::producer_version_problem::producer_version_problem; +use sqlbuild_core::lifecycle_events::main::publishable_hook_name::publishable_hook_name; +use sqlbuild_core::lifecycle_events::main::run_bounds_problem::run_bounds_problem; +use sqlbuild_core::lifecycle_events::models::{HookFacts, SequenceFact}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::lifecycle_events::payloads::validation_error; +use crate::bindings::_helpers::lifecycle_events::validation::{ + frozen_json, known_event_check, optional_text_check, required_text_check, schema_version_check, + timestamp_check, +}; + +const EVENT_REQUIRED_TEXT: [&str; 5] = [ + "event_id", + "event_type", + "producer", + "producer_version", + "invocation_id", +]; +const EVENT_OPTIONAL_TEXT: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; +const DIAGNOSTIC_REQUIRED_TEXT: [&str; 6] = [ + "producer", + "producer_version", + "severity", + "logger", + "source", + "message", +]; +const DIAGNOSTIC_OPTIONAL_TEXT: [&str; 7] = [ + "log_stream_id", + "invocation_id", + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; + +/// Whether a value is an `int` other than a `bool`. +fn plain_int(value: &Bound<'_, PyAny>) -> bool { + !value.is_instance_of::() && value.is_instance_of::() +} + +/// Whether a value is a non-negative `int` other than a `bool`. +fn nonnegative_int(value: &Bound<'_, PyAny>) -> PyResult { + Ok(plain_int(value) && !value.lt(0)?) +} + +/// An integer, saturated into `i128`. +fn saturated(value: &Bound<'_, PyAny>) -> PyResult { + match value.extract::() { + Ok(number) => Ok(number), + Err(_) if value.lt(0)? => Ok(i128::MIN), + Err(_) => Ok(i128::MAX), + } +} + +/// The text a value holds when it is a string. +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + match value.downcast::() { + Ok(text) => Ok(Some(text.extract()?)), + Err(_) => Ok(None), + } +} + +/// Freeze a JSON object field of a frozen dataclass in place. +fn freeze_object_field(owner: &Bound<'_, PyAny>, field: &str, path: &str) -> PyResult<()> { + let py: Python<'_> = owner.py(); + let frozen: Bound<'_, PyAny> = frozen_json(&owner.getattr(field)?, path)?; + if !frozen.is_instance(&imported(py, "collections.abc", "Mapping")?)? { + return Err(validation_error( + py, + &format!("{path} must be a JSON object"), + )); + } + imported(py, "builtins", "object")? + .getattr("__setattr__")? + .call1((owner, field, frozen))?; + Ok(()) +} + +/// Validate and freeze one lifecycle event envelope in place. +#[pyfunction(name = "validate_lifecycle_event")] +#[pyo3(signature = (*, event))] +fn lifecycle_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = event.py(); + for field in EVENT_REQUIRED_TEXT { + required_text_check(&event.getattr(field)?, field)?; + } + let schema_version: Bound<'_, PyAny> = event.getattr("schema_version")?; + schema_version_check(&schema_version)?; + let sequence: Bound<'_, PyAny> = event.getattr("invocation_sequence")?; + let sequence: SequenceFact = if sequence.is_none() { + SequenceFact::Absent + } else if nonnegative_int(&sequence)? { + SequenceFact::Valid + } else { + SequenceFact::Invalid + }; + let has_external_context: bool = event.getattr("external_context")?.is_truthy()?; + if let Some(problem) = + lifecycle_version_problem(saturated(&schema_version)?, sequence, has_external_context) + { + return Err(validation_error(py, &problem)); + } + timestamp_check(&event.getattr("occurred_at")?)?; + for field in EVENT_OPTIONAL_TEXT { + optional_text_check(&event.getattr(field)?, field)?; + } + freeze_object_field(event, "external_context", "external_context")?; + freeze_object_field(event, "payload", "payload")?; + known_event_check(event) +} + +/// Validate a dispatcher-assigned invocation sequence. +#[pyfunction(name = "validate_lifecycle_invocation_sequence")] +#[pyo3(signature = (*, value))] +fn invocation_sequence_check(value: &Bound<'_, PyAny>) -> PyResult<()> { + if !nonnegative_int(value)? { + return Err(validation_error(value.py(), INVOCATION_SEQUENCE_PROBLEM)); + } + Ok(()) +} + +/// Validate and freeze one diagnostic log record in place. +#[pyfunction(name = "validate_diagnostic_log")] +#[pyo3(signature = (*, record))] +fn diagnostic_log_check(record: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = record.py(); + let schema_version: Bound<'_, PyAny> = record.getattr("schema_version")?; + schema_version_check(&schema_version)?; + if let Some(problem) = diagnostic_version_problem(saturated(&schema_version)?) { + return Err(validation_error(py, &problem)); + } + for field in DIAGNOSTIC_REQUIRED_TEXT { + required_text_check(&record.getattr(field)?, field)?; + } + let severity: String = record.getattr("severity")?.extract()?; + if let Some(problem) = diagnostic_severity_problem(&severity) { + return Err(validation_error(py, problem)); + } + timestamp_check(&record.getattr("occurred_at")?)?; + for field in DIAGNOSTIC_OPTIONAL_TEXT { + optional_text_check(&record.getattr(field)?, field)?; + } + freeze_object_field(record, "fields", "fields") +} + +/// Validate a run's selection, concurrency and worker counts. +#[pyfunction(name = "validate_run_bounds")] +#[pyo3(signature = (*, selected_count, configured_concurrency, worker_count))] +fn run_bounds_check( + selected_count: &Bound<'_, PyAny>, + configured_concurrency: &Bound<'_, PyAny>, + worker_count: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = selected_count.py(); + let values: [(&str, &Bound<'_, PyAny>); 3] = [ + ("selected_count", selected_count), + ("configured_concurrency", configured_concurrency), + ("worker_count", worker_count), + ]; + for (field, value) in values { + if !plain_int(value) { + return Err(validation_error( + py, + &format!("{field} must be an integer excluding bool"), + )); + } + } + match run_bounds_problem( + saturated(selected_count)?, + saturated(configured_concurrency)?, + saturated(worker_count)?, + ) { + Some(problem) => Err(validation_error(py, problem)), + None => Ok(()), + } +} + +/// The allow-listed, non-negative operation metadata counts. +#[pyfunction] +#[pyo3(signature = (*, metadata))] +fn validated_operation_metadata<'py>( + py: Python<'py>, + metadata: &Bound<'py, PyAny>, +) -> PyResult> { + let result: Bound<'py, PyDict> = PyDict::new(py); + if metadata.is_none() { + return Ok(result); + } + for key in metadata.try_iter()? { + if !operation_metadata_field(text(&key?)?.as_deref()) { + return Err(validation_error( + py, + "operation metadata contains a non-allowlisted field", + )); + } + } + for item in metadata.call_method0("items")?.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !nonnegative_int(&value)? { + return Err(validation_error( + py, + "operation metadata values must be nonnegative integers excluding bool", + )); + } + result.set_item(key, value)?; + } + Ok(result) +} + +/// Whether a hook name is publishable, raising like `re.fullmatch` for non-string names. +fn hook_name_publishable(name: &Bound<'_, PyAny>) -> PyResult { + if let Some(name) = text(name)? { + return Ok(publishable_hook_name(&name)); + } + if name.is_instance_of::() || name.is_instance_of::() { + return Err(PyTypeError::new_err( + "cannot use a string pattern on a bytes-like object", + )); + } + let type_name: String = name.get_type().name()?.extract()?; + Err(PyTypeError::new_err(format!( + "expected string or bytes-like object, got '{type_name}'" + ))) +} + +/// The validated hook fields of an operation, empty when none are supplied. +#[pyfunction] +#[pyo3(signature = (*, hook_phase, hook_index, hook_type, hook_name))] +fn validated_operation_hook_fields<'py>( + hook_phase: &Bound<'py, PyAny>, + hook_index: &Bound<'py, PyAny>, + hook_type: &Bound<'py, PyAny>, + hook_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = hook_phase.py(); + let result: Bound<'py, PyDict> = PyDict::new(py); + let supplied: bool = [hook_phase, hook_index, hook_type, hook_name] + .iter() + .any(|value| !value.is_none()); + if !supplied { + return Ok(result); + } + let phase: Option = text(hook_phase)?; + let hook_type_text: Option = text(hook_type)?; + let facts = HookFacts { + phase: phase.as_deref(), + hook_type: hook_type_text.as_deref(), + valid_index: nonnegative_int(hook_index)?, + }; + if let Some(problem) = hook_problem(facts) { + return Err(validation_error(py, problem)); + } + result.set_item("hook_phase", hook_phase)?; + result.set_item("hook_index", hook_index)?; + result.set_item("hook_type", hook_type)?; + if !hook_name.is_none() && hook_name_publishable(hook_name)? { + result.set_item("hook_name", hook_name)?; + } + Ok(result) +} + +/// Validate that a producer other than SQLBuild names its version. +#[pyfunction(name = "validate_lifecycle_producer")] +#[pyo3(signature = (*, producer, producer_version))] +fn producer_check( + producer: &Bound<'_, PyAny>, + producer_version: &Bound<'_, PyAny>, +) -> PyResult<()> { + match producer_version_problem(text(producer)?.as_deref(), !producer_version.is_none()) { + Some(problem) => Err(validation_error(producer.py(), problem)), + None => Ok(()), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lifecycle_event_check, module)?)?; + module.add_function(wrap_pyfunction!(invocation_sequence_check, module)?)?; + module.add_function(wrap_pyfunction!(diagnostic_log_check, module)?)?; + module.add_function(wrap_pyfunction!(run_bounds_check, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_metadata, module)?)?; + module.add_function(wrap_pyfunction!(validated_operation_hook_fields, module)?)?; + module.add_function(wrap_pyfunction!(producer_check, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs new file mode 100644 index 0000000000..ccd5f51ee1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod catalog; +pub(crate) mod envelopes; +pub(crate) mod payloads; +pub(crate) mod registration; +pub(crate) mod validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs new file mode 100644 index 0000000000..17c0d9c764 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/payloads.rs @@ -0,0 +1,157 @@ +//! Frozen JSON values read from Python into payload values, and JSON values frozen in place. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; +use sqlbuild_core::lifecycle_events::models::PayloadValue; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.runtime.observability.exceptions"; + +/// `ObservabilityValidationError(message)`. +pub(crate) fn validation_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "ObservabilityValidationError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Whether `value` is a `collections.abc.Mapping`. +pub(crate) fn abc_mapping(value: &Bound<'_, PyAny>) -> PyResult { + value.is_instance(&imported(value.py(), "collections.abc", "Mapping")?) +} + +/// One segment of a JSON value path: a key or a list index. +#[derive(Clone)] +pub(crate) enum PathSegment { + Key(String), + Index(usize), +} + +/// The root name, then `.key` and `[index]` segments. +fn rendered_path(path: &[PathSegment]) -> String { + let mut text: String = String::new(); + for (position, segment) in path.iter().enumerate() { + match segment { + PathSegment::Index(index) => text.push_str(&format!("[{index}]")), + PathSegment::Key(key) if position > 0 => text.push_str(&format!(".{key}")), + PathSegment::Key(key) => text.push_str(key), + } + } + text +} + +/// `path` followed by one more segment. +fn extended(path: &[PathSegment], segment: PathSegment) -> Vec { + let mut extended: Vec = path.to_vec(); + extended.push(segment); + extended +} + +fn scalar(value: &Bound<'_, PyAny>) -> bool { + value.is_none() || value.is_instance_of::() || value.is_instance_of::() +} + +/// Validate and recursively freeze a JSON-compatible value. +pub(crate) fn frozen<'py>( + value: &Bound<'py, PyAny>, + path: &[PathSegment], +) -> PyResult> { + let py: Python<'py> = value.py(); + if scalar(value) { + return Ok(value.clone()); + } + if value.is_instance_of::() { + if !value.extract::()?.is_finite() { + return Err(validation_error( + py, + &format!("{} must not contain NaN or infinity", rendered_path(path)), + )); + } + return Ok(value.clone()); + } + if abc_mapping(value)? { + let copied: Bound<'py, PyDict> = PyDict::new(py); + for item in value.call_method0("items")?.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let Ok(name) = key.downcast::() else { + return Err(validation_error( + py, + &format!("{} keys must be strings", rendered_path(path)), + )); + }; + if scalar(&item) { + copied.set_item(&key, item)?; + continue; + } + let nested: Vec = extended(path, PathSegment::Key(name.extract()?)); + copied.set_item(&key, frozen(&item, &nested)?)?; + } + return imported(py, "types", "MappingProxyType")?.call1((copied,)); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec> = Vec::new(); + for (index, item) in value.try_iter()?.enumerate() { + let nested: Vec = extended(path, PathSegment::Index(index)); + items.push(frozen(&item?, &nested)?); + } + return Ok(PyTuple::new(py, items)?.into_any()); + } + let type_name: String = value.get_type().getattr("__name__")?.extract()?; + Err(validation_error( + py, + &format!( + "{} contains non-JSON value of type {type_name}", + rendered_path(path) + ), + )) +} + +/// A frozen JSON value as a payload value. +pub(crate) fn payload_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(PayloadValue::Null); + } + if value.is_instance_of::() { + return Ok(PayloadValue::Bool(value.is_truthy()?)); + } + if value.is_instance_of::() { + let saturated: i128 = match value.extract::() { + Ok(number) => number, + Err(_) if value.lt(0)? => i128::MIN, + Err(_) => i128::MAX, + }; + return Ok(PayloadValue::Integer { + value: saturated, + text: value.str()?.extract()?, + }); + } + if value.is_instance_of::() { + return Ok(PayloadValue::Float(value.extract()?)); + } + if value.is_instance_of::() { + return Ok(PayloadValue::Text(value.extract()?)); + } + if abc_mapping(value)? { + return Ok(PayloadValue::Object(object_entries(value)?)); + } + let items: Vec = value + .try_iter()? + .map(|item| payload_value(&item?)) + .collect::>()?; + Ok(PayloadValue::Array(items)) +} + +/// The entries of a frozen JSON object in iteration order. +pub(crate) fn object_entries(value: &Bound<'_, PyAny>) -> PyResult> { + value + .call_method0("items")? + .try_iter()? + .map(|item| { + let (key, item): (String, Bound<'_, PyAny>) = item?.extract()?; + Ok((key, payload_value(&item)?)) + }) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs new file mode 100644 index 0000000000..1bb8d62067 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/registration.rs @@ -0,0 +1,11 @@ +//! Register lifecycle event catalogue queries, constants and envelope validation. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::lifecycle_events::{catalog, envelopes, validation}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + catalog::register(module)?; + envelopes::register(module)?; + validation::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs new file mode 100644 index 0000000000..9c609ba121 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lifecycle_events/validation.rs @@ -0,0 +1,115 @@ +//! Envelope field validation, JSON freezing and catalogued event validation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyBool, PyInt, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::lifecycle_events::main::validate_known_event::validate_known_event; +use sqlbuild_core::lifecycle_events::models::KnownEventFacts; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::lifecycle_events::payloads::{ + PathSegment, frozen, object_entries, validation_error, +}; + +const CORRELATION_FIELDS: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; + +/// Validate the shared positive integer schema-version contract. +#[pyfunction(name = "validate_lifecycle_schema_version")] +#[pyo3(signature = (*, value))] +pub(crate) fn schema_version_check(value: &Bound<'_, PyAny>) -> PyResult<()> { + if value.is_instance_of::() || !value.is_instance_of::() || value.le(0)? { + return Err(validation_error( + value.py(), + "schema_version must be a positive integer excluding bool", + )); + } + Ok(()) +} + +/// Validate and recursively freeze a JSON-compatible value. +#[pyfunction(name = "freeze_lifecycle_json")] +#[pyo3(signature = (*, value, path))] +pub(crate) fn frozen_json<'py>( + value: &Bound<'py, PyAny>, + path: &str, +) -> PyResult> { + frozen(value, &[PathSegment::Key(path.to_owned())]) +} + +/// Validate one required non-empty string field. +#[pyfunction(name = "validate_lifecycle_required_text")] +#[pyo3(signature = (*, value, field_name))] +pub(crate) fn required_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { + let valid: bool = value + .downcast::() + .is_ok_and(|text| text.len().is_ok_and(|length| length > 0)); + if !valid { + return Err(validation_error( + value.py(), + &format!("{field_name} must be a non-empty string"), + )); + } + Ok(()) +} + +/// Validate one optional non-empty string field. +#[pyfunction(name = "validate_lifecycle_optional_text")] +#[pyo3(signature = (*, value, field_name))] +pub(crate) fn optional_text_check(value: &Bound<'_, PyAny>, field_name: &str) -> PyResult<()> { + if value.is_none() { + return Ok(()); + } + required_text_check(value, field_name) +} + +/// Validate a timezone-aware UTC timestamp. +#[pyfunction(name = "validate_lifecycle_timestamp")] +#[pyo3(signature = (*, value))] +pub(crate) fn timestamp_check(value: &Bound<'_, PyAny>) -> PyResult<()> { + let py = value.py(); + let datetime: Bound<'_, PyAny> = imported(py, "datetime", "datetime")?; + if !value.is_instance(&datetime)? || value.getattr("tzinfo")?.is_none() { + return Err(validation_error( + py, + "occurred_at must be a timezone-aware UTC datetime", + )); + } + let utc_offset: Bound<'_, PyAny> = + imported(py, "datetime", "UTC")?.call_method1("utcoffset", (value,))?; + if value.call_method0("utcoffset")?.ne(utc_offset)? { + return Err(validation_error(py, "occurred_at must use UTC")); + } + Ok(()) +} + +/// Validate correlations and safe payload fields for a catalogued event. +#[pyfunction(name = "validate_known_lifecycle_event")] +#[pyo3(signature = (*, event))] +pub(crate) fn known_event_check(event: &Bound<'_, PyAny>) -> PyResult<()> { + let mut correlations: Vec<(&'static str, bool)> = Vec::new(); + for field in CORRELATION_FIELDS { + correlations.push((field, !event.getattr(field)?.is_none())); + } + let facts = KnownEventFacts { + schema_version: event.getattr("schema_version")?.extract()?, + event_type: event.getattr("event_type")?.extract()?, + correlations, + payload: object_entries(&event.getattr("payload")?)?, + }; + validate_known_event(&facts).map_err(|message| validation_error(event.py(), &message)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(schema_version_check, module)?)?; + module.add_function(wrap_pyfunction!(frozen_json, module)?)?; + module.add_function(wrap_pyfunction!(required_text_check, module)?)?; + module.add_function(wrap_pyfunction!(optional_text_check, module)?)?; + module.add_function(wrap_pyfunction!(timestamp_check, module)?)?; + module.add_function(wrap_pyfunction!(known_event_check, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index 53c7ff78cf..f7ec90bca9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -11,6 +11,9 @@ use sqlbuild_analysis::lineage_graph::main::edge_index::edge_index; use sqlbuild_analysis::lineage_graph::models::{EdgeIndex, EdgeKey}; use crate::bindings::_helpers::lineage::lineage_objects::LineageClasses; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + confidence_member, transform_kind_member, +}; /// One compact model's facts and root-star flag. struct CompactModel { @@ -470,7 +473,10 @@ impl RecordBuilder { } } -/// `facts.transform_kind(code)` and `facts.confidence(code)`, looked up once per code. +/// Looks up the enum member of one native code. +type CodeLookup = fn(Python<'_>, usize) -> PyResult>; + +/// The transform kind and confidence members of native codes, looked up once per code. #[derive(Default)] struct CodeMembers { members: HashMap<(&'static str, i64), Py>, @@ -478,26 +484,24 @@ struct CodeMembers { impl CodeMembers { fn transform(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { - self.member(facts, "transform_kind", code) + self.member(facts.py(), ("transform_kind", transform_kind_member), code) } fn confidence(&mut self, facts: &Bound<'_, PyAny>, code: i64) -> PyResult> { - self.member(facts, "confidence", code) + self.member(facts.py(), ("confidence", confidence_member), code) } fn member( &mut self, - facts: &Bound<'_, PyAny>, - method: &'static str, + py: Python<'_>, + (kind, lookup): (&'static str, CodeLookup), code: i64, ) -> PyResult> { - if let Some(member) = self.members.get(&(method, code)) { - return Ok(member.clone_ref(facts.py())); + if let Some(member) = self.members.get(&(kind, code)) { + return Ok(member.clone_ref(py)); } - let member: Py = facts.call_method1(method, (code,))?.unbind(); - let _ = self - .members - .insert((method, code), member.clone_ref(facts.py())); + let member: Py = lookup(py, usize::try_from(code)?)?; + let _ = self.members.insert((kind, code), member.clone_ref(py)); Ok(member) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 698fbfa063..bb75f84d8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -50,7 +50,7 @@ const SELECTION_MODULE: &str = "sqlbuild._native"; const SELECTOR_TOKENS_MODULE: &str = "sqlbuild._native"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const SELECTOR_MATCH_MODULE: &str = "sqlbuild._native"; -const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; const RICH_MODE: &str = "rich"; const ANALYSIS_HELP: &str = "enable SQL analysis or install SQLBuild with Polyglot support"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs index 5b18e3cbef..5edcd89fd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs @@ -6,7 +6,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use sqlbuild_analysis::lineage_command::models::{LineageIndex, LineageKey, LineageNode}; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const GRAPH_MODELS: &str = "sqlbuild._native"; /// A Python `CompiledObjectKey` as a lineage key. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs index 31b7d98744..a55df86374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -7,7 +7,7 @@ use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; const LINEAGE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LOGGING_MODULE: &str = "logging"; const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs new file mode 100644 index 0000000000..47c179dd7c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs @@ -0,0 +1,123 @@ +//! The dlt destination factory, arguments and dataset for one SQLBuild warehouse adapter. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::dlt_destination::constants::DEFAULT_DESTINATION_OPTIONS; +use sqlbuild_core::dlt_destination::main::dlt_destination_plan::dlt_destination_plan; +use sqlbuild_core::dlt_destination::models::{ + DltDataset, DltDatasetName, DltDestinationPlan, DltDestinationRequest, DltValue, +}; +use sqlbuild_core::loader_connection::models::LoaderConnection; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::load::ingestr_command::connection; + +const DLT_ERROR: (&str, &str) = ( + "sqlbuild.integrations.dlt.exceptions", + "DltIntegrationError", +); + +/// `(factory, arguments, options, dataset_name)` of one dlt destination. +type DestinationCall<'py> = ( + &'static str, + Bound<'py, PyDict>, + Bound<'py, PyDict>, + Bound<'py, PyAny>, +); + +/// A `DltIntegrationError`, or the failure to construct one. +fn dlt_error(py: Python<'_>, message: String) -> PyErr { + imported(py, DLT_ERROR.0, DLT_ERROR.1) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// The text of a string value, or `None` for any other value. +fn string(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +/// One factory argument as a Python value. +fn argument<'py>( + value: &DltValue, + connection_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = connection_config.py(); + match value { + DltValue::Text(text) => Ok(PyString::new(py, text).into_any()), + DltValue::Port { default } => connection_config.call_method1("get", ("port", *default)), + DltValue::Object(fields) => Ok(arguments(fields, connection_config)?.into_any()), + } +} + +fn arguments<'py>( + fields: &[(&str, DltValue)], + connection_config: &Bound<'py, PyAny>, +) -> PyResult> { + let values: Bound<'py, PyDict> = PyDict::new(connection_config.py()); + for (key, value) in fields { + values.set_item(*key, argument(value, connection_config)?)?; + } + Ok(values) +} + +/// `(factory, arguments, options, dataset_name)`; callers unpack `arguments` and `options` apart. +#[pyfunction] +#[pyo3(signature = (*, adapter_name, connection_config, destination_config, dataset_name))] +fn dlt_destination_arguments<'py>( + adapter_name: &str, + connection_config: &Bound<'py, PyAny>, + destination_config: &Bound<'py, PyAny>, + dataset_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = connection_config.py(); + let dict: Bound<'py, PyAny> = imported(py, "builtins", "dict")?; + let destination: Bound<'py, PyDict> = if destination_config.is_truthy()? { + dict.call1((destination_config,))?.downcast_into()? + } else { + PyDict::new(py) + }; + let mut destination_keys: Vec> = Vec::new(); + for key in destination.keys() { + destination_keys.push(string(&key)?); + } + let settings: Bound<'py, PyDict> = dict.call1((connection_config,))?.downcast_into()?; + let name = DltDatasetName { + text: string(dataset_name)?, + truthy: dataset_name.is_truthy()?, + }; + let connection: LoaderConnection = connection(&settings)?; + let request = DltDestinationRequest { + adapter: adapter_name, + connection: &connection, + destination_keys: &destination_keys, + dataset_name: &name, + }; + let plan: DltDestinationPlan = + dlt_destination_plan(&request).map_err(|error| dlt_error(py, error.message))?; + let options: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in DEFAULT_DESTINATION_OPTIONS { + options.set_item(key, value)?; + } + options.update(destination.as_mapping())?; + let dataset: Bound<'py, PyAny> = match plan.dataset { + DltDataset::Given => dataset_name.clone(), + DltDataset::Default(name) => PyString::new(py, name).into_any(), + }; + Ok(( + plan.factory, + arguments(&plan.arguments, connection_config)?, + options, + dataset, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(dlt_destination_arguments, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs new file mode 100644 index 0000000000..11a990d48a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs @@ -0,0 +1,217 @@ +//! The ingestr command line for one declarative ingestr source, built from its runtime values. + +use std::collections::BTreeMap; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::ingestr_command::main::format_ingestr_command::format_ingestr_command as native_format_ingestr_command; +use sqlbuild_core::ingestr_command::main::ingestr_command_arguments::ingestr_command_arguments; +use sqlbuild_core::ingestr_command::main::ingestr_destination_table::ingestr_destination_table; +use sqlbuild_core::ingestr_command::main::ingestr_destination_uri::ingestr_destination_uri; +use sqlbuild_core::ingestr_command::models::{IngestrDestination, IngestrSource}; +use sqlbuild_core::loader_connection::models::LoaderConnection; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::model_config::template_data::expanded_template_data; + +const INGESTR_ERROR: (&str, &str) = ( + "sqlbuild.integrations.ingestr.exceptions", + "IngestrIntegrationError", +); +const INGESTR_SOURCE_CONFIG: (&str, &str) = ( + "sqlbuild.integrations.ingestr.models", + "IngestrSourceConfig", +); +const INGESTR_KIND: (&str, &str) = ( + "sqlbuild.integrations.ingestr.constants", + "INGESTR_INTEGRATION_KIND", +); +const TEMPLATE_LABEL: &str = "ingestr source config"; +const TEMPLATE_FIELDS: [&str; 7] = [ + "source_uri", + "source_table", + "strategy", + "incremental_key", + "primary_key", + "columns", + "extra_args", +]; +const LIST_FIELDS: [&str; 2] = ["primary_key", "extra_args"]; + +/// An `IngestrIntegrationError`, or the failure to construct one. +fn ingestr_error(py: Python<'_>, message: String) -> PyErr { + imported(py, INGESTR_ERROR.0, INGESTR_ERROR.1) + .and_then(|class| class.call1((message,))) + .map_or_else(|error| error, PyErr::from_value) +} + +/// The source's ingestr config, raising when the source declares no valid one. +fn ingestr_config<'py>(source_entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source_entry.py(); + let name: String = source_entry.getattr("name")?.str()?.extract()?; + let loader: Bound<'py, PyAny> = source_entry.getattr("integration_loader")?; + if loader.is_none() + || !loader + .getattr("kind")? + .eq(imported(py, INGESTR_KIND.0, INGESTR_KIND.1)?)? + { + return Err(ingestr_error( + py, + format!("Source '{name}' does not define ingestr config"), + )); + } + let config: Bound<'py, PyAny> = loader.getattr("config")?; + if !config.is_instance(&imported( + py, + INGESTR_SOURCE_CONFIG.0, + INGESTR_SOURCE_CONFIG.1, + )?)? { + return Err(ingestr_error( + py, + format!("Source '{name}' defines invalid ingestr config"), + )); + } + Ok(config) +} + +/// `str(value)`, or `None` when the value is `None` or renders empty. +fn optional_text(value: Option>) -> PyResult> { + let Some(value) = value.filter(|value| !value.is_none()) else { + return Ok(None); + }; + let text: String = value.str()?.extract()?; + Ok((!text.is_empty()).then_some(text)) +} + +/// One string per item of a list value, or the value itself as one string. +fn text_list(value: Option>) -> PyResult> { + let Some(value) = value.filter(|value| !value.is_none()) else { + return Ok(Vec::new()); + }; + if value.is_instance_of::() { + return Ok(vec![value.extract()?]); + } + match value.try_iter() { + Ok(items) => items + .map(|item| item?.str()?.extract()) + .collect::>>(), + Err(_) => Ok(vec![value.str()?.extract()?]), + } +} + +/// The source config with `${...}` templates expanded against vars and run context. +fn resolved_source( + config: &Bound<'_, PyAny>, + vars: &Bound<'_, PyDict>, + (environment, run_id): (Option<&str>, &str), +) -> PyResult { + let py: Python<'_> = config.py(); + let value: Bound<'_, PyDict> = PyDict::new(py); + for field in TEMPLATE_FIELDS { + let raw: Bound<'_, PyAny> = config.getattr(field)?; + if LIST_FIELDS.contains(&field) { + value.set_item( + field, + PyList::new(py, raw.try_iter()?.collect::>>()?)?, + )?; + } else { + value.set_item(field, raw)?; + } + } + let context: Bound<'_, PyDict> = PyDict::new(py); + context.set_item("environment", environment)?; + context.set_item("run_id", run_id)?; + let expanded: Bound<'_, PyAny> = expanded_template_data( + value.as_any(), + (vars, &context), + ( + TemplateOptions { + allow_context: true, + preserve_context_tokens: false, + preserve_unknown_context: true, + }, + TEMPLATE_LABEL, + ), + )?; + let expanded: &Bound<'_, PyDict> = expanded.downcast()?; + let get = |key: &str| expanded.get_item(key); + Ok(IngestrSource { + source_uri: expanded.as_any().get_item("source_uri")?.str()?.extract()?, + source_table: expanded + .as_any() + .get_item("source_table")? + .str()? + .extract()?, + strategy: optional_text(get("strategy")?)?, + incremental_key: optional_text(get("incremental_key")?)?, + primary_key: text_list(get("primary_key")?)?, + columns: optional_text(get("columns")?)?, + extra_args: text_list(get("extra_args")?)?, + }) +} + +/// The string settings of a connection and its rendered port. +pub(crate) fn connection(connection_config: &Bound<'_, PyDict>) -> PyResult { + let mut strings: BTreeMap = BTreeMap::new(); + for (key, value) in connection_config.iter() { + if let (Ok(key), Ok(value)) = (key.extract::(), value.downcast::()) { + strings.insert(key, value.extract()?); + } + } + let port: Option = connection_config + .get_item("port")? + .map(|port| port.str()?.extract()) + .transpose()?; + Ok(LoaderConnection { strings, port }) +} + +/// The ingestr CLI command for one source entry. +#[pyfunction] +#[pyo3(signature = (**request))] +fn build_ingestr_command<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request: Request<'_, 'py> = Request::new("build_ingestr_command", request)?; + let py: Python<'py> = request.py(); + let source_entry: Bound<'py, PyAny> = request.get("source_entry")?; + let config: Bound<'py, PyAny> = ingestr_config(&source_entry)?; + let environment: Option = request.get("environment")?.extract()?; + let run_id: String = request.get("run_id")?.extract()?; + let source: IngestrSource = resolved_source( + &config, + request.get("vars")?.downcast()?, + (environment.as_deref(), &run_id), + )?; + let adapter_name: String = request.get("adapter_name")?.extract()?; + let uri: String = ingestr_destination_uri( + &adapter_name, + &connection(request.get("connection_config")?.downcast()?)?, + ) + .map_err(|error| ingestr_error(py, error.message))?; + let schema: Option = source_entry.getattr("schema")?.extract()?; + let destination_name: String = request.get("destination_name")?.extract()?; + let destination: IngestrDestination = IngestrDestination { + uri, + table: ingestr_destination_table(schema.as_deref(), &destination_name), + is_reload: request.get("is_reload")?.is_truthy()?, + }; + PyTuple::new(py, ingestr_command_arguments(&source, &destination)) +} + +/// The display-safe, shell-quoted form of an ingestr command. +#[pyfunction] +#[pyo3(signature = (command, /))] +fn format_ingestr_command(command: Vec) -> String { + native_format_ingestr_command(&command) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_ingestr_command, module)?)?; + module.add_function(wrap_pyfunction!(format_ingestr_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 8f35985120..5cffcea16c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -36,7 +36,7 @@ const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild._native"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_MODELS: &str = "sqlbuild._native"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.python_nodes.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 70299b2a80..9f46362752 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const ERRORS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS: &str = "sqlbuild._native"; const DESTINATION_MODULE: &str = "sqlbuild._native"; const CONNECTION_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs index 68a14122c2..59c61242c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod dlt_destination; +pub(crate) mod ingestr_command; pub(crate) mod load_command; pub(crate) mod load_progress; pub(crate) mod load_selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs index 198a9ed693..a6a6c41ef0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/registration.rs @@ -2,9 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::load::{load_command, load_progress, load_selection}; +use crate::bindings::_helpers::load::{ + dlt_destination, ingestr_command, load_command, load_progress, load_selection, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + dlt_destination::register(module)?; + ingestr_command::register(module)?; load_command::register(module)?; load_progress::register(module)?; load_selection::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs index 5d7f5d9752..d5c16fb94f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/manifest/rows.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::python_values::models::PythonValue; use crate::bindings::_helpers::python_values::values::python_value; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const DECIMAL_KIND: &str = "decimal"; const MACRO_KIND: &str = "macro"; const DEFAULT_MATERIALIZATION: &str = "view"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs new file mode 100644 index 0000000000..dca974875b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs @@ -0,0 +1,90 @@ +//! Microbatch events, scopes and intervals read from their Python shapes. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_cursor::algebra::main::cursor_sort_key::cursor_sort_key; +use sqlbuild_cursor::algebra::models::{Scalar, SortKey}; +use sqlbuild_state::microbatches::models::{BoundInterval, CursorBound}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::conversion::CursorSession; +use crate::bindings::_helpers::cursor::scalars::as_scalar; + +pub(crate) const MODELS_MODULE: &str = "sqlbuild.microbatches.models"; +pub(crate) const TYPES_MODULE: &str = "sqlbuild.microbatches.types"; +pub(crate) const CONSTANTS_MODULE: &str = "sqlbuild.microbatches.constants"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.microbatches.exceptions"; + +/// `MicrobatchStateError(message)`. +pub(crate) fn state_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "MicrobatchStateError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `str(value)` of a string enum member or string. +pub(crate) fn enum_text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +/// `None`, or the string an optional field holds. +pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.extract()?)) +} + +/// `value.timestamp()` of a `datetime`. +pub(crate) fn timestamp(value: &Bound<'_, PyAny>) -> PyResult { + value.call_method0("timestamp")?.extract() +} + +/// Raw cursor bounds of one cursor type, parsed once per distinct text. +pub(crate) struct BoundParser<'py> { + session: CursorSession<'py>, + cursor_type: Bound<'py, PyAny>, + keys: HashMap, +} + +impl<'py> BoundParser<'py> { + pub(crate) fn new(cursor_type: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { + session: CursorSession::new(cursor_type.py())?, + cursor_type: cursor_type.clone(), + keys: HashMap::new(), + }) + } + + /// One raw bound and its sort key. + pub(crate) fn bound(&mut self, raw: &Bound<'py, PyAny>) -> PyResult { + let text: String = raw.extract()?; + if let Some(key) = self.keys.get(&text) { + return Ok(CursorBound { text, key: *key }); + } + let py: Python<'py> = raw.py(); + let scalar: Scalar = as_scalar( + &self.session, + &PyString::new(py, &text).into_any(), + &self.cursor_type, + )?; + let key: SortKey = self + .session + .checked(cursor_sort_key(&scalar, &self.session))?; + self.keys.insert(text.clone(), key); + Ok(CursorBound { text, key }) + } + + /// The bounds of an object with `start` and `end` attributes. + pub(crate) fn interval(&mut self, interval: &Bound<'py, PyAny>) -> PyResult { + Ok(BoundInterval { + start: self.bound(&interval.getattr("start")?)?, + end: self.bound(&interval.getattr("end")?)?, + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs new file mode 100644 index 0000000000..e1b436ef3d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod conversion; +pub(crate) mod projection; +pub(crate) mod registration; +pub(crate) mod statements; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs new file mode 100644 index 0000000000..10c5c32f7e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/projection.rs @@ -0,0 +1,249 @@ +//! Coverage, replay-requirement and latest-requirement projections over append-only events. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_state::microbatches::main::latest_requirement::latest_requirement; +use sqlbuild_state::microbatches::main::project_coverage::project_coverage; +use sqlbuild_state::microbatches::main::project_replay::project_replay; +use sqlbuild_state::microbatches::models::{ + BoundInterval, CompletionFact, CoveragePiece, CoverageProjection, ReplayProjection, + ReplayState, ReplayVersions, RequirementFact, TextInterval, +}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::microbatches::conversion::{ + BoundParser, MODELS_MODULE, TYPES_MODULE, enum_text, optional_text, timestamp, +}; + +const COMPLETION_RECORD_TYPES: [&str; 2] = ["partition_completion", "synthetic_completion"]; +const ORDINARY_COMPLETION: &str = "partition_completion"; +const REPLAY_REQUIREMENT: &str = "replay_requirement"; +const UNKNOWN_FINGERPRINT: &str = "unknown"; + +fn python_intervals<'py>( + py: Python<'py>, + intervals: &[TextInterval], +) -> PyResult> { + let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "MicrobatchInterval")?; + let built: Vec> = intervals + .iter() + .map(|interval| { + call_with( + &class, + &[ + ("start", PyString::new(py, &interval.start).into_any()), + ("end", PyString::new(py, &interval.end).into_any()), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, built) +} + +fn python_coverage<'py>( + py: Python<'py>, + projection: &CoverageProjection, + events: &[Bound<'py, PyAny>], +) -> PyResult> { + let projected_class: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, "ProjectedMicrobatchInterval")?; + let mut projected: Vec> = Vec::with_capacity(projection.segments.len()); + for segment in &projection.segments { + let event: &Bound<'py, PyAny> = &events[segment.completion]; + projected.push(call_with( + &projected_class, + &[ + ( + "start", + PyString::new(py, &segment.interval.start).into_any(), + ), + ("end", PyString::new(py, &segment.interval.end).into_any()), + ("fingerprint_status", event.getattr("fingerprint_status")?), + ("model_version_hash", event.getattr("model_version_hash")?), + ("record_type", event.getattr("record_type")?), + ("completion_type", event.getattr("completion_type")?), + ("event_id", event.getattr("event_id")?), + ], + )?); + } + call_with( + &imported(py, MODELS_MODULE, "MicrobatchCoverageProjection")?, + &[ + ("intervals", PyTuple::new(py, projected)?.into_any()), + ( + "contiguous_frontier", + projection + .contiguous_frontier + .as_deref() + .into_bound_py_any(py)?, + ), + ( + "known_missing", + python_intervals(py, &projection.known_missing)?.into_any(), + ), + ( + "unaccounted", + python_intervals(py, &projection.unaccounted)?.into_any(), + ), + ( + "unknown_fingerprints", + python_intervals(py, &projection.unknown_fingerprints)?.into_any(), + ), + ], + ) +} + +/// Fold completion facts and classify uncovered expected intervals. +#[pyfunction(name = "project_microbatch_coverage")] +#[pyo3(signature = (*, events, expected_intervals, cursor_type))] +fn coverage<'py>( + events: &Bound<'py, PyAny>, + expected_intervals: &Bound<'py, PyAny>, + cursor_type: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = events.py(); + let mut parser: BoundParser<'py> = BoundParser::new(cursor_type)?; + let mut completion_events: Vec> = Vec::new(); + let mut completions: Vec = Vec::new(); + for event in events.try_iter()? { + let event: Bound<'py, PyAny> = event?; + let record_type: String = enum_text(&event.getattr("record_type")?)?; + let (start, end) = ( + event.getattr("partition_start")?, + event.getattr("partition_end")?, + ); + if !COMPLETION_RECORD_TYPES.contains(&record_type.as_str()) + || start.is_none() + || end.is_none() + { + continue; + } + completions.push(CompletionFact { + interval: BoundInterval { + start: parser.bound(&start)?, + end: parser.bound(&end)?, + }, + ordinary: record_type == ORDINARY_COMPLETION, + unknown_fingerprint: enum_text(&event.getattr("fingerprint_status")?)? + == UNKNOWN_FINGERPRINT, + created: timestamp(&event.getattr("created_at")?)?, + event_id: event.getattr("event_id")?.extract()?, + }); + completion_events.push(event); + } + let expected: Vec = expected_intervals + .try_iter()? + .map(|interval| parser.interval(&interval?)) + .collect::>()?; + python_coverage( + py, + &project_coverage(&completions, &expected), + &completion_events, + ) +} + +fn replay_state_member(state: ReplayState) -> &'static str { + match state { + ReplayState::Incomplete => "INCOMPLETE", + ReplayState::VerifiedComplete => "VERIFIED_COMPLETE", + ReplayState::CompleteWithUnknownFingerprints => "COMPLETE_WITH_UNKNOWN_FINGERPRINTS", + ReplayState::Superseded => "SUPERSEDED", + } +} + +/// Derive replay completion without writing mutable requirement status. +#[pyfunction(name = "project_replay_requirement")] +#[pyo3(signature = (**request))] +fn replay<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("project_replay_requirement", request)?; + let py: Python<'py> = request.py(); + let requirement: Bound<'py, PyAny> = request.get("requirement")?; + let current: String = request.get("current_model_version_hash")?.extract()?; + let required: Option = + optional_text(&requirement.getattr("required_model_version_hash")?)?; + let versions = ReplayVersions { + required: required.as_deref(), + current: ¤t, + }; + let expected_objects: Vec> = request + .get("expected_intervals")? + .try_iter()? + .collect::>()?; + let mut expected: Vec = Vec::new(); + let mut pieces: Vec = Vec::new(); + if required.as_deref() == Some(current.as_str()) { + let mut parser: BoundParser<'py> = BoundParser::new(&request.get("cursor_type")?)?; + for interval in &expected_objects { + expected.push(parser.interval(interval)?); + } + for piece in request.get("coverage")?.getattr("intervals")?.try_iter()? { + let piece: Bound<'py, PyAny> = piece?; + pieces.push(CoveragePiece { + interval: parser.interval(&piece)?, + unknown_fingerprint: enum_text(&piece.getattr("fingerprint_status")?)? + == UNKNOWN_FINGERPRINT, + model_version_hash: optional_text(&piece.getattr("model_version_hash")?)?, + }); + } + } + let projection: ReplayProjection = project_replay(versions, &expected, &pieces); + let selected = |indexes: &[usize]| { + PyTuple::new( + py, + indexes.iter().map(|index| expected_objects[*index].clone()), + ) + }; + call_with( + &imported(py, MODELS_MODULE, "ReplayRequirementProjection")?, + &[ + ("requirement", requirement), + ( + "state", + imported(py, TYPES_MODULE, "ReplayRequirementState")? + .getattr(replay_state_member(projection.state))?, + ), + ("missing", selected(&projection.missing)?.into_any()), + ( + "unknown_fingerprints", + selected(&projection.unknown_fingerprints)?.into_any(), + ), + ], + ) +} + +/// Return only the latest requirement created for the currently compiled version. +#[pyfunction(name = "latest_active_replay_requirement")] +#[pyo3(signature = (*, events, current_model_version_hash))] +fn latest<'py>( + events: &Bound<'py, PyAny>, + current_model_version_hash: &str, +) -> PyResult>> { + let objects: Vec> = events.try_iter()?.collect::>()?; + let mut facts: Vec = Vec::with_capacity(objects.len()); + for event in &objects { + let is_requirement: bool = enum_text(&event.getattr("record_type")?)? == REPLAY_REQUIREMENT; + let required: Option = + optional_text(&event.getattr("required_model_version_hash")?)?; + let matching: bool = + is_requirement && required.as_deref() == Some(current_model_version_hash); + facts.push(RequirementFact { + is_requirement, + required_model_version_hash: required, + created: if matching { + timestamp(&event.getattr("created_at")?)? + } else { + 0.0 + }, + event_id: event.getattr("event_id")?.extract()?, + }); + } + Ok(latest_requirement(&facts, current_model_version_hash).map(|index| objects[index].clone())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(coverage, module)?)?; + module.add_function(wrap_pyfunction!(replay, module)?)?; + module.add_function(wrap_pyfunction!(latest, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs new file mode 100644 index 0000000000..b114a939df --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/registration.rs @@ -0,0 +1,10 @@ +//! Register the native microbatch history projections and statements. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::microbatches::{projection, statements}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + projection::register(module)?; + statements::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs new file mode 100644 index 0000000000..abec6c6f0c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/microbatches/statements.rs @@ -0,0 +1,295 @@ +//! Direct-mode microbatch state DDL, idempotent inserts, reads and event IDs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_state::events::main::microbatch_event_id::microbatch_event_id; +use sqlbuild_state::microbatches::main::create_index_sqls::create_index_sqls; +use sqlbuild_state::microbatches::main::existing_event_ids_sql::existing_event_ids_sql; +use sqlbuild_state::microbatches::main::insert_many_sql::insert_many_sql; +use sqlbuild_state::microbatches::main::read_scope_sql::read_scope_sql; +use sqlbuild_state::microbatches::models::{MicrobatchEventIdentity, ScopeRead}; +use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; +use sqlbuild_state::state_tables::models::TableColumn; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::microbatches::conversion::{ + CONSTANTS_MODULE, optional_text, state_error, +}; + +const CODEC_MODULE: &str = "sqlbuild.microbatches.classes.event_codec"; +const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +const MISSING_SCHEMA: &str = "microbatch history requires a target schema"; +const EMPTY_INSERT: &str = "bulk microbatch event insert requires at least one event"; +const EMPTY_LOOKUP: &str = "microbatch event lookup requires at least one event"; +const INTEGER_COLUMN_TYPE: &str = "BIGINT"; +const EVENT_ID_COLUMN: &str = "event_id"; + +/// The state table columns and each column's declared state value type. +struct Layout { + columns: Vec, + declared: Vec, +} + +impl Layout { + fn new(py: Python<'_>) -> PyResult { + let constants: Bound<'_, PyModule> = py.import(CONSTANTS_MODULE)?; + let columns: Vec = constants.getattr("MICROBATCH_COLUMNS")?.extract()?; + let types: Bound<'_, PyAny> = constants.getattr("MICROBATCH_COLUMN_TYPES")?; + let declared: Vec = columns + .iter() + .map(|column| types.get_item(column)?.str()?.extract()) + .collect::>()?; + Ok(Self { columns, declared }) + } + + fn declared(&self, column: &str) -> &str { + self.columns + .iter() + .position(|known| known == column) + .map_or("", |index| self.declared[index].as_str()) + } + + /// One event's column literals in column order. + fn literals(&self, event: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = event.py(); + let values: Bound<'_, PyAny> = + imported(py, CODEC_MODULE, "MicrobatchEventCodec")?.call_method1("values", (event,))?; + values + .try_iter()? + .zip(&self.declared) + .map(|(value, declared)| render_state_sql_literal(py, &value?, declared)) + .collect() + } + + fn literal(&self, column: &str, value: &Bound<'_, PyAny>) -> PyResult { + render_state_sql_literal(value.py(), value, self.declared(column)) + } +} + +/// The adapter's qualified name of the state table, failing without a schema. +fn qualified_table<'py>( + render_qualified_name: &Bound<'py, PyAny>, + (database, schema): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = render_qualified_name.py(); + let table_name: Bound<'py, PyAny> = imported(py, CONSTANTS_MODULE, "MICROBATCH_TABLE_NAME")?; + optional_text(&call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ("name", table_name), + ], + )?) +} + +/// The state table of one scope: its target database and required schema. +fn scope_table( + render_qualified_name: &Bound<'_, PyAny>, + scope: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = scope.py(); + let schema: Bound<'_, PyAny> = scope.getattr("target_schema")?; + if schema.is_none() { + return Err(state_error(py, MISSING_SCHEMA)); + } + qualified_table( + render_qualified_name, + (&scope.getattr("target_database")?, &schema), + )? + .ok_or_else(|| state_error(py, MISSING_SCHEMA)) +} + +/// The events as a list, and the table of the first event's scope; empty fails with `empty`. +fn events_table<'py>( + events: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, + empty: &str, +) -> PyResult<(Vec>, String)> { + let listed: Vec> = events.try_iter()?.collect::>()?; + let Some(first) = listed.first() else { + return Err(state_error(events.py(), empty)); + }; + let table: String = scope_table(render_qualified_name, &first.getattr("scope")?)?; + Ok((listed, table)) +} + +/// Build one guarded set-based insert for events in the same state table. +#[pyfunction(name = "build_microbatch_insert_many_sql")] +#[pyo3(signature = (*, events, render_qualified_name))] +fn insert_many( + events: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + let (listed, table) = events_table(events, render_qualified_name, EMPTY_INSERT)?; + let layout: Layout = Layout::new(events.py())?; + let rows: Vec> = listed + .iter() + .map(|event| layout.literals(event)) + .collect::>()?; + Ok(insert_many_sql(&table, &layout.columns, &rows)) +} + +/// Build one lookup for event IDs already present before a bulk insert. +#[pyfunction(name = "build_microbatch_existing_event_ids_sql")] +#[pyo3(signature = (*, events, render_qualified_name))] +fn existing_ids( + events: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + let (listed, table) = events_table(events, render_qualified_name, EMPTY_LOOKUP)?; + let layout: Layout = Layout::new(events.py())?; + let literals: Vec = listed + .iter() + .map(|event| layout.literal(EVENT_ID_COLUMN, &event.getattr(EVENT_ID_COLUMN)?)) + .collect::>()?; + Ok(existing_event_ids_sql(&table, &literals)) +} + +/// Build the ordered read of one scope's events, every generation for the wildcard. +#[pyfunction(name = "build_microbatch_read_scope_sql")] +#[pyo3(signature = (*, scope, render_qualified_name))] +fn read_scope( + scope: &Bound<'_, PyAny>, + render_qualified_name: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = scope.py(); + let table: String = scope_table(render_qualified_name, scope)?; + let layout: Layout = Layout::new(py)?; + let wildcard: String = + imported(py, CONSTANTS_MODULE, "MICROBATCH_GENERATION_WILDCARD")?.extract()?; + let generation: Bound<'_, PyAny> = scope.getattr("physical_generation_id")?; + let generation: Option = if generation.extract::()? == wildcard { + None + } else { + Some(layout.literal("physical_generation_id", &generation)?) + }; + let scope_kind: String = layout.literal("scope_kind", &scope.getattr("scope_kind")?)?; + let scope_key: String = layout.literal("scope_key", &scope.getattr("scope_key")?)?; + Ok(read_scope_sql( + &table, + &ScopeRead { + columns: &layout.columns, + scope_kind: &scope_kind, + scope_key: &scope_key, + generation: generation.as_deref(), + }, + )) +} + +/// Build direct microbatch-state DDL with the adapter's text and timestamp types. +#[pyfunction(name = "build_microbatch_create_table_sql")] +#[pyo3(signature = (**request))] +fn create_table(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("build_microbatch_create_table_sql", request)?; + let py: Python<'_> = request.py(); + let table: String = qualified_table( + &request.get("render_qualified_name")?, + (&request.get("database")?, &request.get("schema")?), + )? + .ok_or_else(|| state_error(py, MISSING_SCHEMA))?; + let render_framework_type: Bound<'_, PyAny> = request.get("render_framework_type")?; + let framework: Bound<'_, PyAny> = imported(py, ADAPTER_TYPES, "FrameworkType")?; + let text_type: String = render_framework_type + .call1((framework.getattr("STRING")?,))? + .extract()?; + let timestamp_type: String = render_framework_type + .call1((framework.getattr("TIMESTAMP")?,))? + .extract()?; + let layout: Layout = Layout::new(py)?; + let columns: Vec = layout + .columns + .iter() + .zip(&layout.declared) + .map(|(name, declared)| TableColumn { + name: name.clone(), + rendered_type: match declared.as_str() { + "timestamp" => timestamp_type.clone(), + "integer" => INTEGER_COLUMN_TYPE.to_owned(), + _ => text_type.clone(), + }, + required: false, + }) + .collect(); + Ok(create_table_sql(&table, &columns)) +} + +/// Build non-unique scope and provenance indexes for indexed warehouses. +#[pyfunction(name = "build_microbatch_create_index_sqls")] +#[pyo3(signature = (**request))] +fn create_indexes<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("build_microbatch_create_index_sqls", request)?; + let py: Python<'_> = request.py(); + let schema: Bound<'_, PyAny> = request.get("schema")?; + let rendered: Bound<'_, PyAny> = call_with( + &request.get("render_qualified_name")?, + &[ + ("database", request.get("database")?), + ("schema", schema.clone()), + ( + "name", + imported(py, CONSTANTS_MODULE, "MICROBATCH_TABLE_NAME")?, + ), + ], + )?; + let table: String = if rendered.is_truthy()? { + rendered.extract()? + } else { + let name: String = imported(py, CONSTANTS_MODULE, "MICROBATCH_TABLE_NAME")?.extract()?; + format!("{}.{name}", schema.str()?) + }; + let render_identifier: Bound<'_, PyAny> = request.get("render_identifier")?; + let identifier = |name: &Bound<'_, PyAny>| -> PyResult { + render_identifier.call1((name,))?.extract() + }; + let mut indexes: Vec<(String, Vec)> = Vec::new(); + let definitions: Bound<'_, PyAny> = + imported(py, CONSTANTS_MODULE, "MICROBATCH_DIRECT_INDEXES")?; + for item in definitions.call_method0("items")?.try_iter()? { + let (name, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let columns: Vec = columns + .try_iter()? + .map(|column| identifier(&column?)) + .collect::>()?; + indexes.push((identifier(&name)?, columns)); + } + PyTuple::new(py, create_index_sqls(&table, &indexes)) +} + +/// Return the content-addressed ID for one logical event. +#[pyfunction(name = "deterministic_microbatch_event_id")] +#[pyo3(signature = (**request))] +fn deterministic_event_id(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("deterministic_microbatch_event_id", request)?; + let scope: Bound<'_, PyAny> = request.get("scope")?; + let field = |name: &str| -> PyResult { scope.getattr(name)?.extract() }; + let optional = + |name: &str| -> PyResult> { optional_text(&scope.getattr(name)?) }; + let identity = MicrobatchEventIdentity { + scope_kind: field("scope_kind")?, + scope_key: field("scope_key")?, + model_name: field("model_name")?, + target_database: optional("target_database")?, + target_schema: optional("target_schema")?, + target_name: field("target_name")?, + physical_generation_id: field("physical_generation_id")?, + record_type: request.get("record_type")?.getattr("value")?.extract()?, + partition_start: optional_text(&request.get("partition_start")?)?, + partition_end: optional_text(&request.get("partition_end")?)?, + completion_reason: request.get("completion_reason")?.extract()?, + }; + microbatch_event_id(&identity).map_err(emit_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(insert_many, module)?)?; + module.add_function(wrap_pyfunction!(existing_ids, module)?)?; + module.add_function(wrap_pyfunction!(read_scope, module)?)?; + module.add_function(wrap_pyfunction!(create_table, module)?)?; + module.add_function(wrap_pyfunction!(create_indexes, module)?)?; + module.add_function(wrap_pyfunction!(deterministic_event_id, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index f59238f6b8..9b7bb46cb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -18,9 +18,12 @@ pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compile_records; pub(crate) mod compiled_project; +pub(crate) mod compiler_enums; +pub(crate) mod compiler_errors; pub(crate) mod compiler_records; pub(crate) mod compute_logs; pub(crate) mod contract_command; +pub(crate) mod contract_records; pub(crate) mod contracts; pub(crate) mod cost_command; pub(crate) mod cost_report; @@ -37,6 +40,7 @@ pub(crate) mod diff_report; pub(crate) mod discovery; pub(crate) mod event_exporting; pub(crate) mod execution_history; +pub(crate) mod exported_constants; pub(crate) mod fingerprints; pub(crate) mod format_command; pub(crate) mod freshness; @@ -45,10 +49,12 @@ pub(crate) mod graph; pub(crate) mod hand_built; pub(crate) mod init_command; pub(crate) mod janitor; +pub(crate) mod lifecycle_events; pub(crate) mod lineage; pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; +pub(crate) mod microbatches; pub(crate) mod model_config; pub(crate) mod native_threads; pub(crate) mod output_capture; @@ -58,8 +64,8 @@ pub(crate) mod planner_changes; pub(crate) mod planner_entries; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; -pub(crate) mod planner_models; pub(crate) mod planner_phases; +pub(crate) mod planner_records; pub(crate) mod planner_registration; pub(crate) mod planner_resolution; pub(crate) mod planner_scenario; @@ -71,14 +77,17 @@ pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod query_command; +pub(crate) mod record_behaviours; pub(crate) mod records; pub(crate) mod refactor; pub(crate) mod references; pub(crate) mod relation_naming; pub(crate) mod render; pub(crate) mod result_output; +pub(crate) mod rule_records; pub(crate) mod rules; pub(crate) mod scenario_command; +pub(crate) mod scope_records; pub(crate) mod scopes; pub(crate) mod seed_command; pub(crate) mod semantic_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs index 48f6bfb0f2..681cf918ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_errors.rs @@ -9,7 +9,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::model_config::config_values::config_value_type_error; use crate::bindings::_helpers::model_config::errors::NativeConfigError; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild._native"; const IDENTITY_ERRORS_MODULE: &str = "sqlbuild._native"; const EXPECTED_STRING: &str = "a string"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs index 1cca55b389..67869c37d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyDict, PyDictMethods}; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const AUDITING_MODELS_MODULE: &str = "sqlbuild.compiler.auditing.models"; +const AUDITING_MODELS_MODULE: &str = "sqlbuild._native"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const DLT_MODELS_MODULE: &str = "sqlbuild.integrations.dlt.models"; const INGESTR_MODELS_MODULE: &str = "sqlbuild.integrations.ingestr.models"; const CONTRACT_CLASSES: [(&str, &str, &str); 6] = [ @@ -126,10 +126,6 @@ fn built_contract_classes(py: Python<'_>) -> PyResult> { "threshold_operators", members_by_value(&imported(py, AUDITING_TYPES_MODULE, "ThresholdOperator")?)?, )?; - classes.set_item( - "allocate", - imported(py, "builtins", "object")?.getattr("__new__")?, - )?; Ok(classes) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 56e7cb86ce..2d91542cb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -21,7 +21,6 @@ pub(crate) struct ContractClasses<'py> { measurement_threshold_bound: Bound<'py, PyAny>, severities: Bound<'py, PyDict>, threshold_operators: Bound<'py, PyDict>, - allocate: Bound<'py, PyAny>, } pub(crate) fn column_tuple<'py>( @@ -50,18 +49,16 @@ pub(crate) fn audit_tuple<'py>( PyTuple::new(py, audits) } -/// Build a frozen contract dataclass the way `copy` does: allocate, then fill its `__dict__`. +/// `class(**fields)`. fn contract_object<'py>( - allocate: &Bound<'py, PyAny>, class: &Bound<'py, PyAny>, fields: &[(&str, Bound<'py, PyAny>)], ) -> PyResult> { - let object = allocate.call1((class,))?; - let attributes = object.getattr("__dict__")?.downcast_into::()?; + let keywords: Bound<'py, PyDict> = PyDict::new(class.py()); for (name, value) in fields { - attributes.set_item(name, value)?; + keywords.set_item(name, value)?; } - Ok(object) + class.call((), Some(&keywords)) } fn optional<'py>(py: Python<'py>, value: Option<&PyNode<'py>>) -> Bound<'py, PyAny> { @@ -83,7 +80,6 @@ fn schema_column<'py>( .map(|audit| audit_instance(py, classes, audit, &location)) .collect::>>()?; contract_object( - &classes.allocate, &classes.schema_column, &[ ("name", column.name.0.clone()), @@ -120,7 +116,6 @@ fn audit_instance<'py>( |node| node.0.clone(), ); contract_object( - &classes.allocate, &classes.schema_audit_instance, &[ ("definition_name", audit.definition_name.0.clone()), @@ -157,7 +152,6 @@ pub(crate) fn thresholds<'py>( return Ok(py.None().into_bound(py)); }; contract_object( - &classes.allocate, &classes.measurement_thresholds, &[ ("warn", threshold_bound(py, classes, parsed.warn)?), @@ -184,7 +178,6 @@ fn threshold_bound<'py>( .get_item(operator)? .unwrap_or_else(|| py.None().into_bound(py)); contract_object( - &classes.allocate, &classes.measurement_threshold_bound, &[ ("operator", operator), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs index dcd165dfa1..b78e53cfda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::model_config::config_templates::{ }; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const UUID_MODULE: &str = "uuid"; const ENVIRONMENT_READ: &str = "env"; const RUN_ID_KEY: &str = "run.id"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs index 5a2970e1dd..64069fdec1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/unicode_environment.rs @@ -8,7 +8,7 @@ use pyo3::{pyclass, pymethods}; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// `os.environ`, raising a clear error for a value holding undecodable bytes. #[pyclass(module = "sqlbuild._native", frozen, mapping)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index bfb27544ff..d26831217b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; +const RULE_MODELS: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const LIST_ACTION: &str = "list"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs index f34fba5aab..fbf071b610 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs @@ -3,20 +3,19 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::conversion::GRAIN_NAMES; use crate::bindings::_helpers::plan_output::display::{ aligned_timestamp_bounds, resolve_effective_microbatch_batch_size, }; use crate::bindings::_helpers::plan_output::values::{optional_text, text}; const BOUND_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; -const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild._native"; const MICROBATCH_MODE: &str = "microbatch"; const TIMESTAMP_CURSOR: &str = "timestamp"; const INTEGER_CURSOR: &str = "integer"; const WATERMARK_STRATEGY: &str = "watermark"; const SECOND_GRAIN: &str = "second"; -/// `GRAIN_ORDER`, finest first. -const GRAIN_ORDER: [&str; 6] = ["second", "minute", "hour", "day", "month", "year"]; /// `CursorBoundsOwner` and `CursorResolutionStatus` of one model, as their values. pub(crate) const RUNTIME_OWNER: &str = "runtime"; @@ -44,9 +43,9 @@ pub(crate) struct CursorDetails<'py> { } fn grain_rank(grain: &str) -> PyResult { - GRAIN_ORDER + GRAIN_NAMES .iter() - .position(|known| *known == grain) + .position(|(known, _)| *known == grain) .ok_or_else(|| { pyo3::exceptions::PyValueError::new_err(format!("'{grain}' is not a valid CursorGrain")) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs index c5da29096c..f7eb47e474 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs @@ -24,9 +24,9 @@ use crate::bindings::_helpers::cursor::scalars::{ }; use crate::bindings::_helpers::plan_output::values::{optional_text, text}; -const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; -const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const PLANNER_TYPES: &str = "sqlbuild._native"; +const PLANNER_MODELS: &str = "sqlbuild._native"; +const CURSOR_MODELS: &str = "sqlbuild._native"; const DATETIME_MODULE: &str = "datetime"; fn label_facts(entry: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 729f2e9f83..b6ec6ecca0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const FINAL_NEWLINE: &str = "\n"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs index 8fd29dcaf2..9151fe572d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -11,9 +11,9 @@ use sqlbuild_analysis::planner_changes::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -pub(crate) const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const MODELS_MODULE: &str = "sqlbuild._native"; +pub(crate) const TYPES_MODULE: &str = "sqlbuild._native"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// `PlannerInputError(message)` for an unknown replay policy. pub(crate) fn replay_policy_error(py: Python<'_>, error: UnknownReplayPolicy) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 2086ff2412..c3402cd2e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -1,6 +1,6 @@ //! Static plans for commands that do not build models, and direct seed identity reads. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, }; @@ -11,10 +11,10 @@ use crate::bindings::_helpers::planner_entries::values::{ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; use std::collections::{BTreeMap, BTreeSet}; -const FINGERPRINT_CONSTANTS: &str = "sqlbuild.compiler.fingerprints.constants"; - /// Optional fields of a static command plan; absent ones take `PlanOutput` defaults. #[derive(Default)] struct StaticPlanParts<'py> { @@ -370,9 +370,7 @@ fn read_selected_seed_fingerprints<'py>( seeds: &Bound<'py, PyAny>, ) -> PyResult> { let py = adapter.py(); - let table_name: String = - imported(py, FINGERPRINT_CONSTANTS, "FINGERPRINT_TABLE_NAME")?.extract()?; - let seed_node_type = imported(py, FINGERPRINT_CONSTANTS, "NODE_TYPE_SEED")?; + let table_name: &str = FINGERPRINT_TABLE_NAME; let mut names_by_target: BTreeMap<(String, String), (Target, BTreeSet)> = BTreeMap::new(); for seed in seeds.try_iter()? { @@ -393,7 +391,7 @@ fn read_selected_seed_fingerprints<'py>( let fingerprints = PyDict::new(py); let mut relations_by_target: Vec> = Vec::new(); for ((database, schema), _) in names_by_target.values() { - let names = PyTuple::new(py, [table_name.as_str()])?; + let names = PyTuple::new(py, [table_name])?; let schemas = PyTuple::new(py, [schema.as_str()])?; let kwargs = PyDict::new(py); kwargs.set_item("connection", connection)?; @@ -449,7 +447,7 @@ fn read_selected_seed_fingerprints<'py>( )?; for item in items.iter() { let (name, fingerprint) = (item.get_item(0)?, item.get_item(1)?); - if fingerprint.getattr("node_type")?.eq(&seed_node_type)? + if fingerprint.getattr("node_type")?.eq(NODE_TYPE_SEED)? && names.contains(&name.extract::()?) { fingerprints.set_item(name, fingerprint)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index ca8c68b3eb..f35a07d9ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -7,15 +7,15 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; -pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; -pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(super) const PLANNER_MODELS: &str = "sqlbuild._native"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild._native"; +pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; -pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild._native"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -pub(super) const DISCOVERY_MODELS: &str = "sqlbuild.compiler.discovery.models"; +pub(super) const COMPILE_MODELS: &str = "sqlbuild._native"; +pub(super) const DISCOVERY_MODELS: &str = "sqlbuild._native"; /// `sqlbuild._native.name(**pairs)`. pub(super) fn native<'py>( @@ -26,7 +26,7 @@ pub(super) fn native<'py>( call_with(&imported(py, NATIVE_MODULE, name)?, pairs) } -/// `sqlbuild.compiler.planner.models.name(**pairs)`. +/// The planner record `name(**pairs)`. pub(super) fn planner_model<'py>( py: Python<'py>, name: &str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index 7659f048da..bc34edda00 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -29,7 +29,7 @@ use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::planner_identity::hashing::graph_key; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const TABLE_FN_TEST_MODE: &str = "table_fn"; const INLINE_SQL_HOOK: &str = "inline SQL"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs index af4645c209..65d24bc6d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::types::CompilerDetach; -const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// The query fingerprint of a model, or the planner input error naming the model. pub(crate) fn model_query_hash( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 743e4061b8..85904dedb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -10,7 +10,7 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::compatibility_findings::compatibility_findings; use sqlbuild_analysis::planner_migrations::models::CompatibilityFinding; -const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const TYPES_MODULE: &str = "sqlbuild._native"; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { model diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index 110d7f3ace..5a37210972 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -1,12 +1,10 @@ -pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(super) const PLANNER_MODELS: &str = "sqlbuild._native"; +pub(super) const MIGRATION_TYPES: &str = "sqlbuild._native"; pub(super) const AUTOMATIC: &str = "automatic"; pub(super) const DONE: &str = "done"; pub(super) const PENDING_ARCHIVE: &str = "pending_archive"; pub(super) const PENDING_VIEW: &str = "pending_view"; pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; -pub(super) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -pub(super) const NODE_TYPE_MODEL: &str = "model"; pub(super) const LIST_RELATIONS_FOR_INSPECTION: (&str, &str) = ( "sqlbuild.adapter.relations.main._list_relations_for_inspection", "list_relations_for_inspection", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs index ac1fb2aa3d..86203c56d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs @@ -77,12 +77,7 @@ pub(super) fn decide<'py>(args: &KeywordArguments<'py>) -> PyResult( )?; let project_config = runtime.getattr("project_config")?; let snapshots_config = if project_config.is_none() { - imported(py, "sqlbuild.spec.contracts.models", "SnapshotsConfig")?.call0()? + imported(py, "sqlbuild._native", "SnapshotsConfig")?.call0()? } else { project_config.getattr("snapshots")? }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index e45c22183f..42506b327f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -23,7 +23,7 @@ impl<'py> OldNamePlanner<'py> { let py = runtime.py(); let project_config = runtime.getattr("project_config")?; let config = if project_config.is_none() { - imported(py, "sqlbuild.spec.contracts.models", "MigrationsConfig")?.call0()? + imported(py, "sqlbuild._native", "MigrationsConfig")?.call0()? } else { project_config.getattr("migrations")? }; @@ -282,7 +282,7 @@ impl<'py> OldNamePlanner<'py> { } } let retention = history.getattr("required")?.getattr("view_retention")?; - let duration = imported(py, "sqlbuild.cursor_algebra.models", "Duration")?.call_method1( + let duration = imported(py, "sqlbuild._native", "Duration")?.call_method1( "parse", (if retention.is_truthy()? { retention.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs index 9c8a9026c4..140befe160 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_objects.rs @@ -31,11 +31,7 @@ pub(super) fn location<'py>( )?; fields.push(("qualified_name", qualified)); call_with( - &imported( - runtime.py(), - "sqlbuild.compiler.compile.models", - "CompiledRelationLocation", - )?, + &imported(runtime.py(), "sqlbuild._native", "CompiledRelationLocation")?, &fields, ) } @@ -56,8 +52,8 @@ pub(super) fn expires<'py>( retention: &Bound<'py, PyAny>, now: &Bound<'py, PyAny>, ) -> PyResult> { - let duration = imported(now.py(), "sqlbuild.cursor_algebra.models", "Duration")? - .call_method1("parse", (retention,))?; + let duration = + imported(now.py(), "sqlbuild._native", "Duration")?.call_method1("parse", (retention,))?; if duration.is_none() { Ok(now.py().None().into_bound(now.py())) } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index d255cbc1eb..975eaeb167 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -19,11 +19,7 @@ pub(super) fn location<'py>( let mut fields = fields.to_vec(); fields.push(("qualified_name", qualified)); call_with( - &imported( - runtime.py(), - "sqlbuild.compiler.compile.models", - "CompiledRelationLocation", - )?, + &imported(runtime.py(), "sqlbuild._native", "CompiledRelationLocation")?, &fields, ) } @@ -65,12 +61,7 @@ fn named<'py>( fn input_error(py: Python<'_>, message: String) -> PyResult { Ok(PyErr::from_value( - imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )? - .call1((message,))?, + imported(py, "sqlbuild._native", "PlannerInputError")?.call1((message,))?, )) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs index 05ea22662c..ad53285986 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs @@ -6,22 +6,23 @@ use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyRef, PyRefMut, PyResult, Python, pyclass, pymethods, }; use pyo3::types::{PyBool, PyDict, PyString, PyTuple, PyTupleMethods}; -use sqlbuild_state::events::constants::{ - COLUMN_MIGRATION_TABLE_NAME, MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, -}; +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey, FingerprintOrder}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::constants::{ - FINGERPRINT_TABLE_NAME, GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, - NATIVE_MODULE, NODE_TYPE_MODEL, + GET_COLUMNS_FOR_INSPECTION, LIST_RELATIONS_FOR_INSPECTION, NATIVE_MODULE, }; use crate::bindings::_helpers::state_rows::constants::{COLUMN_KIND, MODEL_KIND, OLD_NAME_KIND}; use crate::bindings::_helpers::state_rows::migration_io::{read_events, stored_old_name_columns}; use crate::bindings::_helpers::state_rows::migration_objects::layout; use crate::bindings::_helpers::state_rows::migration_projections::old_name_views; use crate::bindings::_helpers::state_rows::models::StateAccess; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; /// A lowercased schema and relation name. type RelationKey = (String, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs deleted file mode 100644 index 7680642569..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Planner enumerations declared natively. - -/// Each planner `StrEnum`: its name and its members with their values, in declaration order. -pub(crate) const PLANNER_STR_ENUMS: &[(&str, &[(&str, &str)])] = &[ - ( - "GraphResourceKind", - &[ - ("MODEL", "model"), - ("SEED", "seed"), - ("SOURCE", "source"), - ("FUNCTION", "function"), - ], - ), - ( - "ChangeKind", - &[ - ("FIRST_RUN", "first_run"), - ("RENAMED", "renamed"), - ("QUERY_CHANGED", "query_changed"), - ("FUNCTION_CHANGED", "function_changed"), - ("CONFIG_CHANGED", "config_changed"), - ("SCHEMA_CHANGED", "schema_changed"), - ("NO_CHANGE", "no_change"), - ], - ), - ( - "SchemaChangeKind", - &[ - ("COLUMN_ADDED", "column_added"), - ("COLUMN_REMOVED", "column_removed"), - ("COLUMN_TYPE_CHANGED", "column_type_changed"), - ], - ), - ( - "SchemaColumnSource", - &[("YML", "yml"), ("SQL_ANALYSIS", "sql_analysis")], - ), - ("CursorWatermarkMode", &[("ALL", "all"), ("ANY", "any")]), - ( - "SchemaActionKind", - &[ - ("ADD_COLUMN", "add_column"), - ("DROP_COLUMN", "drop_column"), - ("ALTER_COLUMN_TYPE", "alter_column_type"), - ], - ), - ( - "WarningSeverity", - &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], - ), -]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs deleted file mode 100644 index cca4508a64..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Creates the planner `StrEnum` classes in the native module. - -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_models::constants::PLANNER_STR_ENUMS; -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyList}; - -pub(crate) fn register_enums(module: &Bound<'_, PyModule>) -> PyResult<()> { - let py = module.py(); - let str_enum = imported(py, "enum", "StrEnum")?; - for (name, members) in PLANNER_STR_ENUMS { - let options = PyDict::new(py); - options.set_item("module", "sqlbuild._native")?; - options.set_item("qualname", *name)?; - let class = str_enum.call((*name, PyList::new(py, members.iter())?), Some(&options))?; - module.add(*name, class)?; - } - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs deleted file mode 100644 index 0be40906ca..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Planner value records, enumerations and caches hosted natively. - -pub(crate) mod constants; -pub(crate) mod enums; -pub(crate) mod fixture_inferences; -pub(crate) mod record; -pub(crate) mod records; -pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs deleted file mode 100644 index f9cac51993..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs +++ /dev/null @@ -1,214 +0,0 @@ -//! Frozen value records with dataclass construction, equality, hashing and repr. - -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use pyo3::IntoPyObject; -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; - -/// How a record field is filled when the caller leaves it out. -pub(crate) enum FieldDefault { - /// The caller must pass the field. - Required, - /// `None`. - NoneValue, - /// A boolean. - Bool(bool), - /// A string. - Text(&'static str), - /// A new empty tuple. - EmptyTuple, - /// A new empty frozenset. - EmptyFrozenSet, - /// A new empty dict. - EmptyDict, - /// An enum member, by class and member name. - Member((&'static str, &'static str), &'static str), - /// A new instance of a class called without arguments. - Class((&'static str, &'static str)), - /// A value built by a function. - Factory(for<'py> fn(Python<'py>) -> PyResult>), -} - -/// One record field. -pub(crate) struct FieldSpec { - pub(crate) name: &'static str, - pub(crate) default: FieldDefault, - pub(crate) compare: bool, - pub(crate) repr: bool, -} - -/// The fields of one record class. -pub(crate) struct RecordSpec { - pub(crate) name: &'static str, - pub(crate) kw_only: bool, - pub(crate) fields: &'static [FieldSpec], -} - -fn default_value<'py>(py: Python<'py>, default: &FieldDefault) -> PyResult> { - match default { - FieldDefault::Required => Ok(py.None().into_bound(py)), - FieldDefault::NoneValue => Ok(py.None().into_bound(py)), - FieldDefault::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().into_any()), - FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), - FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), - FieldDefault::EmptyFrozenSet => Ok(PyFrozenSet::empty(py)?.into_any()), - FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), - FieldDefault::Member((module, class), member) => { - imported(py, module, class)?.getattr(*member) - } - FieldDefault::Class((module, class)) => imported(py, module, class)?.call0(), - FieldDefault::Factory(build) => build(py), - } -} - -/// Bind constructor arguments to the record's fields in order, as a dataclass `__init__` does. -pub(crate) fn bind_fields( - spec: &RecordSpec, - args: &Bound<'_, PyTuple>, - kwargs: Option<&Bound<'_, PyDict>>, -) -> PyResult>> { - let py = args.py(); - if args.len() > 0 && spec.kw_only { - return Err(PyTypeError::new_err(format!( - "{}.__init__() takes 1 positional argument but {} were given", - spec.name, - args.len() + 1 - ))); - } - if args.len() > spec.fields.len() { - return Err(PyTypeError::new_err(format!( - "{}.__init__() takes {} positional arguments but {} were given", - spec.name, - spec.fields.len() + 1, - args.len() + 1 - ))); - } - let mut values: Vec>> = args.iter().map(Some).collect(); - values.resize_with(spec.fields.len(), || None); - if let Some(kwargs) = kwargs { - for (key, value) in kwargs.iter() { - let key: String = key.extract()?; - let Some(index) = spec.fields.iter().position(|field| field.name == key) else { - return Err(PyTypeError::new_err(format!( - "{}.__init__() got an unexpected keyword argument '{key}'", - spec.name - ))); - }; - if values[index].is_some() { - return Err(PyTypeError::new_err(format!( - "{}.__init__() got multiple values for argument '{key}'", - spec.name - ))); - } - values[index] = Some(value); - } - } - spec.fields - .iter() - .zip(values) - .map(|(field, value)| match value { - Some(value) => Ok(value.unbind()), - None if matches!(field.default, FieldDefault::Required) => { - Err(PyTypeError::new_err(format!( - "{}.__init__() missing 1 required argument: '{}'", - spec.name, field.name - ))) - } - None => Ok(default_value(py, &field.default)?.unbind()), - }) - .collect() -} - -/// The current field values with `changes` applied, as `dataclasses.replace` binds them. -pub(crate) fn replaced_fields( - spec: &RecordSpec, - current: Vec>, - changes: Option<&Bound<'_, PyDict>>, -) -> PyResult>> { - let mut values: Vec> = current.into_iter().map(Bound::unbind).collect(); - let Some(changes) = changes else { - return Ok(values); - }; - for (key, value) in changes.iter() { - let key: String = key.extract()?; - let Some(index) = spec.fields.iter().position(|field| field.name == key) else { - return Err(PyTypeError::new_err(format!( - "{}.__init__() got an unexpected keyword argument '{key}'", - spec.name - ))); - }; - values[index] = value.unbind(); - } - Ok(values) -} - -fn compared<'py>( - py: Python<'py>, - spec: &RecordSpec, - values: &[Bound<'py, PyAny>], -) -> PyResult> { - let kept: Vec> = spec - .fields - .iter() - .zip(values) - .filter(|(field, _)| field.compare) - .map(|(_, value)| value.clone()) - .collect(); - PyTuple::new(py, kept) -} - -/// Dataclass equality: same class and equal compared fields. -pub(crate) fn record_eq<'py>( - spec: &RecordSpec, - (left, right): (&[Bound<'py, PyAny>], Option<&[Bound<'py, PyAny>]>), - py: Python<'py>, -) -> PyResult> { - let Some(right) = right else { - return Ok(py.NotImplemented()); - }; - let equal: bool = compared(py, spec, left)?.eq(compared(py, spec, right)?)?; - Ok(equal.into_pyobject(py)?.to_owned().into_any().unbind()) -} - -/// Dataclass hash: the hash of the compared fields. -pub(crate) fn record_hash<'py>( - spec: &RecordSpec, - values: &[Bound<'py, PyAny>], - py: Python<'py>, -) -> PyResult { - compared(py, spec, values)?.hash() -} - -/// Dataclass repr: `Name(field=value, ...)` over the represented fields. -pub(crate) fn record_repr(spec: &RecordSpec, values: &[Bound<'_, PyAny>]) -> PyResult { - let mut parts: Vec = Vec::new(); - for (field, value) in spec.fields.iter().zip(values) { - if field.repr { - parts.push(format!("{}={}", field.name, value.repr()?)); - } - } - Ok(format!("{}({})", spec.name, parts.join(", "))) -} - -/// Give a record class the dataclass field table `dataclasses.replace`, `fields` and `asdict` read. -pub(crate) fn install_dataclass_fields( - class: &Bound<'_, PyAny>, - spec: &RecordSpec, -) -> PyResult<()> { - let py = class.py(); - let dataclasses = py.import("dataclasses")?; - let field_kind = dataclasses.getattr("_FIELD")?; - let table = PyDict::new(py); - for field in spec.fields { - let options = PyDict::new(py); - options.set_item("compare", field.compare)?; - options.set_item("repr", field.repr)?; - let described = dataclasses.call_method("field", (), Some(&options))?; - described.setattr("name", field.name)?; - described.setattr("type", "object")?; - described.setattr("_field_type", &field_kind)?; - table.set_item(field.name, described)?; - } - class.setattr("__dataclass_fields__", table) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs deleted file mode 100644 index bcba81d3e3..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs +++ /dev/null @@ -1,775 +0,0 @@ -//! Planner value records the native planner builds and reads. - -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_models::record::FieldDefault; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyDict; - -/// Declares a frozen record class over `Py` fields. -macro_rules! planner_record { - (@convert $py:ident, $values:expr) => {{ - let _ = $py; - $values - }}; - (@convert $py:ident, $values:expr, $convert:path) => { $convert($py, $values)? }; - ( - $(#[$meta:meta])* - $name:ident, $kw_only:literal, [ - $(($field:ident = $field_name:literal, $default:expr, $compare:literal, $repr:literal)),* $(,)? - ] - $(, convert = $convert:path)? - $(, methods { $($methods:tt)* })? - ) => { - $(#[$meta])* - #[pyo3::pyclass(module = "sqlbuild._native", frozen)] - pub(crate) struct $name { - $( - #[pyo3(get)] - $field: pyo3::Py, - )* - } - - impl $name { - const SPEC: $crate::bindings::_helpers::planner_models::record::RecordSpec = - $crate::bindings::_helpers::planner_models::record::RecordSpec { - name: stringify!($name), - kw_only: $kw_only, - fields: &[$( - $crate::bindings::_helpers::planner_models::record::FieldSpec { - name: $field_name, - default: $default, - compare: $compare, - repr: $repr, - }, - )*], - }; - - /// Add the class to `module` with its dataclass field table. - pub(crate) fn register_class( - module: &pyo3::Bound<'_, pyo3::types::PyModule>, - ) -> pyo3::PyResult<()> { - use pyo3::types::{PyAnyMethods, PyModuleMethods}; - module.add_class::()?; - $crate::bindings::_helpers::planner_models::record::install_dataclass_fields( - &module.getattr(stringify!($name))?, - &Self::SPEC, - ) - } - - fn values<'py>(&self, py: pyo3::Python<'py>) -> Vec> { - vec![$(self.$field.bind(py).clone()),*] - } - - fn from_values(values: Vec>) -> pyo3::PyResult { - let mut values = values.into_iter(); - Ok(Self { - $( - $field: values.next().ok_or_else(|| { - pyo3::exceptions::PyTypeError::new_err(concat!("missing ", $field_name)) - })?, - )* - }) - } - } - - #[pyo3::pymethods] - impl $name { - #[new] - #[pyo3(signature = (*args, **kwargs))] - fn __new__( - args: &pyo3::Bound<'_, pyo3::types::PyTuple>, - kwargs: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, - ) -> pyo3::PyResult { - let py = args.py(); - let values = $crate::bindings::_helpers::planner_models::record::bind_fields( - &Self::SPEC, - args, - kwargs, - )?; - Self::from_values(planner_record!(@convert py, values $(, $convert)?)) - } - - #[pyo3(signature = (**changes))] - fn __replace__( - &self, - py: pyo3::Python<'_>, - changes: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, - ) -> pyo3::PyResult { - let values = $crate::bindings::_helpers::planner_models::record::replaced_fields( - &Self::SPEC, - self.values(py), - changes, - )?; - Self::from_values(planner_record!(@convert py, values $(, $convert)?)) - } - - fn __eq__( - &self, - other: &pyo3::Bound<'_, pyo3::PyAny>, - ) -> pyo3::PyResult> { - let py = other.py(); - let right = match other.cast::() { - Ok(right) => Some(right.get().values(py)), - Err(_) => None, - }; - $crate::bindings::_helpers::planner_models::record::record_eq( - &Self::SPEC, - (&self.values(py), right.as_deref()), - py, - ) - } - - fn __hash__(&self, py: pyo3::Python<'_>) -> pyo3::PyResult { - $crate::bindings::_helpers::planner_models::record::record_hash( - &Self::SPEC, - &self.values(py), - py, - ) - } - - fn __repr__(&self, py: pyo3::Python<'_>) -> pyo3::PyResult { - $crate::bindings::_helpers::planner_models::record::record_repr( - &Self::SPEC, - &self.values(py), - ) - } - - $($($methods)*)? - } - }; -} - -fn forward_only_backfill(py: Python<'_>) -> PyResult> { - let kwargs = PyDict::new(py); - kwargs.set_item( - "action", - imported(py, "sqlbuild.compiler.planner.types", "BackfillAction")? - .getattr("FORWARD_ONLY")?, - )?; - imported(py, "sqlbuild._native", "BackfillResult")?.call((), Some(&kwargs)) -} - -fn utc_now(py: Python<'_>) -> PyResult> { - imported(py, "datetime", "datetime")?.call_method1("now", (imported(py, "datetime", "UTC")?,)) -} - -fn scenario_artifact_kind(py: Python<'_>, mut values: Vec>) -> PyResult>> { - if let Some(kind) = values.first_mut() { - *kind = imported( - py, - "sqlbuild.compiler.planner.types", - "ScenarioArtifactKind", - )? - .call1((kind.bind(py),))? - .unbind(); - } - Ok(values) -} - -fn rename_hint_message( - identical: &Bound<'_, PyAny>, - candidates: &Bound<'_, PyAny>, - added: &Bound<'_, PyAny>, -) -> PyResult { - let relation: &str = if identical.is_truthy()? { - "same expression as" - } else { - "similar to" - }; - let candidates: Vec = candidates - .try_iter()? - .map(|candidate| Ok(candidate?.str()?.to_string())) - .collect::>()?; - let origin: &str = match candidates.as_slice() { - [only] => only, - _ => "", - }; - Ok(format!( - "{relation} {}; if this is a rename, add {} (migrate_from {origin})", - candidates.join(", "), - added.str()? - )) -} - -planner_record! { - /// One native-planned SQL-test artifact and its distinct planning error messages. - NativeSqlTestArtifact, false, [ - (sql = "sql", FieldDefault::Required, true, true), - (model_names = "model_names", FieldDefault::Required, true, true), - (error_messages = "error_messages", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// One statically provable invalid fixture condition. - RelationFixtureDiagnostic, false, [ - (key = "key", FieldDefault::Required, true, true), - (message = "message", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Completed fixture SQL plus metadata reused by SQL-test validation. - RelationFixtureCompletion, false, [ - (fixture_sql_by_key = "fixture_sql_by_key", FieldDefault::Required, true, true), - (inferred_by_fixture = "inferred_by_fixture", FieldDefault::Required, true, true), - (expected_types = "expected_types", FieldDefault::Required, true, true), - (diagnostics = "diagnostics", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Known type and nullability for one relation fixture column. - FixtureColumnMetadata, false, [ - (name = "name", FieldDefault::Required, true, true), - (r#type = "type", FieldDefault::Required, true, true), - (nullable = "nullable", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Known columns and whether they form an authoritative relation shape. - FixtureRelationMetadata, false, [ - (columns = "columns", FieldDefault::Required, true, true), - (authoritative_names = "authoritative_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Project-wide fixture metadata shared across test and scenario planning. - RelationFixturePlanningContext, false, [ - (models_by_name = "models_by_name", FieldDefault::Required, true, true), - (relations = "relations", FieldDefault::Required, true, true), - (authoritative_columns = "authoritative_columns", FieldDefault::Required, true, true), - (expected_types = "expected_types", FieldDefault::Required, true, true), - (fixture_inferences = "fixture_inferences", FieldDefault::Class(("sqlbuild._native", "FixtureColumnInferences")), false, true), - ] -} - -planner_record! { - /// One selector split into graph expansion flags and core text. - SelectorExpansion, false, [ - (core = "core", FieldDefault::Required, true, true), - (upstream = "upstream", FieldDefault::Bool(false), true, true), - (downstream = "downstream", FieldDefault::Bool(false), true, true), - ] -} - -planner_record! { - /// Parsed physical name for one planner-owned scenario artifact. - ParsedScenarioArtifactName, false, [ - (hash_prefix = "hash_prefix", FieldDefault::Required, true, true), - (kind = "kind", FieldDefault::Required, true, true), - (logical_name = "logical_name", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// One upstream dependency missing from both scope and warehouse. - MissingUpstream, false, [ - (key = "key", FieldDefault::Required, true, true), - (required_by = "required_by", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Latest direct fingerprints grouped by node type. - WarehouseFingerprints, false, [ - (models = "models", FieldDefault::EmptyDict, true, true), - (functions = "functions", FieldDefault::EmptyDict, true, true), - (seeds = "seeds", FieldDefault::EmptyDict, true, true), - (python_nodes = "python_nodes", FieldDefault::EmptyDict, true, true), - (unfiltered_schemas = "unfiltered_schemas", FieldDefault::EmptyDict, false, false), - (unfiltered_database = "unfiltered_database", FieldDefault::NoneValue, false, false), - ] -} - -planner_record! { - /// Execution selection used for cursor validation and runtime producer resolution. - CursorSnapshotScope, false, [ - (model_keys = "model_keys", FieldDefault::Required, true, true), - (runtime_producer_keys = "runtime_producer_keys", FieldDefault::Required, true, true), - (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), - (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), - (cursor_overrides = "cursor_overrides", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// One model rename that downstream references may have been rewritten for. - ReferenceRename, false, [ - (new_name = "new_name", FieldDefault::Required, true, true), - (origin_name = "origin_name", FieldDefault::Required, true, true), - (recorded_at = "recorded_at", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Frozen point-in-time picture of warehouse state for planning. - WarehouseSnapshot, false, [ - (existing_relations = "existing_relations", FieldDefault::EmptyDict, true, true), - (existing_columns = "existing_columns", FieldDefault::EmptyDict, true, true), - (fingerprints = "fingerprints", FieldDefault::Class(("sqlbuild._native", "WarehouseFingerprints")), true, true), - (cursor_snapshots = "cursor_snapshots", FieldDefault::EmptyDict, true, true), - (source_freshness_state_schemas = "source_freshness_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - (column_dialect = "column_dialect", FieldDefault::NoneValue, true, true), - (renamed_models = "renamed_models", FieldDefault::EmptyFrozenSet, true, true), - (reference_renames = "reference_renames", FieldDefault::EmptyTuple, true, true), - (migration_state_schemas = "migration_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - (old_name_view_state_schemas = "old_name_view_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - (listed_state_schemas = "listed_state_schemas", FieldDefault::EmptyFrozenSet, true, true), - ] -} - -planner_record! { - /// Neutral node identity for shared selection-aware staleness classification. - SelectionStalenessNodeKey, false, [ - (resource_type = "resource_type", FieldDefault::Required, true, true), - (name = "name", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Neutral graph inputs for selected-model stale warning classification. - SelectionStalenessGraph, false, [ - (upstream_deps = "upstream_deps", FieldDefault::Required, true, true), - (selected_model_names = "selected_model_names", FieldDefault::Required, true, true), - (run_model_names = "run_model_names", FieldDefault::Required, true, true), - (run_seed_names = "run_seed_names", FieldDefault::Required, true, true), - (run_source_names = "run_source_names", FieldDefault::Required, true, true), - (changed_model_names = "changed_model_names", FieldDefault::Required, true, true), - (changed_seed_names = "changed_seed_names", FieldDefault::Required, true, true), - (changed_source_names = "changed_source_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Neutral stale warning classification for one selected model. - SelectionStalenessWarning, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (trigger_names = "trigger_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Resolved backfill action from a change detection policy. - BackfillResult, false, [ - (action = "action", FieldDefault::Required, true, true), - (duration = "duration", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Per-model output from change detection and policy resolution. - ChangeDetectionResult, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (change_kind = "change_kind", FieldDefault::Required, true, true), - (query_changed = "query_changed", FieldDefault::Bool(false), true, true), - (config_changed = "config_changed", FieldDefault::Bool(false), true, true), - (fingerprint_metadata_json = "fingerprint_metadata_json", FieldDefault::NoneValue, true, true), - (previous_metadata_json = "previous_metadata_json", FieldDefault::NoneValue, true, true), - (fingerprint_version_hash = "fingerprint_version_hash", FieldDefault::NoneValue, true, true), - (previous_version_hash = "previous_version_hash", FieldDefault::NoneValue, true, true), - (schema_findings = "schema_findings", FieldDefault::EmptyTuple, true, true), - (backfill = "backfill", FieldDefault::Factory(forward_only_backfill), true, true), - (recorded_build_relation_missing = "recorded_build_relation_missing", FieldDefault::Bool(false), true, true), - (changed_functions = "changed_functions", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Resource selectors left after unit-test selectors moved into a test selection. - UnitTestSelectorSplit, false, [ - (select = "select", FieldDefault::Required, true, true), - (exclude = "exclude", FieldDefault::Required, true, true), - (sql_test_selection = "sql_test_selection", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Relation lookups and source columns for building one model plan entry. - ModelPlanContext, false, [ - (model_locations = "model_locations", FieldDefault::Required, true, true), - (models_by_name = "models_by_name", FieldDefault::Required, true, true), - (seed_locations = "seed_locations", FieldDefault::Required, true, true), - (function_locations = "function_locations", FieldDefault::Required, true, true), - (source_map = "source_map", FieldDefault::Required, true, true), - (source_warehouse_columns = "source_warehouse_columns", FieldDefault::Required, true, true), - (star_exclude_keyword = "star_exclude_keyword", FieldDefault::Required, true, true), - (functions_by_name = "functions_by_name", FieldDefault::EmptyDict, true, true), - (runtime_cursor_producer_names = "runtime_cursor_producer_names", FieldDefault::EmptyFrozenSet, true, true), - (future_cursor_config = "future_cursor_config", FieldDefault::NoneValue, true, true), - (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), - (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Resolved per-model cursor start/end overrides for plan entry construction. - CursorOverridePair, false, [ - (start_cursor_override = "start_cursor_override", FieldDefault::NoneValue, true, true), - (end_cursor_override = "end_cursor_override", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Blocked models and cursor overrides for plan entry building. - PlanEntryBuildInputs, false, [ - (source_freshness_blocked_model_names = "source_freshness_blocked_model_names", FieldDefault::EmptyFrozenSet, true, true), - (external_blocked_model_names = "external_blocked_model_names", FieldDefault::EmptyFrozenSet, true, true), - (start_cursor_override = "start_cursor_override", FieldDefault::NoneValue, true, true), - (end_cursor_override = "end_cursor_override", FieldDefault::NoneValue, true, true), - (future_cursor_config = "future_cursor_config", FieldDefault::NoneValue, true, true), - (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), - (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), - (max_microbatches = "max_microbatches", FieldDefault::NoneValue, true, true), - (max_microbatches_is_override = "max_microbatches_is_override", FieldDefault::Bool(false), true, true), - (microbatch_limit_action = "microbatch_limit_action", FieldDefault::Member(("sqlbuild.spec.contracts.types", "MicrobatchLimitAction"), "ERROR"), true, true), - ] -} - -planner_record! { - /// Per-function output from change detection. - FunctionChangeResult, false, [ - (fingerprint_sql = "fingerprint_sql", FieldDefault::Required, true, true), - (reason = "reason", FieldDefault::Member(("sqlbuild.compiler.planner.types", "PlanReason"), "NO_CHANGE"), true, true), - ] -} - -planner_record! { - /// Change detection output for selected planner resources. - PlannerChangeResults, false, [ - (models = "models", FieldDefault::Required, true, true), - (functions = "functions", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Effective model change and backfill decided from the model's own change. - ResolvedModelAction, false, [ - (change = "change", FieldDefault::Required, true, true), - (backfill = "backfill", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Per-model planning decisions keyed by model name. - PlannerResolvedActions, false, [ - (models = "models", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Current direct model version identity values by model name. - DirectModelVersionIdentities, false, [ - (function_local_hashes = "function_local_hashes", FieldDefault::Required, true, true), - (seed_version_hashes = "seed_version_hashes", FieldDefault::Required, true, true), - (seed_metadata_jsons = "seed_metadata_jsons", FieldDefault::Required, true, true), - (model_metadata_jsons = "model_metadata_jsons", FieldDefault::Required, true, true), - (model_local_hashes = "model_local_hashes", FieldDefault::Required, true, true), - (model_version_hashes = "model_version_hashes", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Model plan-entry phase output. - PlannerModelEntryResults, false, [ - (entries = "entries", FieldDefault::Required, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// One concrete schema change action to apply to a target table. - SchemaAction, false, [ - (kind = "kind", FieldDefault::Required, true, true), - (column_name = "column_name", FieldDefault::Required, true, true), - (column_type = "column_type", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// One warning or error produced during plan resolution. - PlanWarning, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (severity = "severity", FieldDefault::Required, true, true), - (message = "message", FieldDefault::Required, true, true), - (code = "code", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// A removed column that an added column may have been renamed from. - ColumnRenameHint, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (added_column = "added_column", FieldDefault::Required, true, true), - (candidate_columns = "candidate_columns", FieldDefault::Required, true, true), - (identical = "identical", FieldDefault::Bool(false), true, true), - ], - methods { - /// The one-line hint shown beside the added column. - #[getter] - fn message(&self, py: Python<'_>) -> PyResult { - rename_hint_message( - self.identical.bind(py), - self.candidate_columns.bind(py), - self.added_column.bind(py), - ) - } - } -} - -planner_record! { - /// One named top-level output column of a query and its canonical expression. - QueryProjection, false, [ - (name = "name", FieldDefault::Required, true, true), - (expression = "expression", FieldDefault::Required, true, true), - (references = "references", FieldDefault::Required, true, true), - (passthrough = "passthrough", FieldDefault::Bool(false), true, true), - ], - methods { - /// The case-insensitive output name. - #[getter] - fn key<'py>(&self, py: Python<'py>) -> PyResult> { - self.name.bind(py).call_method0("lower") - } - } -} - -planner_record! { - /// Canonical top-level projections and the rest of one query, ignoring formatting. - QueryShape, false, [ - (projections = "projections", FieldDefault::Required, true, true), - (body = "body", FieldDefault::Required, true, true), - (alias_clauses = "alias_clauses", FieldDefault::Text("{}"), true, true), - (positional = "positional", FieldDefault::Bool(false), true, true), - (input_relations = "input_relations", FieldDefault::NoneValue, true, true), - (local_input_columns = "local_input_columns", FieldDefault::EmptyFrozenSet, true, true), - (clause_references = "clause_references", FieldDefault::EmptyFrozenSet, true, true), - ] -} - -planner_record! { - /// Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly. - PlanOutputExtras, false, [ - (seed_version_hashes = "seed_version_hashes", FieldDefault::NoneValue, true, true), - (seed_metadata_jsons = "seed_metadata_jsons", FieldDefault::NoneValue, true, true), - (seed_plan_reasons = "seed_plan_reasons", FieldDefault::NoneValue, true, true), - (planned_sql_tests = "planned_sql_tests", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// SQL test entries planned for one selection; reusable only for the same selection. - PlannedSqlTests, false, [ - (selected_keys = "selected_keys", FieldDefault::Required, true, true), - (sql_test_selection = "sql_test_selection", FieldDefault::Class(("sqlbuild._native", "SqlTestSelection")), true, true), - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Native chain and assertion steps for one SQL test, with SQL when rendered. - NativeSqlTestPlan, false, [ - (chain = "chain", FieldDefault::Required, true, true), - (assertions = "assertions", FieldDefault::Required, true, true), - (model_names = "model_names", FieldDefault::Required, true, true), - (warnings = "warnings", FieldDefault::Required, true, true), - (error_messages = "error_messages", FieldDefault::Required, true, true), - (sql = "sql", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// One SQL test's plan entry and warnings, or the fixture diagnostics preventing it. - SqlTestPlanResult, false, [ - (entry = "entry", FieldDefault::Required, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - (fixture_diagnostics = "fixture_diagnostics", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Inferred graph slice and fixture boundaries for one SQL scenario. - ScenarioGraphPlan, false, [ - (key = "key", FieldDefault::Required, true, true), - (name = "name", FieldDefault::Required, true, true), - (target_model_names = "target_model_names", FieldDefault::EmptyTuple, true, true), - (assertion_target_model_names = "assertion_target_model_names", FieldDefault::EmptyTuple, true, true), - (model_names = "model_names", FieldDefault::EmptyTuple, true, true), - (source_fixture_names = "source_fixture_names", FieldDefault::EmptyTuple, true, true), - (ref_fixture_names = "ref_fixture_names", FieldDefault::EmptyTuple, true, true), - (seed_names = "seed_names", FieldDefault::EmptyTuple, true, true), - (seed_fixture_names = "seed_fixture_names", FieldDefault::EmptyTuple, true, true), - (dbt_ref_fixture_names = "dbt_ref_fixture_names", FieldDefault::EmptyTuple, true, true), - (function_deps = "function_deps", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Logical identity for one scenario-owned physical artifact. - ScenarioArtifactIdentity, false, [ - (kind = "kind", FieldDefault::Required, true, true), - (logical_name = "logical_name", FieldDefault::Required, true, true), - ], - convert = scenario_artifact_kind -} - -planner_record! { - /// Resolved physical relation name for one scenario artifact. - ScenarioArtifactName, false, [ - (identity = "identity", FieldDefault::Required, true, true), - (physical_name = "physical_name", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Provider usage metadata scoped to planned work. - PlanProviderUsage, false, [ - (provider_name = "provider_name", FieldDefault::Required, true, true), - (consumer_kind = "consumer_kind", FieldDefault::Required, true, true), - (consumer_name = "consumer_name", FieldDefault::Required, true, true), - (parameter_name = "parameter_name", FieldDefault::Required, true, true), - (annotation_class_name = "annotation_class_name", FieldDefault::NoneValue, true, true), - (annotation_module = "annotation_module", FieldDefault::NoneValue, true, true), - ] -} - -planner_record! { - /// Resolved execution environment shared by planner phases. - PlannerRuntime, false, [ - (project = "project", FieldDefault::Required, true, true), - (adapter = "adapter", FieldDefault::Required, true, true), - (connection = "connection", FieldDefault::Required, true, true), - (project_config = "project_config", FieldDefault::NoneValue, true, true), - (local_config = "local_config", FieldDefault::NoneValue, true, true), - (on_progress = "on_progress", FieldDefault::NoneValue, true, true), - (invocation_time = "invocation_time", FieldDefault::Factory(utc_now), true, true), - ] -} - -planner_record! { - /// Resolved planner scopes for one plan build. - PlannerScopeResolution, false, [ - (selected_scope = "selected_scope", FieldDefault::Required, true, true), - (stale_warning_scope = "stale_warning_scope", FieldDefault::Required, true, true), - (inspection_scope = "inspection_scope", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Warehouse snapshot and inspection relations gathered once per plan. - PlannerWarehouseState, false, [ - (snapshot = "snapshot", FieldDefault::Required, true, true), - (inspection_relations = "inspection_relations", FieldDefault::Required, true, true), - (migration_entries = "migration_entries", FieldDefault::EmptyTuple, true, true), - (migration_warnings = "migration_warnings", FieldDefault::EmptyTuple, true, true), - (column_migration_entries = "column_migration_entries", FieldDefault::EmptyTuple, true, true), - (column_rename_hints = "column_rename_hints", FieldDefault::EmptyTuple, true, true), - (old_name_view_entries = "old_name_view_entries", FieldDefault::EmptyTuple, true, true), - (old_name_views = "old_name_views", FieldDefault::EmptyTuple, true, true), - (migration_fingerprints = "migration_fingerprints", FieldDefault::Class(("sqlbuild._native", "MigrationFingerprintCache")), true, true), - ] -} - -planner_record! { - /// One destination model whose data should come from an earlier relation. - ModelMigrationRequest, false, [ - (model = "model", FieldDefault::Required, true, true), - (discovery = "discovery", FieldDefault::Required, true, true), - (raw_origin = "raw_origin", FieldDefault::NoneValue, true, true), - (origin_location = "origin_location", FieldDefault::NoneValue, true, true), - (origin_model = "origin_model", FieldDefault::NoneValue, true, true), - (force = "force", FieldDefault::Bool(false), true, true), - (identity_only = "identity_only", FieldDefault::Bool(false), true, true), - ] -} - -planner_record! { - /// One project model's explicit migrate_from, resolved to a relation. - ModelMigrationDeclaration, false, [ - (model_name = "model_name", FieldDefault::Required, true, true), - (origin_location = "origin_location", FieldDefault::Required, true, true), - (origin_model = "origin_model", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Manual and automatically discovered migration requests for one plan. - ModelMigrationDiscovery, false, [ - (requests = "requests", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - (destination_fingerprints = "destination_fingerprints", FieldDefault::EmptyDict, true, true), - ] -} - -planner_record! { - /// Compatibility status plus human-readable blocking findings. - MigrationCompatibilityResult, false, [ - (status = "status", FieldDefault::Required, true, true), - (findings = "findings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Migration decisions plus the warehouse snapshot as it will look after moves. - ModelMigrationPlanning, false, [ - (snapshot = "snapshot", FieldDefault::Required, true, true), - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Old-name steps for this build plus every compatibility view still recorded. - OldNameViewPlanning, false, [ - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (views = "views", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Column rename decisions plus the warehouse snapshot as it will look after renames. - ColumnMigrationPlanning, false, [ - (snapshot = "snapshot", FieldDefault::Required, true, true), - (entries = "entries", FieldDefault::EmptyTuple, true, true), - (hints = "hints", FieldDefault::EmptyTuple, true, true), - (warnings = "warnings", FieldDefault::EmptyTuple, true, true), - ] -} - -planner_record! { - /// Expected version identities for inspection and stale-warning scopes. - PlannerIdentityContext, false, [ - (version_identities = "version_identities", FieldDefault::Required, true, true), - (stale_warning_identities = "stale_warning_identities", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Scope pruning outcome with pruned scopes, actions, and staleness state. - PlannerScopePruningResult, false, [ - (inspection_scope = "inspection_scope", FieldDefault::Required, true, true), - (execution_scope = "execution_scope", FieldDefault::Required, true, true), - (resolved_actions = "resolved_actions", FieldDefault::Required, true, true), - (pruned_direct_model_names = "pruned_direct_model_names", FieldDefault::Required, true, true), - (direct_identity_stale_model_names = "direct_identity_stale_model_names", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Write-hash-honest changes merged back into resolved actions. - PlannerChangeReconciliation, false, [ - (changes = "changes", FieldDefault::Required, true, true), - (resolved_actions = "resolved_actions", FieldDefault::Required, true, true), - ] -} - -planner_record! { - /// Model plan entries for one plan build. - PlannerEntryResults, false, [ - (model_entry_results = "model_entry_results", FieldDefault::Required, true, true), - ] -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs deleted file mode 100644 index 242379387b..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! Registers the planner value records. - -use crate::bindings::_helpers::planner_models::enums::register_enums; -use crate::bindings::_helpers::planner_models::fixture_inferences::FixtureColumnInferences; -use crate::bindings::_helpers::planner_models::records::{ - BackfillResult, ChangeDetectionResult, ColumnMigrationPlanning, ColumnRenameHint, - CursorOverridePair, CursorSnapshotScope, DirectModelVersionIdentities, FixtureColumnMetadata, - FixtureRelationMetadata, FunctionChangeResult, MigrationCompatibilityResult, MissingUpstream, - ModelMigrationDeclaration, ModelMigrationDiscovery, ModelMigrationPlanning, - ModelMigrationRequest, ModelPlanContext, NativeSqlTestArtifact, NativeSqlTestPlan, - OldNameViewPlanning, ParsedScenarioArtifactName, PlanEntryBuildInputs, PlanOutputExtras, - PlanProviderUsage, PlanWarning, PlannedSqlTests, PlannerChangeReconciliation, - PlannerChangeResults, PlannerEntryResults, PlannerIdentityContext, PlannerModelEntryResults, - PlannerResolvedActions, PlannerRuntime, PlannerScopePruningResult, PlannerScopeResolution, - PlannerWarehouseState, QueryProjection, QueryShape, ReferenceRename, RelationFixtureCompletion, - RelationFixtureDiagnostic, RelationFixturePlanningContext, ResolvedModelAction, - ScenarioArtifactIdentity, ScenarioArtifactName, ScenarioGraphPlan, SchemaAction, - SelectionStalenessGraph, SelectionStalenessNodeKey, SelectionStalenessWarning, - SelectorExpansion, SqlTestPlanResult, UnitTestSelectorSplit, WarehouseFingerprints, - WarehouseSnapshot, -}; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - register_enums(module)?; - module.add_class::()?; - NativeSqlTestArtifact::register_class(module)?; - RelationFixtureDiagnostic::register_class(module)?; - RelationFixtureCompletion::register_class(module)?; - FixtureColumnMetadata::register_class(module)?; - FixtureRelationMetadata::register_class(module)?; - RelationFixturePlanningContext::register_class(module)?; - SelectorExpansion::register_class(module)?; - ParsedScenarioArtifactName::register_class(module)?; - MissingUpstream::register_class(module)?; - WarehouseFingerprints::register_class(module)?; - CursorSnapshotScope::register_class(module)?; - ReferenceRename::register_class(module)?; - WarehouseSnapshot::register_class(module)?; - SelectionStalenessNodeKey::register_class(module)?; - SelectionStalenessGraph::register_class(module)?; - SelectionStalenessWarning::register_class(module)?; - BackfillResult::register_class(module)?; - ChangeDetectionResult::register_class(module)?; - UnitTestSelectorSplit::register_class(module)?; - ModelPlanContext::register_class(module)?; - CursorOverridePair::register_class(module)?; - PlanEntryBuildInputs::register_class(module)?; - FunctionChangeResult::register_class(module)?; - PlannerChangeResults::register_class(module)?; - ResolvedModelAction::register_class(module)?; - PlannerResolvedActions::register_class(module)?; - DirectModelVersionIdentities::register_class(module)?; - PlannerModelEntryResults::register_class(module)?; - SchemaAction::register_class(module)?; - PlanWarning::register_class(module)?; - ColumnRenameHint::register_class(module)?; - QueryProjection::register_class(module)?; - QueryShape::register_class(module)?; - PlanOutputExtras::register_class(module)?; - PlannedSqlTests::register_class(module)?; - NativeSqlTestPlan::register_class(module)?; - SqlTestPlanResult::register_class(module)?; - ScenarioGraphPlan::register_class(module)?; - ScenarioArtifactIdentity::register_class(module)?; - ScenarioArtifactName::register_class(module)?; - PlanProviderUsage::register_class(module)?; - PlannerRuntime::register_class(module)?; - PlannerScopeResolution::register_class(module)?; - PlannerWarehouseState::register_class(module)?; - ModelMigrationRequest::register_class(module)?; - ModelMigrationDeclaration::register_class(module)?; - ModelMigrationDiscovery::register_class(module)?; - MigrationCompatibilityResult::register_class(module)?; - ModelMigrationPlanning::register_class(module)?; - OldNameViewPlanning::register_class(module)?; - ColumnMigrationPlanning::register_class(module)?; - PlannerIdentityContext::register_class(module)?; - PlannerScopePruningResult::register_class(module)?; - PlannerChangeReconciliation::register_class(module)?; - PlannerEntryResults::register_class(module)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index f80cc80940..957bc533a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -1,12 +1,12 @@ //! Python modules and names the planner phases exchange values with. pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; -pub(crate) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; -pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(crate) const PLANNER_MODELS: &str = "sqlbuild._native"; +pub(crate) const PLANNER_TYPES: &str = "sqlbuild._native"; +pub(crate) const PLANNER_EXCEPTIONS: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS: &str = "sqlbuild.adapter.contract.models"; pub(crate) const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; -pub(crate) const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const SPEC_MODELS: &str = "sqlbuild._native"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; pub(crate) const BIGQUERY_ADAPTER: &str = "bigquery"; pub(crate) const TRANSIENT_TABLE_TYPE: &str = "transient"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index cc5e3514be..cce89afb2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -161,7 +161,7 @@ pub(crate) fn model_key<'py>( name: &Bound<'py, PyAny>, ) -> PyResult> { call_with( - &imported(py, "sqlbuild.compiler.compile.models", "CompiledObjectKey")?, + &imported(py, "sqlbuild._native", "CompiledObjectKey")?, &[ ( "resource_type", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs new file mode 100644 index 0000000000..9d4703965a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -0,0 +1,1696 @@ +//! Layouts of the planner, migration, cursor and fingerprint value records hosted natively. + +use crate::bindings::_helpers::build_command::support::utc_now; +use crate::bindings::_helpers::record_behaviours::planner::forward_only_backfill; +use crate::bindings::_helpers::records::models::{EnumSpec, FieldDefault, FieldSpec, RecordSpec}; + +const R_PARSED_SELECTOR: RecordSpec = RecordSpec { + name: "ParsedSelector", + doc: "One parsed selector token before graph resolution.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("value"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_PATH_SELECTOR: RecordSpec = RecordSpec { + name: "PathSelector", + doc: "A directed path selector between two model names with optional endpoint expansion.", + fields: &[ + FieldSpec::required("start_name"), + FieldSpec::required("end_name"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_SQL_TEST_SELECTION: RecordSpec = RecordSpec { + name: "SqlTestSelection", + doc: "Unit tests selected or excluded by name, beside the tests of the selected resources.", + fields: &[ + FieldSpec::new("names", FieldDefault::EmptyFrozenset), + FieldSpec::new("excluded_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("models_selected", FieldDefault::True), + ], +}; +const R_PLANNER_SCOPE: RecordSpec = RecordSpec { + name: "PlannerScope", + doc: "Resolved graph scope for one planner invocation.", + fields: &[ + FieldSpec::required("upstream_deps"), + FieldSpec::required("downstream_deps"), + FieldSpec::required("all_keys"), + FieldSpec::required("models_by_name"), + FieldSpec::required("selected_keys"), + FieldSpec::required("execution_order"), + FieldSpec::new("user_selected_keys", FieldDefault::EmptyFrozenset), + FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PLANNER_RELATIONS_CONTEXT: RecordSpec = RecordSpec { + name: "PlannerRelationsContext", + doc: "Resolved relation and source inputs for plan entry construction.", + fields: &[ + FieldSpec::required("model_locations"), + FieldSpec::required("seed_locations"), + FieldSpec::required("function_locations"), + FieldSpec::required("source_map"), + FieldSpec::required("source_read_map"), + FieldSpec::required("python_source_read_map"), + FieldSpec::required("source_warehouse_columns"), + FieldSpec::required("star_exclude_keyword"), + FieldSpec::new("listed_source_names", FieldDefault::EmptyFrozenset), + ], +}; +const R_SQL_TEST_ASSERTION_STEP: RecordSpec = RecordSpec { + name: "SqlTestAssertionStep", + doc: "One zero-row assertion step in a SQL-native unit test.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("resolved_sql"), + FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("comparison_body_sql", FieldDefault::None), + ], +}; +const R_PLANNER_SELECTION: RecordSpec = RecordSpec { + name: "PlannerSelection", + doc: "User-facing selection inputs for one planner invocation.", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("selected_keys", FieldDefault::None), + FieldSpec::new("python_read_source_names", FieldDefault::EmptyFrozenset), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + ], +}; +const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { + name: "PlannerOverrides", + doc: "Explicit user overrides that force or bound planner decisions.", + fields: &[ + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("reload_sources", FieldDefault::False), + FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("max_microbatches", FieldDefault::None), + ], +}; +const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { + name: "DeferralInputs", + doc: "Deferred relation and source-deferral inputs for planning.", + fields: &[ + FieldSpec::new("deferred_locations", FieldDefault::None), + FieldSpec::new("deferred_relations", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("source_deferral_enabled", FieldDefault::True), + ], +}; +const R_PLANNER_POLICIES: RecordSpec = RecordSpec { + name: "PlannerPolicies", + doc: "Behavior policies selected by the caller for one planner invocation.", + fields: &[ + FieldSpec::new("auto_load_sources", FieldDefault::False), + FieldSpec::new("selection_diagnostics", FieldDefault::False), + FieldSpec::new("plan_sql_tests", FieldDefault::True), + FieldSpec::new("record_migration_fingerprints", FieldDefault::True), + ], +}; +const R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT: RecordSpec = RecordSpec { + name: "DirectSourceFreshnessPlanningResult", + doc: "Direct planning-time source freshness observations and comparisons.", + fields: &[ + FieldSpec::new("observed_records", FieldDefault::EmptyTuple), + FieldSpec::new("previous_records", FieldDefault::EmptyTuple), + FieldSpec::new("changed_identities", FieldDefault::EmptyFrozenset), + FieldSpec::new("unchanged_identities", FieldDefault::EmptyFrozenset), + FieldSpec::new("unknown_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("unknown_sources", FieldDefault::EmptyDict), + FieldSpec::new("age_statuses", FieldDefault::EmptyDict), + FieldSpec::new("propagation", FieldDefault::None), + ], +}; +const R_FINGERPRINT: RecordSpec = RecordSpec { + name: "Fingerprint", + doc: "One applied fingerprint record from a successful materialization.", + fields: &[ + FieldSpec::required("node_type"), + FieldSpec::required("node_name"), + FieldSpec::required("target_database"), + FieldSpec::required("target_schema"), + FieldSpec::required("target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("definition_hash"), + FieldSpec::required("schema_fingerprint"), + FieldSpec::required("definition"), + FieldSpec::required("ts"), + FieldSpec::new("metadata_json", FieldDefault::Text("{}")), + FieldSpec::new("version_hash", FieldDefault::Text("")), + ], +}; +const R_FINGERPRINT_SET: RecordSpec = RecordSpec { + name: "FingerprintSet", + doc: "Latest fingerprints per node name for one target schema.", + fields: &[ + FieldSpec::required("schema"), + FieldSpec::required("fingerprints"), + FieldSpec::new("fingerprints_by_identity", FieldDefault::None), + ], +}; +const R_SOURCE_FRESHNESS_RECORD: RecordSpec = RecordSpec { + name: "SourceFreshnessRecord", + doc: "One append-only direct observation whose observed time is always aware UTC.", + fields: &[ + FieldSpec::required("source_name"), + FieldSpec::required("target_database"), + FieldSpec::required("target_schema"), + FieldSpec::required("target_name"), + FieldSpec::required("run_id"), + FieldSpec::required("strategy"), + FieldSpec::required("value_kind"), + FieldSpec::required("data_version"), + FieldSpec::required("data_version_hash"), + FieldSpec::required("observed_at"), + ], +}; +const R_TIMESTAMP_VALUE: RecordSpec = RecordSpec { + name: "TimestampValue", + doc: "Timestamp cursor value; aware datetimes are normalized to UTC at ingestion.", + fields: &[ + FieldSpec::required("value"), + FieldSpec { + name: "source_text", + default: FieldDefault::None, + compared: false, + shown: false, + }, + ], +}; +const R_DATE_VALUE: RecordSpec = RecordSpec { + name: "DateValue", + doc: "Date cursor value which must remain date-formatted when rendered.", + fields: &[FieldSpec::required("value")], +}; +const R_INTEGER_VALUE: RecordSpec = RecordSpec { + name: "IntegerValue", + doc: "Integral cursor value.", + fields: &[FieldSpec::required("value")], +}; +const R_ALIGNED_INTERVAL: RecordSpec = RecordSpec { + name: "AlignedInterval", + doc: "Validated half-open cursor interval.", + fields: &[ + FieldSpec::required("start"), + FieldSpec::required("end"), + FieldSpec::required("grain"), + ], +}; +const R_DURATION: RecordSpec = RecordSpec { + name: "Duration", + doc: "A calendar-aware duration (years/months plus fixed time units).", + fields: &[ + FieldSpec::new("years", FieldDefault::Int(0)), + FieldSpec::new("months", FieldDefault::Int(0)), + FieldSpec::new("days", FieldDefault::Int(0)), + FieldSpec::new("hours", FieldDefault::Int(0)), + FieldSpec::new("minutes", FieldDefault::Int(0)), + FieldSpec::new("seconds", FieldDefault::Int(0)), + ], +}; +const R_MIGRATION_RELATION: RecordSpec = RecordSpec { + name: "MigrationRelation", + doc: "One physical relation named by a migration event.", + fields: &[ + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("name"), + ], +}; +const R_MIGRATION_EVENT: RecordSpec = RecordSpec { + name: "MigrationEvent", + doc: "One immutable fact that a origin relation was cloned over a destination.", + fields: &[ + FieldSpec::required("event_id"), + FieldSpec::required("target_name"), + FieldSpec::required("origin_model"), + FieldSpec::required("origin"), + FieldSpec::required("destination_model"), + FieldSpec::required("destination"), + FieldSpec::required("origin_version_hash"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("run_id"), + FieldSpec::required("created_at"), + ], +}; +const R_COLUMN_MIGRATION_EVENT: RecordSpec = RecordSpec { + name: "ColumnMigrationEvent", + doc: "One immutable fact that a column of a model relation was renamed in place.", + fields: &[ + FieldSpec::required("event_id"), + FieldSpec::required("target_name"), + FieldSpec::required("model_name"), + FieldSpec::required("relation"), + FieldSpec::required("origin_column"), + FieldSpec::required("destination_column"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("run_id"), + FieldSpec::required("created_at"), + ], +}; +const R_OLD_NAME_VIEW_EVENT: RecordSpec = RecordSpec { + name: "OldNameViewEvent", + doc: "One immutable fact about the old name of one recorded move.", + fields: &[ + FieldSpec::required("event_id"), + FieldSpec::required("target_name"), + FieldSpec::required("event_type"), + FieldSpec::required("migration_event_id"), + FieldSpec::required("destination_model"), + FieldSpec::required("old"), + FieldSpec::required("new"), + FieldSpec::required("run_id"), + FieldSpec::required("created_at"), + FieldSpec::new("view_retention", FieldDefault::None), + FieldSpec::new("archive_name", FieldDefault::None), + FieldSpec::new("column_aliases", FieldDefault::EmptyTuple), + FieldSpec::new("expires_at", FieldDefault::None), + FieldSpec::new("drop_reason", FieldDefault::None), + FieldSpec::new("grants_copied", FieldDefault::None), + FieldSpec::new("view_sql", FieldDefault::None), + ], +}; +const R_OLD_NAME_VIEW_HISTORY: RecordSpec = RecordSpec { + name: "OldNameViewHistory", + doc: "The recorded move and every old-name fact that references it.", + fields: &[ + FieldSpec::required("move"), + FieldSpec::required("required"), + FieldSpec::new("archived", FieldDefault::None), + FieldSpec::new("created", FieldDefault::None), + FieldSpec::new("dropped", FieldDefault::None), + ], +}; +const R_CURSOR_OVERRIDES: RecordSpec = RecordSpec { + name: "CursorOverrides", + doc: "Cursor override values from CLI flags; untyped `start`/`end` await planner typing.", + fields: &[ + FieldSpec::new("start_ts", FieldDefault::None), + FieldSpec::new("end_ts", FieldDefault::None), + FieldSpec::new("start_int", FieldDefault::None), + FieldSpec::new("end_int", FieldDefault::None), + FieldSpec::new("start", FieldDefault::None), + FieldSpec::new("end", FieldDefault::None), + ], +}; +const R_EMPTY_CURSOR_INPUT_DECISION: RecordSpec = RecordSpec { + name: "EmptyCursorInputDecision", + doc: "Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.", + fields: &[ + FieldSpec::new("input_names", FieldDefault::EmptyTuple), + FieldSpec::new("waiting_on_empty_inputs", FieldDefault::False), + ], +}; +const R_MODEL_CURSOR_SNAPSHOT: RecordSpec = RecordSpec { + name: "ModelCursorSnapshot", + doc: "Cursor MIN/MAX values gathered from warehouse for one incremental model.", + fields: &[ + FieldSpec::required("target_max"), + FieldSpec::required("upstream_mins"), + FieldSpec::required("upstream_maxes"), + FieldSpec { + name: "physical_target_max", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec::new("target_eligible_max", FieldDefault::None), + FieldSpec { + name: "target_relation", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec { + name: "destination_cursor_column", + default: FieldDefault::None, + compared: false, + shown: false, + }, + FieldSpec { + name: "input_evidence", + default: FieldDefault::EmptyTuple, + compared: false, + shown: false, + }, + FieldSpec { + name: "expected_watermark_count", + default: FieldDefault::Int(0), + compared: false, + shown: false, + }, + FieldSpec::new("unavailable_watermark_tags", FieldDefault::EmptyTuple), + FieldSpec::new( + "cursor_watermark_mode", + FieldDefault::Member("sqlbuild._native", "CursorWatermarkMode", "ALL"), + ), + FieldSpec::new("upstream_terminal_starts", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_terminal_ends", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_end_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_availability_ends", FieldDefault::EmptyTuple), + FieldSpec { + name: "empty_input_names", + default: FieldDefault::EmptyTuple, + compared: false, + shown: false, + }, + FieldSpec { + name: "unreadable_input_names", + default: FieldDefault::EmptyTuple, + compared: false, + shown: false, + }, + ], +}; +const R_CURSOR_BOUNDS: RecordSpec = RecordSpec { + name: "CursorBounds", + doc: "Effective cursor start and end values for one incremental model.", + fields: &[ + FieldSpec::required("start"), + FieldSpec::required("end"), + FieldSpec::new("future_safety", FieldDefault::None), + FieldSpec::new("maximum_start_safety", FieldDefault::None), + ], +}; +const R_CURSOR_INPUT_EVIDENCE: RecordSpec = RecordSpec { + name: "CursorInputEvidence", + doc: "Observed bounds for one physical cursor input.", + fields: &[ + FieldSpec::required("relation"), + FieldSpec::required("cursor_column"), + FieldSpec::required("minimum"), + FieldSpec::required("maximum"), + ], +}; +const R_FUTURE_CURSOR_SAFETY_EVIDENCE: RecordSpec = RecordSpec { + name: "FutureCursorSafetyEvidence", + doc: "Structured evidence for one applied future-cursor cap.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::required("max_distance"), + FieldSpec::required("invocation_time"), + FieldSpec::required("discovered_start"), + FieldSpec::required("discovered_end"), + FieldSpec::required("applied_start"), + FieldSpec::required("applied_end"), + FieldSpec::required("maximum_allowed_start"), + FieldSpec::required("maximum_allowed_end"), + FieldSpec::required("future_start_detected"), + FieldSpec::required("future_end_detected"), + FieldSpec::required("determining_relation"), + FieldSpec::required("determining_cursor_column"), + FieldSpec::new("inputs", FieldDefault::EmptyTuple), + ], +}; +const R_MAXIMUM_START_SAFETY_EVIDENCE: RecordSpec = RecordSpec { + name: "MaximumStartSafetyEvidence", + doc: "Structured evidence for an automatic-start eligibility decision.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::required("max_ahead"), + FieldSpec::required("invocation_time"), + FieldSpec::required("physical_target_max"), + FieldSpec::required("highest_eligible_target_max"), + FieldSpec::required("effective_start"), + FieldSpec::required("maximum_allowed_start"), + FieldSpec::required("target_relation"), + FieldSpec::required("cursor_column"), + ], +}; +const R_CURSOR_INPUT_RELATION: RecordSpec = RecordSpec { + name: "CursorInputRelation", + doc: "One cursor-bearing input relation for runtime range discovery.", + fields: &[ + FieldSpec::required("relation"), + FieldSpec::required("cursor_column"), + FieldSpec::new("cursor_grain", FieldDefault::None), + FieldSpec::new("is_model_backed", FieldDefault::False), + FieldSpec::new("is_runtime_produced", FieldDefault::False), + FieldSpec::new("terminal_cursor_start", FieldDefault::None), + FieldSpec::new("terminal_cursor_end", FieldDefault::None), + FieldSpec::new("input_name", FieldDefault::None), + FieldSpec::new("input_kind", FieldDefault::None), + ], +}; +const R_MAXIMUM_START_POLICY_INPUTS: RecordSpec = RecordSpec { + name: "MaximumStartPolicyInputs", + doc: "Effective policy and materialization safety inputs for automatic starts.", + fields: &[ + FieldSpec::new("config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("incremental_strategy", FieldDefault::None), + FieldSpec::new("incremental_mode", FieldDefault::None), + ], +}; +const R_SCHEMA_FINDING: RecordSpec = RecordSpec { + name: "SchemaFinding", + doc: "One detected schema difference between expected and warehouse columns.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("column_name"), + FieldSpec::required("source"), + FieldSpec::new("expected_type", FieldDefault::None), + FieldSpec::new("actual_type", FieldDefault::None), + ], +}; +const R_RETENTION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "RetentionPlanEntry", + doc: "Warehouse retention work independent of model identity actions.", + fields: &[ + FieldSpec::required("request"), + FieldSpec::required("model_names"), + FieldSpec::required("actual_days"), + FieldSpec::required("effective_days"), + FieldSpec::required("source"), + FieldSpec::required("direction"), + FieldSpec::required("phase"), + FieldSpec::new("statements", FieldDefault::EmptyTuple), + FieldSpec::new("irreversible_warning", FieldDefault::None), + FieldSpec::new("decrease_policy", FieldDefault::Text("deny")), + ], +}; +const R_TABLE_TYPE_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "TableTypePlanEntry", + doc: "Snowflake table-type work independent of model identity actions.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("destination"), + FieldSpec::required("copy_name"), + FieldSpec::required("desired_type"), + FieldSpec::required("actual_type"), + FieldSpec::required("source"), + FieldSpec::required("downgrade"), + FieldSpec::required("downgrade_policy"), + FieldSpec::new("irreversible_warning", FieldDefault::None), + ], +}; +const R_MODEL_MIGRATION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "ModelMigrationPlanEntry", + doc: "One declared or discovered model migration and its per-run decision.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("compatibility"), + FieldSpec::required("origin_model"), + FieldSpec::required("origin"), + FieldSpec::required("destination"), + FieldSpec::required("target_name"), + FieldSpec::new("origin_version_hash", FieldDefault::Text("")), + FieldSpec::new("origin_is_transient", FieldDefault::False), + FieldSpec::new("stage_is_transient", FieldDefault::None), + FieldSpec::new("transfer", FieldDefault::None), + FieldSpec::new("transfer_fallback", FieldDefault::None), + FieldSpec::new("promotion", FieldDefault::None), + FieldSpec::new("completed_at", FieldDefault::None), + FieldSpec::new("compatibility_findings", FieldDefault::EmptyTuple), + FieldSpec::new("message", FieldDefault::None), + FieldSpec::new("origin_tracked", FieldDefault::False), + FieldSpec::new( + "missing_origin_policy", + FieldDefault::Member("sqlbuild._native", "MissingMigrationOriginPolicy", "ALLOW"), + ), + ], +}; +const R_OLD_NAME_VIEW_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "OldNameViewPlanEntry", + doc: "What one build does at the old name of one migrated model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("origin"), + FieldSpec::required("destination"), + FieldSpec::required("action"), + FieldSpec::required("target_name"), + FieldSpec::new("retention", FieldDefault::None), + FieldSpec::new("expires_at", FieldDefault::None), + FieldSpec::new("column_aliases", FieldDefault::EmptyTuple), + FieldSpec::new("stores_history", FieldDefault::False), + FieldSpec::new("migration_event_id", FieldDefault::None), + FieldSpec::new("records_requirement", FieldDefault::False), + FieldSpec::new("reason", FieldDefault::None), + FieldSpec::new("grants_copied", FieldDefault::None), + FieldSpec::new("grants_supported", FieldDefault::False), + FieldSpec::new("archived", FieldDefault::False), + ], +}; +const R_OLD_NAME_VIEW: RecordSpec = RecordSpec { + name: "OldNameView", + doc: "One compatibility view recorded at an old name and not recorded as dropped.", + fields: &[ + FieldSpec::required("destination_model"), + FieldSpec::required("old"), + FieldSpec::required("new"), + FieldSpec::required("expires_at"), + FieldSpec::required("column_aliases"), + FieldSpec::required("migration_event_id"), + FieldSpec::required("target_name"), + FieldSpec::new("name_reused_by", FieldDefault::None), + ], +}; +const R_COLUMN_MIGRATION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "ColumnMigrationPlanEntry", + doc: "One declared or detected in-place column rename and its per-run decision.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("destination"), + FieldSpec::required("origin_column"), + FieldSpec::required("destination_column"), + FieldSpec::required("discovery"), + FieldSpec::required("decision"), + FieldSpec::required("target_name"), + FieldSpec::new("completed_at", FieldDefault::None), + FieldSpec::new("message", FieldDefault::None), + FieldSpec::new( + "missing_origin_policy", + FieldDefault::Member("sqlbuild._native", "MissingMigrationOriginPolicy", "ALLOW"), + ), + ], +}; +const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "ModelPlanEntry", + doc: "Per-model execution plan entry with action, reason, and resolved artifacts.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("materialization_type"), + FieldSpec::required("action"), + FieldSpec::required("reason"), + FieldSpec::required("destination"), + FieldSpec::required("fingerprint_query_sql"), + FieldSpec::required("resolved_sql"), + FieldSpec::required("logical_ddl"), + FieldSpec::new("migration_fingerprint", FieldDefault::None), + FieldSpec::new("incremental_strategy", FieldDefault::None), + FieldSpec::new("incremental_mode", FieldDefault::None), + FieldSpec::new("microbatch_strategy", FieldDefault::None), + FieldSpec::new("cursor_watermark_mode", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("cursor_type", FieldDefault::None), + FieldSpec::new("cursor_grain", FieldDefault::None), + FieldSpec::new("cursor_start", FieldDefault::None), + FieldSpec::new("cursor_end", FieldDefault::None), + FieldSpec::new("lookback", FieldDefault::None), + FieldSpec::new("cursor_bounds", FieldDefault::None), + FieldSpec::new("empty_cursor_inputs", FieldDefault::EmptyTuple), + FieldSpec::new("waiting_on_empty_inputs", FieldDefault::False), + FieldSpec::new("cursor_input_relations", FieldDefault::EmptyTuple), + FieldSpec::new("batch_size", FieldDefault::None), + FieldSpec::new("batch_concurrency", FieldDefault::Int(1)), + FieldSpec::new("unaccounted_partition_policy", FieldDefault::None), + FieldSpec::new("microbatch_range", FieldDefault::None), + FieldSpec::new("microbatch_limit", FieldDefault::None), + FieldSpec::new("declared_microbatch_limit_action", FieldDefault::None), + FieldSpec::new("microbatch_safety_limit", FieldDefault::None), + FieldSpec::new("microbatch_safety_limit_action", FieldDefault::None), + FieldSpec::new("microbatch_limit_count", FieldDefault::None), + FieldSpec::new("microbatch_limit_action", FieldDefault::None), + FieldSpec::new("microbatch_limit_warning", FieldDefault::None), + FieldSpec::new("start_cursor_override", FieldDefault::None), + FieldSpec::new("end_cursor_override", FieldDefault::None), + FieldSpec::new("future_cursor_config", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("merge_exclude_columns", FieldDefault::EmptyTuple), + FieldSpec::new("snapshot_strategy", FieldDefault::None), + FieldSpec::new("updated_at_column", FieldDefault::None), + FieldSpec::new("check_columns", FieldDefault::EmptyTuple), + FieldSpec::new("observed_at_column", FieldDefault::None), + FieldSpec::new("historical_input", FieldDefault::None), + FieldSpec::new("valid_from_column", FieldDefault::None), + FieldSpec::new("valid_to_column", FieldDefault::None), + FieldSpec::new("initial_valid_from", FieldDefault::None), + FieldSpec::new("invalidate_hard_deletes", FieldDefault::False), + FieldSpec::new("snapshot_full_refresh", FieldDefault::None), + FieldSpec::new("snapshot_schema_change", FieldDefault::None), + FieldSpec::new("table_type", FieldDefault::Text("transient")), + FieldSpec::new("on_schema_change", FieldDefault::None), + FieldSpec::new("type_enforcement", FieldDefault::False), + FieldSpec::new("declared_columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract_enforced", FieldDefault::False), + FieldSpec::new("contract_columns", FieldDefault::EmptyTuple), + FieldSpec::new("contract_dynamic_columns", FieldDefault::EmptyTuple), + FieldSpec::new("pre_hooks", FieldDefault::None), + FieldSpec::new("post_hooks", FieldDefault::None), + FieldSpec::new("previous_query_sql", FieldDefault::None), + FieldSpec::new("fingerprint_metadata_json", FieldDefault::None), + FieldSpec::new("previous_metadata_json", FieldDefault::None), + FieldSpec::new("fingerprint_version_hash", FieldDefault::None), + FieldSpec::new("previous_version_hash", FieldDefault::None), + FieldSpec::new("query_changed", FieldDefault::False), + FieldSpec::new("config_changed", FieldDefault::False), + FieldSpec::new("schema_actions", FieldDefault::EmptyTuple), + FieldSpec::new("schema_findings", FieldDefault::EmptyTuple), + FieldSpec::new("backfill", FieldDefault::Factory(forward_only_backfill)), + FieldSpec::new("changed_functions", FieldDefault::EmptyTuple), + FieldSpec::new("custom_materialization_name", FieldDefault::None), + FieldSpec::new("custom_config", FieldDefault::EmptyDict), + FieldSpec::new("custom_placeholders", FieldDefault::EmptyDict), + FieldSpec::new("column_rename_hints", FieldDefault::EmptyTuple), + ], +}; +const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "CloneSourcePlanEntry", + doc: "One managed source relation selected for target cloning.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("destination"), + ], +}; +const R_SEED_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "SeedPlanEntry", + doc: "Per-seed execution plan entry.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("destination"), + FieldSpec::required("file_path"), + FieldSpec::required("columns"), + FieldSpec::required("csv_settings"), + FieldSpec::new("fingerprint_definition", FieldDefault::Text("")), + FieldSpec::new("fingerprint_version_hash", FieldDefault::Text("")), + FieldSpec::new("fingerprint_metadata_json", FieldDefault::Text("{}")), + FieldSpec::new( + "action", + FieldDefault::Member("sqlbuild._native", "PlanAction", "LOAD_SEED"), + ), + FieldSpec::new( + "reason", + FieldDefault::Member("sqlbuild._native", "PlanReason", "FIRST_RUN"), + ), + ], +}; +const R_SOURCE_LOAD_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "SourceLoadPlanEntry", + doc: "Per-managed-source loader execution plan entry.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("loader"), + FieldSpec::required("destination"), + FieldSpec::new( + "resource_kind", + FieldDefault::Member( + "sqlbuild.runtime.contracts.types", + "ExecutionResourceKind", + "SOURCE", + ), + ), + FieldSpec::new("write_strategy", FieldDefault::None), + FieldSpec::new("cursor_column", FieldDefault::None), + FieldSpec::new("unique_key", FieldDefault::EmptyTuple), + FieldSpec::new("is_reload", FieldDefault::False), + FieldSpec::new("integration_kind", FieldDefault::None), + ], +}; +const R_FUNCTION_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "FunctionPlanEntry", + doc: "Per-SQL-function execution plan entry.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("relative_path"), + FieldSpec::required("destination"), + FieldSpec::required("arguments"), + FieldSpec::required("returns"), + FieldSpec::required("body_sql"), + FieldSpec::required("fingerprint_query_sql"), + FieldSpec::required("fingerprint_destination"), + FieldSpec::new("return_columns", FieldDefault::EmptyTuple), + FieldSpec::new( + "language", + FieldDefault::Member("sqlbuild.compiler.compile.types", "FunctionLanguage", "SQL"), + ), + FieldSpec::new("source_file_path", FieldDefault::None), + FieldSpec::new("runtime_version", FieldDefault::None), + FieldSpec::new("entry_point", FieldDefault::None), + FieldSpec::new("packages", FieldDefault::EmptyTuple), + FieldSpec::new("previous_query_sql", FieldDefault::None), + FieldSpec::new( + "reason", + FieldDefault::Member("sqlbuild._native", "PlanReason", "NO_CHANGE"), + ), + ], +}; +const R_AUDIT_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "AuditPlanEntry", + doc: "Per-audit execution plan entry with resolved SQL and scheduling metadata.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("definition_name"), + FieldSpec::required("resolved_sql"), + FieldSpec::required("unresolved_sql"), + FieldSpec::required("attachment_kind"), + FieldSpec::required("severity"), + FieldSpec::required("requested_run_scope"), + FieldSpec::required("effective_run_scope"), + FieldSpec::new( + "evaluation_mode", + FieldDefault::Member( + "sqlbuild.compiler.auditing.types", + "AuditEvaluationMode", + "VIOLATIONS", + ), + ), + FieldSpec::new("value_column", FieldDefault::None), + FieldSpec::new("sample_count_column", FieldDefault::None), + FieldSpec::new("sample_unit", FieldDefault::None), + FieldSpec::new("thresholds", FieldDefault::None), + FieldSpec::new("minimum_samples", FieldDefault::None), + FieldSpec::new("evidence_resolved_sql", FieldDefault::None), + FieldSpec::new("evidence_unresolved_sql", FieldDefault::None), + FieldSpec::new("evidence_limit", FieldDefault::None), + FieldSpec::new("scope_deps", FieldDefault::EmptyTuple), + FieldSpec::new("attached_target_kind", FieldDefault::None), + FieldSpec::new("attached_target_name", FieldDefault::None), + FieldSpec::new("attached_column_name", FieldDefault::None), + FieldSpec::new("always_run", FieldDefault::False), + FieldSpec::new("description", FieldDefault::None), + FieldSpec::new("reads_outside_target_lineage", FieldDefault::False), + ], +}; +const R_CHAIN_STEP: RecordSpec = RecordSpec { + name: "ChainStep", + doc: "One step in a chained test execution sequence.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("resolved_sql"), + FieldSpec::new("expected_cte_sql", FieldDefault::None), + FieldSpec::new("lifted_ctes", FieldDefault::EmptyTuple), + FieldSpec::new("comparison_body_sql", FieldDefault::None), + FieldSpec::new("expected_columns", FieldDefault::None), + FieldSpec::new("expected_lifted_ctes", FieldDefault::EmptyTuple), + ], +}; +const R_SQL_TEST_PLAN_ENTRY: RecordSpec = RecordSpec { + name: "SqlTestPlanEntry", + doc: "Per-test execution plan entry with chained resolution.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::new("source_path", FieldDefault::None), + FieldSpec::new("block_index", FieldDefault::None), + FieldSpec::new("parent_name", FieldDefault::None), + FieldSpec::new("case_name", FieldDefault::None), + FieldSpec::new("case_index", FieldDefault::None), + FieldSpec::new("case_fingerprint", FieldDefault::None), + FieldSpec::new("parameter_schema", FieldDefault::EmptyTuple), + FieldSpec::new("parameter_values", FieldDefault::EmptyTuple), + FieldSpec::new("mock_ref_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_source_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_seed_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_dbt_ref_names", FieldDefault::EmptyTuple), + FieldSpec::new("mock_table_function_names", FieldDefault::EmptyTuple), + FieldSpec::new("chain", FieldDefault::EmptyTuple), + FieldSpec::new("assertions", FieldDefault::EmptyTuple), + FieldSpec::new("scope_deps", FieldDefault::EmptyTuple), + FieldSpec::new("function_deps", FieldDefault::EmptyTuple), + FieldSpec::new("sql_analysis_enabled", FieldDefault::True), + ], +}; +const R_SCENARIO_RELATION_MAP: RecordSpec = RecordSpec { + name: "ScenarioRelationMap", + doc: "Resolved scenario hash prefix and scenario-owned relation names.", + fields: &[ + FieldSpec::required("scenario_name"), + FieldSpec::required("hash_prefix"), + FieldSpec::new("artifacts", FieldDefault::EmptyTuple), + ], +}; +const R_SCENARIO_RELATION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioRelationPlan", + doc: "Resolved scenario relation locations for model/source/seed ref resolution.", + fields: &[ + FieldSpec::required("scenario_name"), + FieldSpec::required("relation_map"), + FieldSpec::new("model_locations", FieldDefault::EmptyDict), + FieldSpec::new("seed_locations", FieldDefault::EmptyDict), + FieldSpec::new("project_source_map", FieldDefault::EmptyDict), + FieldSpec::new("source_map", FieldDefault::EmptyDict), + FieldSpec::new("source_fixture_locations", FieldDefault::EmptyDict), + FieldSpec::new("ref_fixture_locations", FieldDefault::EmptyDict), + FieldSpec::new("seed_fixture_locations", FieldDefault::EmptyDict), + FieldSpec::new("dbt_ref_fixture_locations", FieldDefault::EmptyDict), + ], +}; +const R_SCENARIO_FIXTURE_PLAN: RecordSpec = RecordSpec { + name: "ScenarioFixturePlan", + doc: "Self-contained fixture SQL planned for scenario materialization.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("logical_name"), + FieldSpec::required("destination"), + FieldSpec::required("sql"), + ], +}; +const R_SCENARIO_EXPECTED_EXPECTATION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioExpectedExpectationPlan", + doc: "Expected-output comparison inputs for one scenario target model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("actual_destination"), + FieldSpec::required("expected_sql"), + ], +}; +const R_SCENARIO_ASSERTION_EXPECTATION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioAssertionExpectationPlan", + doc: "Zero-row assertion SQL for one scenario assertion CTE.", + fields: &[FieldSpec::required("name"), FieldSpec::required("sql")], +}; +const R_SCENARIO_EXECUTION_PLAN: RecordSpec = RecordSpec { + name: "ScenarioExecutionPlan", + doc: "Dry-run execution plan for a SQL scenario graph slice.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::required("graph_plan"), + FieldSpec::required("relation_plan"), + FieldSpec::new("fixture_plans", FieldDefault::EmptyTuple), + FieldSpec::new("seed_entries", FieldDefault::EmptyTuple), + FieldSpec::new("function_entries", FieldDefault::EmptyTuple), + FieldSpec::new("model_entries", FieldDefault::EmptyTuple), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("expected_expectations", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_expectations", FieldDefault::EmptyTuple), + ], +}; +const R_PLAN_OUTPUT: RecordSpec = RecordSpec { + name: "PlanOutput", + doc: "Complete execution plan produced by the planner.", + fields: &[ + FieldSpec::new("execution_order", FieldDefault::EmptyTuple), + FieldSpec::new("model_entries", FieldDefault::EmptyTuple), + FieldSpec::new("seed_entries", FieldDefault::EmptyTuple), + FieldSpec::new("source_load_entries", FieldDefault::EmptyTuple), + FieldSpec::new("function_entries", FieldDefault::EmptyTuple), + FieldSpec::new("audit_entries", FieldDefault::EmptyTuple), + FieldSpec::new("test_entries", FieldDefault::EmptyTuple), + FieldSpec::new("available_test_case_names", FieldDefault::EmptyTuple), + FieldSpec::new("selected_keys", FieldDefault::EmptyFrozenset), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("retention_entries", FieldDefault::EmptyTuple), + FieldSpec::new("table_type_entries", FieldDefault::EmptyTuple), + FieldSpec::new("migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("column_migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_view_entries", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("upstream_deps", FieldDefault::EmptyDict), + FieldSpec::new("downstream_deps", FieldDefault::EmptyDict), + FieldSpec::new("model_locations", FieldDefault::EmptyDict), + FieldSpec::new("seed_locations", FieldDefault::EmptyDict), + FieldSpec::new("function_locations", FieldDefault::EmptyDict), + FieldSpec::new("source_map", FieldDefault::EmptyDict), + FieldSpec::new("source_read_map", FieldDefault::EmptyDict), + FieldSpec::new("python_source_read_map", FieldDefault::EmptyDict), + FieldSpec::new("hook_functions", FieldDefault::EmptyTuple), + FieldSpec::new("enforce_explicit_references", FieldDefault::True), + FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec::new("source_freshness", FieldDefault::None), + FieldSpec::new("python_identity_fingerprints", FieldDefault::EmptyDict), + FieldSpec::new("metadata", FieldDefault::EmptyDict), + FieldSpec::new("cursor_overrides", FieldDefault::None), + ], +}; + +/// The planner, migration, cursor and fingerprint value records. +pub(crate) const PLANNER_RECORDS: &[RecordSpec] = &[ + R_PARSED_SELECTOR, + R_PATH_SELECTOR, + R_SQL_TEST_SELECTION, + R_PLANNER_SCOPE, + R_PLANNER_RELATIONS_CONTEXT, + R_SQL_TEST_ASSERTION_STEP, + R_PLANNER_SELECTION, + R_PLANNER_OVERRIDES, + R_DEFERRAL_INPUTS, + R_PLANNER_POLICIES, + R_DIRECT_SOURCE_FRESHNESS_PLANNING_RESULT, + R_FINGERPRINT, + R_FINGERPRINT_SET, + R_SOURCE_FRESHNESS_RECORD, + R_TIMESTAMP_VALUE, + R_DATE_VALUE, + R_INTEGER_VALUE, + R_ALIGNED_INTERVAL, + R_DURATION, + R_MIGRATION_RELATION, + R_MIGRATION_EVENT, + R_COLUMN_MIGRATION_EVENT, + R_OLD_NAME_VIEW_EVENT, + R_OLD_NAME_VIEW_HISTORY, + R_CURSOR_OVERRIDES, + R_EMPTY_CURSOR_INPUT_DECISION, + R_MODEL_CURSOR_SNAPSHOT, + R_CURSOR_BOUNDS, + R_CURSOR_INPUT_EVIDENCE, + R_FUTURE_CURSOR_SAFETY_EVIDENCE, + R_MAXIMUM_START_SAFETY_EVIDENCE, + R_CURSOR_INPUT_RELATION, + R_MAXIMUM_START_POLICY_INPUTS, + R_SCHEMA_FINDING, + R_RETENTION_PLAN_ENTRY, + R_TABLE_TYPE_PLAN_ENTRY, + R_MODEL_MIGRATION_PLAN_ENTRY, + R_OLD_NAME_VIEW_PLAN_ENTRY, + R_OLD_NAME_VIEW, + R_COLUMN_MIGRATION_PLAN_ENTRY, + R_MODEL_PLAN_ENTRY, + R_CLONE_SOURCE_PLAN_ENTRY, + R_SEED_PLAN_ENTRY, + R_SOURCE_LOAD_PLAN_ENTRY, + R_FUNCTION_PLAN_ENTRY, + R_AUDIT_PLAN_ENTRY, + R_CHAIN_STEP, + R_SQL_TEST_PLAN_ENTRY, + R_SCENARIO_RELATION_MAP, + R_SCENARIO_RELATION_PLAN, + R_SCENARIO_FIXTURE_PLAN, + R_SCENARIO_EXPECTED_EXPECTATION_PLAN, + R_SCENARIO_ASSERTION_EXPECTATION_PLAN, + R_SCENARIO_EXECUTION_PLAN, + R_PLAN_OUTPUT, +]; + +const R_NATIVE_SQL_TEST_ARTIFACT: RecordSpec = RecordSpec { + name: "NativeSqlTestArtifact", + doc: "One native-planned SQL-test artifact and its distinct planning error messages.", + fields: &[ + FieldSpec::required("sql"), + FieldSpec::required("model_names"), + FieldSpec::new("error_messages", FieldDefault::EmptyTuple), + ], +}; +const R_RELATION_FIXTURE_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "RelationFixtureDiagnostic", + doc: "One statically provable invalid fixture condition.", + fields: &[FieldSpec::required("key"), FieldSpec::required("message")], +}; +const R_RELATION_FIXTURE_COMPLETION: RecordSpec = RecordSpec { + name: "RelationFixtureCompletion", + doc: "Completed fixture SQL plus metadata reused by SQL-test validation.", + fields: &[ + FieldSpec::required("fixture_sql_by_key"), + FieldSpec::required("inferred_by_fixture"), + FieldSpec::required("expected_types"), + FieldSpec::required("diagnostics"), + ], +}; +const R_FIXTURE_COLUMN_METADATA: RecordSpec = RecordSpec { + name: "FixtureColumnMetadata", + doc: "Known type and nullability for one relation fixture column.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("type"), + FieldSpec::required("nullable"), + ], +}; +const R_FIXTURE_RELATION_METADATA: RecordSpec = RecordSpec { + name: "FixtureRelationMetadata", + doc: "Known columns and whether they form an authoritative relation shape.", + fields: &[ + FieldSpec::required("columns"), + FieldSpec::required("authoritative_names"), + ], +}; +const R_RELATION_FIXTURE_PLANNING_CONTEXT: RecordSpec = RecordSpec { + name: "RelationFixturePlanningContext", + doc: "Project-wide fixture metadata shared across test and scenario planning.", + fields: &[ + FieldSpec::required("models_by_name"), + FieldSpec::required("relations"), + FieldSpec::required("authoritative_columns"), + FieldSpec::required("expected_types"), + FieldSpec::uncompared_shown( + "fixture_inferences", + FieldDefault::Call("sqlbuild._native", "FixtureColumnInferences"), + ), + ], +}; +const R_SELECTOR_EXPANSION: RecordSpec = RecordSpec { + name: "SelectorExpansion", + doc: "One selector split into graph expansion flags and core text.", + fields: &[ + FieldSpec::required("core"), + FieldSpec::new("upstream", FieldDefault::False), + FieldSpec::new("downstream", FieldDefault::False), + ], +}; +const R_PARSED_SCENARIO_ARTIFACT_NAME: RecordSpec = RecordSpec { + name: "ParsedScenarioArtifactName", + doc: "Parsed physical name for one planner-owned scenario artifact.", + fields: &[ + FieldSpec::required("hash_prefix"), + FieldSpec::required("kind"), + FieldSpec::required("logical_name"), + ], +}; +const R_MISSING_UPSTREAM: RecordSpec = RecordSpec { + name: "MissingUpstream", + doc: "One upstream dependency missing from both scope and warehouse.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("required_by"), + ], +}; +const R_WAREHOUSE_FINGERPRINTS: RecordSpec = RecordSpec { + name: "WarehouseFingerprints", + doc: "Latest direct fingerprints grouped by node type.", + fields: &[ + FieldSpec::new("models", FieldDefault::EmptyDict), + FieldSpec::new("functions", FieldDefault::EmptyDict), + FieldSpec::new("seeds", FieldDefault::EmptyDict), + FieldSpec::new("python_nodes", FieldDefault::EmptyDict), + FieldSpec::uncompared("unfiltered_schemas", FieldDefault::EmptyDict), + FieldSpec::uncompared("unfiltered_database", FieldDefault::None), + ], +}; +const R_CURSOR_SNAPSHOT_SCOPE: RecordSpec = RecordSpec { + name: "CursorSnapshotScope", + doc: "Execution selection used for cursor validation and runtime producer resolution.", + fields: &[ + FieldSpec::required("model_keys"), + FieldSpec::required("runtime_producer_keys"), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("cursor_overrides", FieldDefault::None), + ], +}; +const R_REFERENCE_RENAME: RecordSpec = RecordSpec { + name: "ReferenceRename", + doc: "One model rename that downstream references may have been rewritten for.", + fields: &[ + FieldSpec::required("new_name"), + FieldSpec::required("origin_name"), + FieldSpec::new("recorded_at", FieldDefault::None), + ], +}; +const R_WAREHOUSE_SNAPSHOT: RecordSpec = RecordSpec { + name: "WarehouseSnapshot", + doc: "Frozen point-in-time picture of warehouse state for planning.", + fields: &[ + FieldSpec::new("existing_relations", FieldDefault::EmptyDict), + FieldSpec::new("existing_columns", FieldDefault::EmptyDict), + FieldSpec::new( + "fingerprints", + FieldDefault::Call("sqlbuild._native", "WarehouseFingerprints"), + ), + FieldSpec::new("cursor_snapshots", FieldDefault::EmptyDict), + FieldSpec::new( + "source_freshness_state_schemas", + FieldDefault::EmptyFrozenset, + ), + FieldSpec::new("column_dialect", FieldDefault::None), + FieldSpec::new("renamed_models", FieldDefault::EmptyFrozenset), + FieldSpec::new("reference_renames", FieldDefault::EmptyTuple), + FieldSpec::new("migration_state_schemas", FieldDefault::EmptyFrozenset), + FieldSpec::new("old_name_view_state_schemas", FieldDefault::EmptyFrozenset), + FieldSpec::new("listed_state_schemas", FieldDefault::EmptyFrozenset), + ], +}; +const R_SELECTION_STALENESS_NODE_KEY: RecordSpec = RecordSpec { + name: "SelectionStalenessNodeKey", + doc: "Neutral node identity for shared selection-aware staleness classification.", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("name"), + ], +}; +const R_SELECTION_STALENESS_GRAPH: RecordSpec = RecordSpec { + name: "SelectionStalenessGraph", + doc: "Neutral graph inputs for selected-model stale warning classification.", + fields: &[ + FieldSpec::required("upstream_deps"), + FieldSpec::required("selected_model_names"), + FieldSpec::required("run_model_names"), + FieldSpec::required("run_seed_names"), + FieldSpec::required("run_source_names"), + FieldSpec::required("changed_model_names"), + FieldSpec::required("changed_seed_names"), + FieldSpec::required("changed_source_names"), + ], +}; +const R_SELECTION_STALENESS_WARNING: RecordSpec = RecordSpec { + name: "SelectionStalenessWarning", + doc: "Neutral stale warning classification for one selected model.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("trigger_names"), + ], +}; +const R_BACKFILL_RESULT: RecordSpec = RecordSpec { + name: "BackfillResult", + doc: "Resolved backfill action from a change detection policy.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::new("duration", FieldDefault::None), + ], +}; +const R_CHANGE_DETECTION_RESULT: RecordSpec = RecordSpec { + name: "ChangeDetectionResult", + doc: "Per-model output from change detection and policy resolution.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("change_kind"), + FieldSpec::new("query_changed", FieldDefault::False), + FieldSpec::new("config_changed", FieldDefault::False), + FieldSpec::new("fingerprint_metadata_json", FieldDefault::None), + FieldSpec::new("previous_metadata_json", FieldDefault::None), + FieldSpec::new("fingerprint_version_hash", FieldDefault::None), + FieldSpec::new("previous_version_hash", FieldDefault::None), + FieldSpec::new("schema_findings", FieldDefault::EmptyTuple), + FieldSpec::new("backfill", FieldDefault::Factory(forward_only_backfill)), + FieldSpec::new("recorded_build_relation_missing", FieldDefault::False), + FieldSpec::new("changed_functions", FieldDefault::EmptyTuple), + ], +}; +const R_UNIT_TEST_SELECTOR_SPLIT: RecordSpec = RecordSpec { + name: "UnitTestSelectorSplit", + doc: "Resource selectors left after unit-test selectors moved into a test selection.", + fields: &[ + FieldSpec::required("select"), + FieldSpec::required("exclude"), + FieldSpec::required("sql_test_selection"), + ], +}; +const R_MODEL_PLAN_CONTEXT: RecordSpec = RecordSpec { + name: "ModelPlanContext", + doc: "Relation lookups and source columns for building one model plan entry.", + fields: &[ + FieldSpec::required("model_locations"), + FieldSpec::required("models_by_name"), + FieldSpec::required("seed_locations"), + FieldSpec::required("function_locations"), + FieldSpec::required("source_map"), + FieldSpec::required("source_warehouse_columns"), + FieldSpec::required("star_exclude_keyword"), + FieldSpec::new("functions_by_name", FieldDefault::EmptyDict), + FieldSpec::new( + "runtime_cursor_producer_names", + FieldDefault::EmptyFrozenset, + ), + FieldSpec::new("future_cursor_config", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + ], +}; +const R_CURSOR_OVERRIDE_PAIR: RecordSpec = RecordSpec { + name: "CursorOverridePair", + doc: "Resolved per-model cursor start/end overrides for plan entry construction.", + fields: &[ + FieldSpec::new("start_cursor_override", FieldDefault::None), + FieldSpec::new("end_cursor_override", FieldDefault::None), + ], +}; +const R_PLAN_ENTRY_BUILD_INPUTS: RecordSpec = RecordSpec { + name: "PlanEntryBuildInputs", + doc: "Blocked models and cursor overrides for plan entry building.", + fields: &[ + FieldSpec::new( + "source_freshness_blocked_model_names", + FieldDefault::EmptyFrozenset, + ), + FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("start_cursor_override", FieldDefault::None), + FieldSpec::new("end_cursor_override", FieldDefault::None), + FieldSpec::new("future_cursor_config", FieldDefault::None), + FieldSpec::new("start_cursor_config", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::None), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("max_microbatches_is_override", FieldDefault::False), + FieldSpec::new( + "microbatch_limit_action", + FieldDefault::Member("sqlbuild._native", "MicrobatchLimitAction", "ERROR"), + ), + ], +}; +const R_FUNCTION_CHANGE_RESULT: RecordSpec = RecordSpec { + name: "FunctionChangeResult", + doc: "Per-function output from change detection.", + fields: &[ + FieldSpec::required("fingerprint_sql"), + FieldSpec::new( + "reason", + FieldDefault::Member("sqlbuild._native", "PlanReason", "NO_CHANGE"), + ), + ], +}; +const R_PLANNER_CHANGE_RESULTS: RecordSpec = RecordSpec { + name: "PlannerChangeResults", + doc: "Change detection output for selected planner resources.", + fields: &[ + FieldSpec::required("models"), + FieldSpec::required("functions"), + ], +}; +const R_RESOLVED_MODEL_ACTION: RecordSpec = RecordSpec { + name: "ResolvedModelAction", + doc: "Effective model change and backfill decided from the model's own change.", + fields: &[ + FieldSpec::required("change"), + FieldSpec::required("backfill"), + ], +}; +const R_PLANNER_RESOLVED_ACTIONS: RecordSpec = RecordSpec { + name: "PlannerResolvedActions", + doc: "Per-model planning decisions keyed by model name.", + fields: &[FieldSpec::required("models")], +}; +const R_DIRECT_MODEL_VERSION_IDENTITIES: RecordSpec = RecordSpec { + name: "DirectModelVersionIdentities", + doc: "Current direct model version identity values by model name.", + fields: &[ + FieldSpec::required("function_local_hashes"), + FieldSpec::required("seed_version_hashes"), + FieldSpec::required("seed_metadata_jsons"), + FieldSpec::required("model_metadata_jsons"), + FieldSpec::required("model_local_hashes"), + FieldSpec::required("model_version_hashes"), + ], +}; +const R_PLANNER_MODEL_ENTRY_RESULTS: RecordSpec = RecordSpec { + name: "PlannerModelEntryResults", + doc: "Model plan-entry phase output.", + fields: &[ + FieldSpec::required("entries"), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_SCHEMA_ACTION: RecordSpec = RecordSpec { + name: "SchemaAction", + doc: "One concrete schema change action to apply to a target table.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("column_name"), + FieldSpec::new("column_type", FieldDefault::None), + ], +}; +const R_PLAN_WARNING: RecordSpec = RecordSpec { + name: "PlanWarning", + doc: "One warning or error produced during plan resolution.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("severity"), + FieldSpec::required("message"), + FieldSpec::new("code", FieldDefault::None), + ], +}; +const R_COLUMN_RENAME_HINT: RecordSpec = RecordSpec { + name: "ColumnRenameHint", + doc: "A removed column that an added column may have been renamed from.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("added_column"), + FieldSpec::required("candidate_columns"), + FieldSpec::new("identical", FieldDefault::False), + ], +}; +const R_QUERY_PROJECTION: RecordSpec = RecordSpec { + name: "QueryProjection", + doc: "One named top-level output column of a query and its canonical expression.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("expression"), + FieldSpec::required("references"), + FieldSpec::new("passthrough", FieldDefault::False), + ], +}; +const R_QUERY_SHAPE: RecordSpec = RecordSpec { + name: "QueryShape", + doc: "Canonical top-level projections and the rest of one query, ignoring formatting.", + fields: &[ + FieldSpec::required("projections"), + FieldSpec::required("body"), + FieldSpec::new("alias_clauses", FieldDefault::Text("{}")), + FieldSpec::new("positional", FieldDefault::False), + FieldSpec::new("input_relations", FieldDefault::None), + FieldSpec::new("local_input_columns", FieldDefault::EmptyFrozenset), + FieldSpec::new("clause_references", FieldDefault::EmptyFrozenset), + ], +}; +const R_PLAN_OUTPUT_EXTRAS: RecordSpec = RecordSpec { + name: "PlanOutputExtras", + doc: "Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.", + fields: &[ + FieldSpec::new("seed_version_hashes", FieldDefault::None), + FieldSpec::new("seed_metadata_jsons", FieldDefault::None), + FieldSpec::new("seed_plan_reasons", FieldDefault::None), + FieldSpec::new("planned_sql_tests", FieldDefault::None), + ], +}; +const R_PLANNED_SQL_TESTS: RecordSpec = RecordSpec { + name: "PlannedSqlTests", + doc: "SQL test entries planned for one selection; reusable only for the same selection.", + fields: &[ + FieldSpec::required("selected_keys"), + FieldSpec::new( + "sql_test_selection", + FieldDefault::Call("sqlbuild._native", "SqlTestSelection"), + ), + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_NATIVE_SQL_TEST_PLAN: RecordSpec = RecordSpec { + name: "NativeSqlTestPlan", + doc: "Native chain and assertion steps for one SQL test, with SQL when rendered.", + fields: &[ + FieldSpec::required("chain"), + FieldSpec::required("assertions"), + FieldSpec::required("model_names"), + FieldSpec::required("warnings"), + FieldSpec::required("error_messages"), + FieldSpec::new("sql", FieldDefault::None), + ], +}; +const R_SQL_TEST_PLAN_RESULT: RecordSpec = RecordSpec { + name: "SqlTestPlanResult", + doc: "One SQL test's plan entry and warnings, or the fixture diagnostics preventing it.", + fields: &[ + FieldSpec::required("entry"), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("fixture_diagnostics", FieldDefault::EmptyTuple), + ], +}; +const R_SCENARIO_GRAPH_PLAN: RecordSpec = RecordSpec { + name: "ScenarioGraphPlan", + doc: "Inferred graph slice and fixture boundaries for one SQL scenario.", + fields: &[ + FieldSpec::required("key"), + FieldSpec::required("name"), + FieldSpec::new("target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("assertion_target_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("model_names", FieldDefault::EmptyTuple), + FieldSpec::new("source_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_names", FieldDefault::EmptyTuple), + FieldSpec::new("seed_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_ref_fixture_names", FieldDefault::EmptyTuple), + FieldSpec::new("function_deps", FieldDefault::EmptyTuple), + ], +}; +const R_SCENARIO_ARTIFACT_IDENTITY: RecordSpec = RecordSpec { + name: "ScenarioArtifactIdentity", + doc: "Logical identity for one scenario-owned physical artifact.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("logical_name"), + ], +}; +const R_SCENARIO_ARTIFACT_NAME: RecordSpec = RecordSpec { + name: "ScenarioArtifactName", + doc: "Resolved physical relation name for one scenario artifact.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("physical_name"), + ], +}; +const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { + name: "PlanProviderUsage", + doc: "Provider usage metadata scoped to planned work.", + fields: &[ + FieldSpec::required("provider_name"), + FieldSpec::required("consumer_kind"), + FieldSpec::required("consumer_name"), + FieldSpec::required("parameter_name"), + FieldSpec::new("annotation_class_name", FieldDefault::None), + FieldSpec::new("annotation_module", FieldDefault::None), + ], +}; +const R_PLANNER_RUNTIME: RecordSpec = RecordSpec { + name: "PlannerRuntime", + doc: "Resolved execution environment shared by planner phases.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("adapter"), + FieldSpec::required("connection"), + FieldSpec::new("project_config", FieldDefault::None), + FieldSpec::new("local_config", FieldDefault::None), + FieldSpec::new("on_progress", FieldDefault::None), + FieldSpec::new("invocation_time", FieldDefault::Factory(utc_now)), + ], +}; +const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { + name: "PlannerScopeResolution", + doc: "Resolved planner scopes for one plan build.", + fields: &[ + FieldSpec::required("selected_scope"), + FieldSpec::required("stale_warning_scope"), + FieldSpec::required("inspection_scope"), + ], +}; +const R_PLANNER_WAREHOUSE_STATE: RecordSpec = RecordSpec { + name: "PlannerWarehouseState", + doc: "Warehouse snapshot and inspection relations gathered once per plan.", + fields: &[ + FieldSpec::required("snapshot"), + FieldSpec::required("inspection_relations"), + FieldSpec::new("migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("migration_warnings", FieldDefault::EmptyTuple), + FieldSpec::new("column_migration_entries", FieldDefault::EmptyTuple), + FieldSpec::new("column_rename_hints", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_view_entries", FieldDefault::EmptyTuple), + FieldSpec::new("old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new( + "migration_fingerprints", + FieldDefault::Call("sqlbuild._native", "MigrationFingerprintCache"), + ), + ], +}; +const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { + name: "ModelMigrationRequest", + doc: "One destination model whose data should come from an earlier relation.", + fields: &[ + FieldSpec::required("model"), + FieldSpec::required("discovery"), + FieldSpec::new("raw_origin", FieldDefault::None), + FieldSpec::new("origin_location", FieldDefault::None), + FieldSpec::new("origin_model", FieldDefault::None), + FieldSpec::new("force", FieldDefault::False), + FieldSpec::new("identity_only", FieldDefault::False), + ], +}; +const R_MODEL_MIGRATION_DECLARATION: RecordSpec = RecordSpec { + name: "ModelMigrationDeclaration", + doc: "One project model's explicit migrate_from, resolved to a relation.", + fields: &[ + FieldSpec::required("model_name"), + FieldSpec::required("origin_location"), + FieldSpec::required("origin_model"), + ], +}; +const R_MODEL_MIGRATION_DISCOVERY: RecordSpec = RecordSpec { + name: "ModelMigrationDiscovery", + doc: "Manual and automatically discovered migration requests for one plan.", + fields: &[ + FieldSpec::new("requests", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + FieldSpec::new("destination_fingerprints", FieldDefault::EmptyDict), + ], +}; +const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { + name: "MigrationCompatibilityResult", + doc: "Compatibility status plus human-readable blocking findings.", + fields: &[ + FieldSpec::required("status"), + FieldSpec::new("findings", FieldDefault::EmptyTuple), + ], +}; +const R_MODEL_MIGRATION_PLANNING: RecordSpec = RecordSpec { + name: "ModelMigrationPlanning", + doc: "Migration decisions plus the warehouse snapshot as it will look after moves.", + fields: &[ + FieldSpec::required("snapshot"), + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_OLD_NAME_VIEW_PLANNING: RecordSpec = RecordSpec { + name: "OldNameViewPlanning", + doc: "Old-name steps for this build plus every compatibility view still recorded.", + fields: &[ + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("views", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_COLUMN_MIGRATION_PLANNING: RecordSpec = RecordSpec { + name: "ColumnMigrationPlanning", + doc: "Column rename decisions plus the warehouse snapshot as it will look after renames.", + fields: &[ + FieldSpec::required("snapshot"), + FieldSpec::new("entries", FieldDefault::EmptyTuple), + FieldSpec::new("hints", FieldDefault::EmptyTuple), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { + name: "PlannerIdentityContext", + doc: "Expected version identities for inspection and stale-warning scopes.", + fields: &[ + FieldSpec::required("version_identities"), + FieldSpec::required("stale_warning_identities"), + ], +}; +const R_PLANNER_SCOPE_PRUNING_RESULT: RecordSpec = RecordSpec { + name: "PlannerScopePruningResult", + doc: "Scope pruning outcome with pruned scopes, actions, and staleness state.", + fields: &[ + FieldSpec::required("inspection_scope"), + FieldSpec::required("execution_scope"), + FieldSpec::required("resolved_actions"), + FieldSpec::required("pruned_direct_model_names"), + FieldSpec::required("direct_identity_stale_model_names"), + ], +}; +const R_PLANNER_CHANGE_RECONCILIATION: RecordSpec = RecordSpec { + name: "PlannerChangeReconciliation", + doc: "Write-hash-honest changes merged back into resolved actions.", + fields: &[ + FieldSpec::required("changes"), + FieldSpec::required("resolved_actions"), + ], +}; +const R_PLANNER_ENTRY_RESULTS: RecordSpec = RecordSpec { + name: "PlannerEntryResults", + doc: "Model plan entries for one plan build.", + fields: &[FieldSpec::required("model_entry_results")], +}; + +/// Planner value records, in registration order. +pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ + R_NATIVE_SQL_TEST_ARTIFACT, + R_RELATION_FIXTURE_DIAGNOSTIC, + R_RELATION_FIXTURE_COMPLETION, + R_FIXTURE_COLUMN_METADATA, + R_FIXTURE_RELATION_METADATA, + R_RELATION_FIXTURE_PLANNING_CONTEXT, + R_SELECTOR_EXPANSION, + R_PARSED_SCENARIO_ARTIFACT_NAME, + R_MISSING_UPSTREAM, + R_WAREHOUSE_FINGERPRINTS, + R_CURSOR_SNAPSHOT_SCOPE, + R_REFERENCE_RENAME, + R_WAREHOUSE_SNAPSHOT, + R_SELECTION_STALENESS_NODE_KEY, + R_SELECTION_STALENESS_GRAPH, + R_SELECTION_STALENESS_WARNING, + R_BACKFILL_RESULT, + R_CHANGE_DETECTION_RESULT, + R_UNIT_TEST_SELECTOR_SPLIT, + R_MODEL_PLAN_CONTEXT, + R_CURSOR_OVERRIDE_PAIR, + R_PLAN_ENTRY_BUILD_INPUTS, + R_FUNCTION_CHANGE_RESULT, + R_PLANNER_CHANGE_RESULTS, + R_RESOLVED_MODEL_ACTION, + R_PLANNER_RESOLVED_ACTIONS, + R_DIRECT_MODEL_VERSION_IDENTITIES, + R_PLANNER_MODEL_ENTRY_RESULTS, + R_SCHEMA_ACTION, + R_PLAN_WARNING, + R_COLUMN_RENAME_HINT, + R_QUERY_PROJECTION, + R_QUERY_SHAPE, + R_PLAN_OUTPUT_EXTRAS, + R_PLANNED_SQL_TESTS, + R_NATIVE_SQL_TEST_PLAN, + R_SQL_TEST_PLAN_RESULT, + R_SCENARIO_GRAPH_PLAN, + R_SCENARIO_ARTIFACT_IDENTITY, + R_SCENARIO_ARTIFACT_NAME, + R_PLAN_PROVIDER_USAGE, + R_PLANNER_RUNTIME, + R_PLANNER_SCOPE_RESOLUTION, + R_PLANNER_WAREHOUSE_STATE, + R_MODEL_MIGRATION_REQUEST, + R_MODEL_MIGRATION_DECLARATION, + R_MODEL_MIGRATION_DISCOVERY, + R_MIGRATION_COMPATIBILITY_RESULT, + R_MODEL_MIGRATION_PLANNING, + R_OLD_NAME_VIEW_PLANNING, + R_COLUMN_MIGRATION_PLANNING, + R_PLANNER_IDENTITY_CONTEXT, + R_PLANNER_SCOPE_PRUNING_RESULT, + R_PLANNER_CHANGE_RECONCILIATION, + R_PLANNER_ENTRY_RESULTS, +]; + +/// Planner string enums, in declaration order. +pub(crate) const PLANNER_ENUMS: &[EnumSpec] = &[ + EnumSpec { + name: "GraphResourceKind", + doc: "", + members: &[ + ("MODEL", "model"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("FUNCTION", "function"), + ], + }, + EnumSpec { + name: "ChangeKind", + doc: "", + members: &[ + ("FIRST_RUN", "first_run"), + ("RENAMED", "renamed"), + ("QUERY_CHANGED", "query_changed"), + ("FUNCTION_CHANGED", "function_changed"), + ("CONFIG_CHANGED", "config_changed"), + ("SCHEMA_CHANGED", "schema_changed"), + ("NO_CHANGE", "no_change"), + ], + }, + EnumSpec { + name: "SchemaChangeKind", + doc: "", + members: &[ + ("COLUMN_ADDED", "column_added"), + ("COLUMN_REMOVED", "column_removed"), + ("COLUMN_TYPE_CHANGED", "column_type_changed"), + ], + }, + EnumSpec { + name: "SchemaColumnSource", + doc: "", + members: &[("YML", "yml"), ("SQL_ANALYSIS", "sql_analysis")], + }, + EnumSpec { + name: "CursorWatermarkMode", + doc: "", + members: &[("ALL", "all"), ("ANY", "any")], + }, + EnumSpec { + name: "SchemaActionKind", + doc: "", + members: &[ + ("ADD_COLUMN", "add_column"), + ("DROP_COLUMN", "drop_column"), + ("ALTER_COLUMN_TYPE", "alter_column_type"), + ], + }, + EnumSpec { + name: "WarningSeverity", + doc: "", + members: &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], + }, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index 62cfa9cd58..a4c60b4b08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,13 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, - planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, - planner_strategy, planner_warehouse, relation_naming, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, + planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, + planner_warehouse, relation_naming, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - planner_models::registration::register(module)?; planner_changes::registration::register(module)?; planner_migrations::registration::register(module)?; planner_phases::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs index f968c85d17..1daebfe0ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_bounds.rs @@ -15,7 +15,7 @@ use sqlbuild_cursor::planner::main::raw_bounds::raw_bounds; use sqlbuild_cursor::planner::main::replay_start::replay_start; use sqlbuild_cursor::planner::models::{CursorKind, CursorSnapshot, ReplayPolicy}; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; const MAXIMUM_START: &str = "sqlbuild._native"; pub(super) fn optional_scalar<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index cce4a7c1d2..c9d3b069c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -12,7 +12,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyDict; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; const CURSOR: &str = "sqlbuild._native"; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { @@ -163,11 +163,7 @@ fn compute_model_cursor_bounds<'py>( ], )?; return Err(PyErr::from_value(call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &[ ("message", refusal.get_item(0)?), ("code", string(py, "S302")), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 5ef35e5179..cdc86a32b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -68,12 +68,12 @@ fn type_cursor_overrides<'py>( } } let kind = override_kind(&models, integer_literals).map_err(|message| { - match imported(py, "sqlbuild.compiler.planner.exceptions", "PlannerInputError").and_then(|class| call_with(&class, &[ + match imported(py, "sqlbuild._native", "PlannerInputError").and_then(|class| call_with(&class, &[ ("message", string(py, &message)), ("code", string(py, "S303")), ("help", string(py, "select models with one cursor type, or set each type explicitly with --start-cursor-ts/--end-cursor-ts and --start-cursor-int/--end-cursor-int")), ])) { Ok(error) => PyErr::from_value(error), Err(error) => error } })?; - let kind = imported(py, "sqlbuild.compiler.planner.types", "CursorType")?.call1((kind,))?; + let kind = imported(py, "sqlbuild._native", "CursorType")?.call1((kind,))?; method(&overrides, "typed_as", &[("cursor_type", kind)]) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs index f0bd2a13dc..fd73879b06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_policies.rs @@ -5,8 +5,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::PyString; use pyo3::{pyfunction, wrap_pyfunction}; -const SPEC_MODELS: &str = "sqlbuild.spec.contracts.models"; -const SPEC_TYPES: &str = "sqlbuild.spec.contracts.types"; +const SPEC_MODELS: &str = "sqlbuild._native"; +const SPEC_TYPES: &str = "sqlbuild._native"; const POLICY_DISABLED: &str = "disabled"; /// The policy class, its distance field, and the model keys overriding distance and action. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs index dcf00e724a..9f973de211 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs @@ -22,10 +22,10 @@ use sqlbuild_cursor::planner::main::rolling_window::rolling_window; use sqlbuild_cursor::planner::models::CursorEndClamp; use std::cmp::Ordering; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; const CURSOR_TYPES: &str = "sqlbuild.cursor_algebra.types"; -const CURSOR_MODELS: &str = "sqlbuild.cursor_algebra.models"; -const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; +const CURSOR_MODELS: &str = "sqlbuild._native"; +const COMPILE_EXCEPTIONS: &str = "sqlbuild._native"; const INCREMENTAL: &str = "incremental"; const MICROBATCH: &str = "microbatch"; const ROLLING_WINDOW: &str = "rolling_window"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs index e26c003d12..104c8965de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_safety.rs @@ -7,8 +7,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_cursor::algebra::models::{DurationSpan, Grain, Scalar}; use sqlbuild_cursor::planner::main::safety_horizon::safety_horizon; -pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; +pub(super) const MODELS: &str = "sqlbuild._native"; +pub(super) const EXCEPTIONS: &str = "sqlbuild._native"; pub(super) fn safety_error(py: Python<'_>, class: &str, message: &str) -> PyErr { match imported(py, EXCEPTIONS, class).and_then(|class| class.call1((message,))) { @@ -44,15 +44,10 @@ pub(super) fn safety_duration(value: &Bound<'_, PyAny>, future: bool) -> PyResul pub(super) fn timestamp_bound(value: &Bound<'_, PyAny>) -> PyResult { let py = value.py(); - Ok(value.is_instance(&imported( - py, - "sqlbuild.cursor_algebra.models", - "DateValue", - )?)? || value.is_instance(&imported( - py, - "sqlbuild.cursor_algebra.models", - "TimestampValue", - )?)?) + Ok( + value.is_instance(&imported(py, "sqlbuild._native", "DateValue")?)? + || value.is_instance(&imported(py, "sqlbuild._native", "TimestampValue")?)?, + ) } pub(super) fn utc_clock<'py>(clock: &Bound<'py, PyAny>) -> PyResult> { @@ -68,11 +63,7 @@ pub(super) fn horizon<'py>( let py = session.py(); let (clock, grain) = clock_and_grain; let mut clock = utc_clock(clock)?; - let date_only = discovered.is_instance(&imported( - py, - "sqlbuild.cursor_algebra.models", - "DateValue", - )?)?; + let date_only = discovered.is_instance(&imported(py, "sqlbuild._native", "DateValue")?)?; if date_only || discovered.getattr("value")?.getattr("tzinfo")?.is_none() { clock = call_with( &clock.getattr("replace")?, @@ -80,7 +71,7 @@ pub(super) fn horizon<'py>( )?; } let clock = call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", clock)], )?; let grain = if grain.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs index 4c268b6fb0..29be985ba0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_windows.rs @@ -10,7 +10,7 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_cursor::algebra::main::floor_to_grain::floor_to_grain; use sqlbuild_cursor::planner::main::empty_inputs::empty_inputs; -const MODELS: &str = "sqlbuild.compiler.planner.models"; +const MODELS: &str = "sqlbuild._native"; #[pyfunction] #[pyo3(signature = (*, empty_input_names, waits_for_every_input, inputs_with_rows, has_window_start))] @@ -94,7 +94,7 @@ fn empty_cursor_window<'py>( let mut bound = parse_value(&raw, &kind)?; if cursor_start.is_none() && !kind.eq("integer")? { bound = call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", bound.getattr("value")?)], )?; } @@ -143,11 +143,7 @@ fn unreadable_cursor_inputs_error<'py>( input_names: Vec, ) -> PyResult> { call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &[ ( "message", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs index 0163650f9e..099ce98ded 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/future_safety.rs @@ -127,7 +127,7 @@ fn resolve_future_cursor_safety<'py>( ( "invocation_time", call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", utc_clock(&clock)?)], )?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index a14a30a3ea..6d078b4098 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -188,7 +188,7 @@ fn apply_maximum_start_policy<'py>( ( "invocation_time", call_with( - &imported(py, "sqlbuild.cursor_algebra.models", "TimestampValue")?, + &imported(py, "sqlbuild._native", "TimestampValue")?, &[("value", utc_clock(&clock)?)], )?, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index 600cfd0673..9e0b106a42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -9,8 +9,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const ERROR_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const HOOK_MODULE: &str = "sqlbuild._native"; +const ERROR_MODULE: &str = "sqlbuild._native"; struct Resolution<'py> { adapter: Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 04db2673d2..9a24f89aa1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -161,11 +161,7 @@ fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { } if let Some(code) = code { return Err(PyErr::from_value(call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &[ ("message", string(py, &messages.join("\n"))), ("code", code), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index d5aca78b04..cb8e14f3d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -142,11 +142,7 @@ fn target_for_artifact<'py>( ], )?; call_with( - &imported( - py, - "sqlbuild.compiler.compile.models", - "CompiledRelationLocation", - )?, + &imported(py, "sqlbuild._native", "CompiledRelationLocation")?, &[ ("database", database), ("schema", schema), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs index a5e5f5805d..57b3d25c93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs @@ -4,8 +4,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use pyo3::prelude::{Bound, PyAny, PyErr, PyResult, Python}; use pyo3::types::PyString; -pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(super) const TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const MODELS: &str = "sqlbuild._native"; +pub(super) const TYPES: &str = "sqlbuild._native"; pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() @@ -22,11 +22,7 @@ pub(super) fn error( args.push(("help", string(py, help))); } Ok(PyErr::from_value(call_with( - &imported( - py, - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - )?, + &imported(py, "sqlbuild._native", "PlannerInputError")?, &args, )?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs index a9fd5203d5..56545f3785 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -8,8 +8,8 @@ use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::planner_scope::models::LoaderGraph; pub(crate) const NATIVE: &str = "sqlbuild._native"; -pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; -pub(crate) const SOURCE_MODELS: &str = "sqlbuild.spec.contracts.models"; +pub(crate) const MODELS: &str = "sqlbuild._native"; +pub(crate) const SOURCE_MODELS: &str = "sqlbuild._native"; pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { values diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs index 0bf31dc9c1..a34b4384f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs @@ -8,7 +8,7 @@ use sqlbuild_analysis::graph::errors::SelectorError; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; /// `str.translate` tables to and from carrier characters; empty when no text holds a surrogate. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index fbf692a524..b0e46157c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::planner_strategy::values::{ }; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const MODEL_RESOURCE_TYPE: &str = "model"; const FINAL_RUN_SCOPE: &str = "final"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs index d46eb709b9..b306b03409 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/constants.rs @@ -3,11 +3,11 @@ pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(crate) const AUDIT_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; pub(crate) const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +pub(crate) const PLAN_MODELS_MODULE: &str = "sqlbuild._native"; pub(crate) const MODEL_ATTACHMENT: &str = "model"; pub(crate) const SOURCE_ATTACHMENT: &str = "source"; -pub(crate) const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +pub(crate) const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; pub(crate) const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; pub(crate) const CURSOR_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs index ea42c15438..596c9526f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatches.rs @@ -24,8 +24,8 @@ use crate::bindings::_helpers::plan_output::display::aligned_timestamp_bounds; use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_strategy::values::text; -const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; -pub(crate) const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild._native"; +pub(crate) const CONTRACT_TYPES_MODULE: &str = "sqlbuild._native"; const INTEGER_CURSOR: &str = "integer"; const TIMESTAMP_CURSOR: &str = "timestamp"; const MONTHS_PER_YEAR: i64 = 12; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs index 61936868fe..14815320f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -15,8 +15,8 @@ use sqlbuild_analysis::planner_strategy::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; -pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; +pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const YML_SOURCE: &str = "yml"; const FULL_BACKFILL: &str = "full"; const FORWARD_ONLY_BACKFILL: &str = "forward"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs index 09151463ce..49765eee0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/constants.rs @@ -42,19 +42,12 @@ pub(crate) const RENDER_SOURCE_READ: (&str, &str) = pub(crate) const YAML_ENTRY_LINE: (&str, &str) = ("sqlbuild._native", "yaml_entry_line"); pub(crate) const RUN_SCOPE: (&str, &str) = ("sqlbuild.runtime.observability.main.run_scope", "run_scope"); -pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = - ("sqlbuild.compiler.discovery.models", "SqlHookEntry"); +pub(crate) const SQL_HOOK_ENTRY: (&str, &str) = ("sqlbuild._native", "SqlHookEntry"); pub(crate) const CURSOR_ALGEBRA_ERROR: (&str, &str) = ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"); -pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild.cursor_algebra.models", "DateValue"); +pub(crate) const DATE_VALUE: (&str, &str) = ("sqlbuild._native", "DateValue"); pub(crate) const PLANNER_LOGGER: &str = "sqlbuild.planner"; -pub(crate) const FINGERPRINT_TABLE_NAME: &str = "_sqlbuild_fingerprints"; -pub(crate) const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; -pub(crate) const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; -pub(crate) const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; pub(crate) const METADATA_NAME_FILTER_LIMIT: usize = 250; -pub(crate) const NODE_TYPE_MODEL: &str = "model"; -pub(crate) const NODE_TYPE_SEED: &str = "seed"; pub(crate) const FUNCTION_NODE_TYPES: [&str; 2] = ["udf", "table_fn"]; pub(crate) const WILDCARD_TABLE_CHARACTER: &str = "*"; pub(crate) const MIGRATE_FROM_CONFIG_KEY: &str = "migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index f46a99e78a..81872bf21a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -7,10 +7,8 @@ use crate::bindings::_helpers::planner_phases::objects::{ by_name, frozenset, model_key, native, text, tuple, }; use crate::bindings::_helpers::planner_warehouse::constants::{ - ACTIVE_CATALOG, FINGERPRINT_TABLE_NAME, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, - MIGRATE_FROM_CONFIG_KEY, MIGRATION_TABLE_NAME, NODE_TYPE_MODEL, NODE_TYPE_SEED, - OLD_NAME_VIEW_TABLE_NAME, RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, - SOURCE_FRESHNESS_TABLE_NAME, + ACTIVE_CATALOG, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, + RELATIONS_MAIN_COLUMNS, RELATIONS_MAIN_LIST, }; use crate::bindings::_helpers::planner_warehouse::cursors::{CursorInputs, cursor_snapshots}; use pyo3::IntoPyObject; @@ -18,6 +16,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_MODEL; +use sqlbuild_analysis::fingerprint_storage::constants::NODE_TYPE_SEED; +use sqlbuild_core::constants::FINGERPRINT_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; use std::collections::{BTreeSet, HashSet}; /// Everything one snapshot gather needs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 4253a2d8c2..a4cb860f93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::project_assembly::models::{ }; use crate::bindings::types::CompilerDetach; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index b6b7f7fe44..28fb19a8fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -12,10 +12,10 @@ use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const GENERIC_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs new file mode 100644 index 0000000000..737a0b63b1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs @@ -0,0 +1,106 @@ +//! Validation of the natively hosted measurement audit records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyFloat}; + +use sqlbuild_model_config::header_metadata::main::check_measurement_contract::check_measurement_contract; +use sqlbuild_model_config::header_metadata::main::check_threshold_bound::check_threshold_bound; +use sqlbuild_model_config::header_metadata::main::check_threshold_pair::check_threshold_pair; +use sqlbuild_model_config::header_metadata::models::{ThresholdBound, ThresholdInput}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +const MEASUREMENT_AUDIT_ERROR: &str = "MeasurementAuditError"; + +/// The `MeasurementAuditError` carrying `message`. +fn measurement_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, NATIVE_MODULE, MEASUREMENT_AUDIT_ERROR) + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// How Python's threshold validation sees one limit value. +fn threshold_input(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(ThresholdInput::Absent); + } + if value.is_instance_of::() { + return Ok(ThresholdInput::Boolean); + } + Ok(ThresholdInput::Number(value.extract::()?)) +} + +/// `MeasurementThresholdBound`'s operator, limit, lower and upper, checked and made floats. +pub(crate) fn finish_threshold_bound( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let operator: String = values[0].bind(py).str()?.extract()?; + let inputs = ( + threshold_input(values[1].bind(py))?, + threshold_input(values[2].bind(py))?, + threshold_input(values[3].bind(py))?, + ); + let float = |value: f64| PyFloat::new(py, value).into_any().unbind(); + let none = || py.None(); + let (limit, lower, upper) = match check_threshold_bound(&operator, inputs) { + Ok(ThresholdBound::Below(limit) | ThresholdBound::Above(limit)) => { + (float(limit), none(), none()) + } + Ok(ThresholdBound::Outside(lower, upper)) => (none(), float(lower), float(upper)), + Err(message) => return Err(measurement_error(py, &message)), + }; + let mut values: Vec> = values; + values[1] = limit; + values[2] = lower; + values[3] = upper; + Ok(values) +} + +/// The checked bound a `MeasurementThresholdBound` record holds, or `None`. +fn record_bound(bound: &Bound<'_, PyAny>) -> PyResult> { + if bound.is_none() { + return Ok(None); + } + let operator: String = bound.getattr("operator")?.str()?.extract()?; + let inputs = ( + threshold_input(&bound.getattr("limit")?)?, + threshold_input(&bound.getattr("lower")?)?, + threshold_input(&bound.getattr("upper")?)?, + ); + check_threshold_bound(&operator, inputs) + .map(Some) + .map_err(|message| measurement_error(bound.py(), &message)) +} + +/// `MeasurementThresholds`' warn and error bounds, checked as one policy. +pub(crate) fn finish_thresholds( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let warn: Option = record_bound(values[0].bind(py))?; + let error: Option = record_bound(values[1].bind(py))?; + check_threshold_pair(warn, error).map_err(|message| measurement_error(py, message))?; + Ok(values) +} + +/// `MeasurementContract`'s named columns and unit, checked non-blank. +pub(crate) fn finish_measurement_contract( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let value_column: String = values[0].bind(py).extract()?; + let sample_count_column: Option = values[1].bind(py).extract()?; + let sample_unit: Option = values[2].bind(py).extract()?; + check_measurement_contract( + &value_column, + sample_count_column.as_deref(), + sample_unit.as_deref(), + ) + .map_err(|message| measurement_error(py, message))?; + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs new file mode 100644 index 0000000000..700d176871 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/call_arguments.rs @@ -0,0 +1,18 @@ +//! Reading the arguments of a native method call. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; + +/// The argument `name`, passed by keyword or at `position`. +pub(crate) fn call_argument<'py>( + (args, kwargs): (&Bound<'py, PyTuple>, Option<&Bound<'py, PyDict>>), + name: &str, + position: usize, +) -> PyResult> { + if let Some(kwargs) = kwargs + && let Some(value) = kwargs.get_item(name)? + { + return Ok(value); + } + args.get_item(position) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs new file mode 100644 index 0000000000..d04e1d558e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -0,0 +1,384 @@ +//! Normalization and derived views of the natively hosted compile output records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; +use crate::bindings::_helpers::rules::relation_keys::entry_relation_key_row; + +/// Module of the compiled resource type enum. +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +/// `CompilerDiagnostic` positions. +const DIAGNOSTIC_PHASE_FIELD: usize = 0; +const DIAGNOSTIC_SEVERITY_FIELD: usize = 1; +const DIAGNOSTIC_RESOURCE_TYPE_FIELD: usize = 4; +const DIAGNOSTIC_PATH_FIELD: usize = 7; +const DIAGNOSTIC_LINE_FIELD: usize = 8; +const DIAGNOSTIC_COLUMN_FIELD: usize = 9; +const DIAGNOSTIC_LOCATION_FIELD: usize = 10; +/// `CompiledSqlScenario` positions. +const SCENARIO_FILE_FIELD: usize = 2; +const SCENARIO_EXPECTED_MODELS_FIELD: usize = 11; +const SCENARIO_ASSERTION_TARGETS_FIELD: usize = 13; +const SCENARIO_TARGET_MODELS_FIELD: usize = 14; +const SCENARIO_SOURCE_PATH_FIELD: usize = 15; +const SCENARIO_OWNERSHIP_ROOT_FIELD: usize = 16; +/// `CompiledSqlTest` positions. +const TEST_FILE_FIELD: usize = 3; +const TEST_BLOCK_FIELD: usize = 4; +const TEST_SOURCE_PATH_FIELD: usize = 8; +const TEST_OWNERSHIP_ROOT_FIELD: usize = 9; +const TEST_BLOCK_INDEX_FIELD: usize = 10; +const TEST_EXPLICIT_NAME_FIELD: usize = 11; +const TEST_PARENT_NAME_FIELD: usize = 12; + +/// `class(value)`. +fn coerce(py: Python<'_>, value: &Py, (module, class): (&str, &str)) -> PyResult> { + Ok(imported(py, module, class)? + .call1((value.bind(py),))? + .unbind()) +} + +/// `value` coerced to `CompiledResourceType` unless it is `None`. +fn coerce_resource_type(py: Python<'_>, value: &Py) -> PyResult> { + if value.is_none(py) { + return Ok(value.clone_ref(py)); + } + coerce(py, value, (COMPILE_TYPES_MODULE, "CompiledResourceType")) +} + +/// `CompilerDiagnostic`: enum fields coerced, and its location and path, line and column aligned. +pub(crate) fn finish_compiler_diagnostic( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + values[DIAGNOSTIC_PHASE_FIELD] = coerce( + py, + &values[DIAGNOSTIC_PHASE_FIELD], + (NATIVE_MODULE, "DiagnosticPhase"), + )?; + values[DIAGNOSTIC_SEVERITY_FIELD] = coerce( + py, + &values[DIAGNOSTIC_SEVERITY_FIELD], + (NATIVE_MODULE, "DiagnosticSeverity"), + )?; + values[DIAGNOSTIC_RESOURCE_TYPE_FIELD] = + coerce_resource_type(py, &values[DIAGNOSTIC_RESOURCE_TYPE_FIELD])?; + let positioned: bool = [ + DIAGNOSTIC_PATH_FIELD, + DIAGNOSTIC_LINE_FIELD, + DIAGNOSTIC_COLUMN_FIELD, + ] + .iter() + .all(|position| !values[*position].is_none(py)); + if values[DIAGNOSTIC_LOCATION_FIELD].is_none(py) && positioned { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("path", values[DIAGNOSTIC_PATH_FIELD].bind(py))?; + keywords.set_item("line", values[DIAGNOSTIC_LINE_FIELD].bind(py))?; + keywords.set_item("column", values[DIAGNOSTIC_COLUMN_FIELD].bind(py))?; + let location: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "SourceLocation")?.call((), Some(&keywords))?; + values[DIAGNOSTIC_LOCATION_FIELD] = location.unbind(); + } + let location: Bound<'_, PyAny> = values[DIAGNOSTIC_LOCATION_FIELD].bind(py).clone(); + if !location.is_none() { + values[DIAGNOSTIC_PATH_FIELD] = location.getattr("path")?.unbind(); + values[DIAGNOSTIC_LINE_FIELD] = location.getattr("line")?.unbind(); + values[DIAGNOSTIC_COLUMN_FIELD] = location.getattr("column")?.unbind(); + } + Ok(values) +} + +/// `CompilerDiagnostic.is_error`: whether the diagnostic fails the command. +pub(crate) fn diagnostic_is_error(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let error: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?; + instance.getattr("severity")?.eq(error)?.into_py_any(py) +} + +/// `CompileSqlReference`: its kind coerced to `SqlReferenceKind`. +pub(crate) fn finish_sql_reference( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + values[0] = coerce(py, &values[0], (NATIVE_MODULE, "SqlReferenceKind"))?; + Ok(values) +} + +/// `CompiledLineageSourceFact` and `CompiledObjectKey`: the resource type coerced. +pub(crate) fn finish_resource_typed( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + values[0] = coerce( + py, + &values[0], + (COMPILE_TYPES_MODULE, "CompiledResourceType"), + )?; + Ok(values) +} + +/// `value`, or `fallback` when `value` is `None`. +fn or_fallback(py: Python<'_>, value: &Py, fallback: Bound<'_, PyAny>) -> Py { + if value.is_none(py) { + return fallback.unbind(); + } + value.clone_ref(py) +} + +/// The distinct names of `first` then `second`, in first-seen order. +fn merged_names<'py>( + first: &Bound<'py, PyAny>, + second: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = first.py(); + let seen: Bound<'py, PyDict> = PyDict::new(py); + for name in first.try_iter()?.chain(second.try_iter()?) { + seen.set_item(name?, py.None())?; + } + Ok(PyTuple::new(py, seen.keys())?.into_any()) +} + +/// `CompiledSqlScenario`: source path, ownership and target models taken from its file. +pub(crate) fn finish_sql_scenario( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let scenario_file: Bound<'_, PyAny> = values[SCENARIO_FILE_FIELD].bind(py).clone(); + values[SCENARIO_SOURCE_PATH_FIELD] = or_fallback( + py, + &values[SCENARIO_SOURCE_PATH_FIELD], + scenario_file.getattr("relative_path")?, + ); + values[SCENARIO_OWNERSHIP_ROOT_FIELD] = or_fallback( + py, + &values[SCENARIO_OWNERSHIP_ROOT_FIELD], + scenario_file.getattr("ownership_root")?, + ); + if !values[SCENARIO_TARGET_MODELS_FIELD].bind(py).is_truthy()? { + let targets: Bound<'_, PyAny> = merged_names( + values[SCENARIO_EXPECTED_MODELS_FIELD].bind(py), + values[SCENARIO_ASSERTION_TARGETS_FIELD].bind(py), + )?; + values[SCENARIO_TARGET_MODELS_FIELD] = targets.unbind(); + } + Ok(values) +} + +/// `CompiledSqlTest`: source path, ownership, block index and names taken from its file and block. +pub(crate) fn finish_sql_test( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let test_file: Bound<'_, PyAny> = values[TEST_FILE_FIELD].bind(py).clone(); + let test_block: Bound<'_, PyAny> = values[TEST_BLOCK_FIELD].bind(py).clone(); + values[TEST_SOURCE_PATH_FIELD] = or_fallback( + py, + &values[TEST_SOURCE_PATH_FIELD], + test_file.getattr("relative_path")?, + ); + values[TEST_OWNERSHIP_ROOT_FIELD] = or_fallback( + py, + &values[TEST_OWNERSHIP_ROOT_FIELD], + test_file.getattr("ownership_root")?, + ); + values[TEST_BLOCK_INDEX_FIELD] = or_fallback( + py, + &values[TEST_BLOCK_INDEX_FIELD], + test_block.getattr("test_index")?, + ); + let block_name: Bound<'_, PyAny> = test_block.getattr("name")?; + if !block_name.is_none() { + values[TEST_EXPLICIT_NAME_FIELD] = + or_fallback(py, &values[TEST_EXPLICIT_NAME_FIELD], block_name.clone()); + } + if values[TEST_PARENT_NAME_FIELD].is_none(py) { + let parent: Bound<'_, PyAny> = if block_name.is_truthy()? { + block_name + } else { + test_file.getattr("relative_path")?.getattr("stem")? + }; + values[TEST_PARENT_NAME_FIELD] = parent.unbind(); + } + Ok(values) +} + +/// `(kind, name, package, call argument count)` of each reference in `references`. +fn reference_rows(references: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = references.py(); + let rows: Bound<'_, PyList> = PyList::empty(py); + for reference in references.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + rows.append(( + reference.getattr("ref_kind")?.str()?, + reference.getattr("ref_name")?, + reference.getattr("ref_package")?, + reference.getattr("call_argument_count")?, + ))?; + } + Ok(rows.into_any().unbind()) +} + +/// `CompileModelInput.reference_rows` and `CompiledModel.reference_rows`. +pub(crate) fn record_reference_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + reference_rows(&instance.getattr("references")?) +} + +/// The declared key facts of model `name` with `instance`'s config and schema entry. +fn model_relation_key_row(instance: &Bound<'_, PyAny>, name: String) -> PyResult> { + let values: Bound<'_, PyAny> = instance.getattr("config")?.getattr("values")?; + entry_relation_key_row( + "ref", + name, + values.cast::()?, + &instance.getattr("schema_entry")?, + )? + .into_py_any(instance.py()) +} + +/// `CompileModelInput.relation_key_row`: keyed by the model file's stem. +pub(crate) fn input_relation_key_row(instance: &Bound<'_, PyAny>) -> PyResult> { + let name: String = instance + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")? + .extract()?; + model_relation_key_row(instance, name) +} + +/// `CompiledModel.relation_key_row`: keyed by the model name. +pub(crate) fn model_relation_key_row_property(instance: &Bound<'_, PyAny>) -> PyResult> { + let name: String = instance.getattr("name")?.extract()?; + model_relation_key_row(instance, name) +} + +/// `CompileModelInput.attached_schema_name`: the attached schema entry's name, or `None`. +pub(crate) fn input_attached_schema_name(instance: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + if entry.is_none() { + return Ok(entry.unbind()); + } + Ok(entry.getattr("name")?.unbind()) +} + +/// `(resource type, name)` of one compiled object key. +fn key_row<'py>(key: &Bound<'py, PyAny>) -> PyResult<(String, String)> { + Ok(( + key.getattr("resource_type")?.str()?.extract()?, + key.getattr("name")?.extract()?, + )) +} + +/// `(resource type, name)` of each compiled object key. +fn key_rows(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| key_row(&key?)).collect() +} + +/// `CompiledModel.key_row`. +pub(crate) fn model_key_row(instance: &Bound<'_, PyAny>) -> PyResult> { + key_row(&instance.getattr("key")?)?.into_py_any(instance.py()) +} + +/// `CompiledModel.dep_rows`. +pub(crate) fn model_dep_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + key_rows(&instance.getattr("deps")?)?.into_py_any(instance.py()) +} + +/// `CompiledModel.destination_row`: `(database, schema, name)` of the destination. +pub(crate) fn model_destination_row(instance: &Bound<'_, PyAny>) -> PyResult> { + let destination: Bound<'_, PyAny> = instance.getattr("destination")?; + ( + destination.getattr("database")?, + destination.getattr("schema")?, + destination.getattr("name")?, + ) + .into_py_any(instance.py()) +} + +/// `CompiledModel.inferred_rows`: each inferred column as `(name, type, nullability)`. +pub(crate) fn model_inferred_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + let columns: Bound<'_, PyAny> = instance.getattr("inferred_columns")?; + if columns.is_none() { + return Ok(columns.unbind()); + } + let rows: Bound<'_, PyList> = PyList::empty(instance.py()); + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + rows.append(( + column.getattr("name")?, + column.getattr("type")?, + column.getattr("nullability")?.str()?, + ))?; + } + Ok(rows.into_any().unbind()) +} + +/// `CompiledModel.schema_column_rows`: each declared schema column as `(name, type)`. +pub(crate) fn model_schema_column_rows(instance: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + if entry.is_none() { + return Ok(entry.unbind()); + } + let rows: Bound<'_, PyList> = PyList::empty(instance.py()); + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + rows.append((column.getattr("name")?, column.getattr("type")?))?; + } + Ok(rows.into_any().unbind()) +} + +/// `CompiledModel.has_dynamic_columns`. +pub(crate) fn model_has_dynamic_columns(instance: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + let dynamic: bool = !entry.is_none() && entry.getattr("dynamic_columns")?.is_truthy()?; + dynamic.into_py_any(instance.py()) +} + +/// `CompiledModel.output_column_count`: inferred, else declared, columns, else zero. +pub(crate) fn model_output_column_count(instance: &Bound<'_, PyAny>) -> PyResult> { + let inferred: Bound<'_, PyAny> = instance.getattr("inferred_columns")?; + let entry: Bound<'_, PyAny> = instance.getattr("schema_entry")?; + let count: usize = if !inferred.is_none() { + inferred.len()? + } else if !entry.is_none() { + entry.getattr("columns")?.len()? + } else { + 0 + }; + count.into_py_any(instance.py()) +} + +/// `ModelInputBuildContext.declaration_expansion`: the public declarations for non-model SQL. +pub(crate) fn build_context_declaration_expansion( + instance: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let declarations: Bound<'_, PyDict> = PyDict::new(py); + declarations.set_item("enums", instance.getattr("public_enums")?)?; + declarations.set_item("constants", instance.getattr("public_constants")?)?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item( + "declarations", + imported(py, NATIVE_MODULE, "DeclarationResolutionContext")? + .call((), Some(&declarations))?, + )?; + keywords.set_item("value_renderer", instance.getattr("value_renderer")?)?; + keywords.set_item( + "collection_rendering", + instance.getattr("collection_rendering")?, + )?; + keywords.set_item("resolver", instance.getattr("declaration_resolver")?)?; + Ok(imported(py, NATIVE_MODULE, "DeclarationExpansionContext")? + .call((), Some(&keywords))? + .unbind()) +} + +/// A `CompileInputError` carrying `message`. +pub(crate) fn compile_input_error(py: Python<'_>, message: &str) -> PyResult { + Ok(PyErr::from_value( + imported(py, NATIVE_MODULE, "CompileInputError")?.call1((message,))?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs new file mode 100644 index 0000000000..5278ad1d62 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/config.rs @@ -0,0 +1,8 @@ +//! Computed attributes of the natively hosted project configuration records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; + +/// `SettingsConfig.sql_validation`: the legacy name of the unified SQL analysis gate. +pub(crate) fn settings_sql_validation(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(instance.getattr("sql_analysis")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs new file mode 100644 index 0000000000..8025590a3b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -0,0 +1,351 @@ +//! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. + +use crate::bindings::_helpers::record_behaviours::auditing::{ + finish_measurement_contract, finish_threshold_bound, finish_thresholds, +}; +use crate::bindings::_helpers::record_behaviours::compiled::{ + build_context_declaration_expansion, diagnostic_is_error, finish_compiler_diagnostic, + finish_resource_typed, finish_sql_reference, finish_sql_scenario, finish_sql_test, + input_attached_schema_name, input_relation_key_row, model_dep_rows, model_destination_row, + model_has_dynamic_columns, model_inferred_rows, model_key_row, model_output_column_count, + model_relation_key_row_property, model_schema_column_rows, record_reference_rows, +}; +use crate::bindings::_helpers::record_behaviours::config::settings_sql_validation; +use crate::bindings::_helpers::record_behaviours::cursors::{ + finish_cursor_bounds, finish_cursor_snapshot, finish_future_evidence, finish_input_evidence, + finish_input_relation, finish_maximum_evidence, input_relation_runtime_owned, + snapshot_watermarks_available, +}; +use crate::bindings::_helpers::record_behaviours::discovery::{ + constant_logical_type, constant_scalar_type, graph_all_keys, graph_downstream_deps, + graph_lineage_views, graph_path_index, graph_tag_index, graph_upstream_deps, +}; +use crate::bindings::_helpers::record_behaviours::durations::{ + duration_add_to, duration_fixed_seconds, duration_has_calendar_component, + duration_is_single_unit_in, duration_is_zero, duration_parse, duration_subtract_from, + duration_total_months, duration_units, finish_aligned_interval, +}; +use crate::bindings::_helpers::record_behaviours::error_inits::{ + macro_argument_error_init, macro_argument_error_located, macro_argument_error_shifted, + rules_config_error_init, sql_test_reference_error_init, +}; +use crate::bindings::_helpers::record_behaviours::freshness::{ + finish_freshness_record, freshness_record_identity, +}; +use crate::bindings::_helpers::record_behaviours::lexical::{ + finish_lexical_syntax, lexical_cache_key, lexical_native_mapping, lexical_reads_differently, +}; +use crate::bindings::_helpers::record_behaviours::lineage_facts::{ + facts_eq, facts_getitem, facts_iter, facts_len, facts_projected, facts_resource_name, +}; +use crate::bindings::_helpers::record_behaviours::macros::{ + context_render_constant, context_render_enum_member, macro_injects_context, + project_lineage_graph, +}; +use crate::bindings::_helpers::record_behaviours::migrations::{ + column_decision_blocks_build, column_decision_label, column_decision_records_event, + column_decision_renames, column_event_mentions, column_event_renamed, decision_blocks_build, + decision_checks_compatibility, decision_moves_data, event_mentions, history_new, history_old, + history_status, relation_identity_property, relation_matches, value_plan_label, +}; +use crate::bindings::_helpers::record_behaviours::planner::{ + column_migration_blocks_build, column_migration_origin_missing, finish_cursor_overrides, + finish_scenario_artifact_kind, materialization_is_table_backed, model_migration_blocks_build, + model_migration_origin_hidden, model_migration_origin_missing, + model_migration_storage_transition, model_migration_target_label, old_name_runs_steps, + overrides_have_untyped, overrides_typed_as, plan_python_source_entries, projection_key, + rename_hint_message, retention_decreases, +}; +use crate::bindings::_helpers::record_behaviours::python_nodes::graph_selected_sql_refs; +use crate::bindings::_helpers::record_behaviours::references::{ + reference_example_call, reference_fixture_cte_prefix, reference_function_name, + reference_placeholder_call, +}; +use crate::bindings::_helpers::record_behaviours::rule_shapes::{ + project_path_depth, project_path_name, project_path_parts, rule_option_boolean, + rule_option_class_getitem, rule_option_integer, rule_option_integer_list, rule_option_string, + rule_option_string_list, rule_project_wide, rule_result_finding_count, +}; +use crate::bindings::_helpers::record_behaviours::scopes::{ + completeness_complete, target_query_unknown, +}; +use crate::bindings::_helpers::record_behaviours::values::{ + finish_fingerprint_set, logical_type_display_name, sql_value_kind, +}; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFinish}; + +/// Compiler records whose bound values are normalized or validated natively. +pub(crate) const BEHAVIOUR_FINISHERS: &[(&str, RecordFinish)] = &[ + ("ScenarioArtifactIdentity", finish_scenario_artifact_kind), + ("MeasurementThresholdBound", finish_threshold_bound), + ("MeasurementThresholds", finish_thresholds), + ("MeasurementContract", finish_measurement_contract), + ("FingerprintSet", finish_fingerprint_set), + ("SourceFreshnessRecord", finish_freshness_record), + ("SqlLexicalSyntax", finish_lexical_syntax), + ("AlignedInterval", finish_aligned_interval), + ("CursorOverrides", finish_cursor_overrides), + ("ModelCursorSnapshot", finish_cursor_snapshot), + ("CursorBounds", finish_cursor_bounds), + ("CursorInputEvidence", finish_input_evidence), + ("FutureCursorSafetyEvidence", finish_future_evidence), + ("MaximumStartSafetyEvidence", finish_maximum_evidence), + ("CursorInputRelation", finish_input_relation), + ("CompilerDiagnostic", finish_compiler_diagnostic), + ("CompileSqlReference", finish_sql_reference), + ("CompiledLineageSourceFact", finish_resource_typed), + ("CompiledObjectKey", finish_resource_typed), + ("CompiledSqlScenario", finish_sql_scenario), + ("CompiledSqlTest", finish_sql_test), +]; +/// Computed attributes of compiler shapes. +pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ + ("ColumnRenameHint", "message", rename_hint_message), + ("QueryProjection", "key", projection_key), + ("ScopeCompleteness", "complete", completeness_complete), + ("ScopeTargetQuery", "unknown", target_query_unknown), + ("SettingsConfig", "sql_validation", settings_sql_validation), + ("SqlLogicalType", "display_name", logical_type_display_name), + ("SqlValue", "kind", sql_value_kind), + ( + "SourceFreshnessRecord", + "identity", + freshness_record_identity, + ), + ("SqlLexicalSyntax", "cache_key", lexical_cache_key), + ("SqlLexicalSyntax", "native_mapping", lexical_native_mapping), + ("Duration", "is_zero", duration_is_zero), + ("Duration", "units", duration_units), + ( + "Duration", + "has_calendar_component", + duration_has_calendar_component, + ), + ("Duration", "fixed_seconds", duration_fixed_seconds), + ("Duration", "total_months", duration_total_months), + ("MigrationDecision", "moves_data", decision_moves_data), + ("MigrationDecision", "blocks_build", decision_blocks_build), + ( + "MigrationDecision", + "checks_compatibility", + decision_checks_compatibility, + ), + ("MigrationDecision", "label", value_plan_label), + ("MigrationPromotion", "label", value_plan_label), + ( + "ColumnMigrationDecision", + "renames", + column_decision_renames, + ), + ( + "ColumnMigrationDecision", + "records_event", + column_decision_records_event, + ), + ( + "ColumnMigrationDecision", + "blocks_build", + column_decision_blocks_build, + ), + ("ColumnMigrationDecision", "label", column_decision_label), + ("MigrationRelation", "identity", relation_identity_property), + ("OldNameViewHistory", "old", history_old), + ("OldNameViewHistory", "new", history_new), + ("CursorOverrides", "has_untyped", overrides_have_untyped), + ( + "ModelCursorSnapshot", + "watermarks_available", + snapshot_watermarks_available, + ), + ( + "CursorInputRelation", + "is_runtime_owned", + input_relation_runtime_owned, + ), + ("RetentionPlanEntry", "decreases", retention_decreases), + ( + "ModelMigrationPlanEntry", + "origin_missing", + model_migration_origin_missing, + ), + ( + "ModelMigrationPlanEntry", + "origin_hidden", + model_migration_origin_hidden, + ), + ( + "ModelMigrationPlanEntry", + "storage_transition", + model_migration_storage_transition, + ), + ( + "ModelMigrationPlanEntry", + "target_label", + model_migration_target_label, + ), + ( + "ModelMigrationPlanEntry", + "blocks_build", + model_migration_blocks_build, + ), + ("OldNameViewPlanEntry", "runs_steps", old_name_runs_steps), + ( + "ColumnMigrationPlanEntry", + "origin_missing", + column_migration_origin_missing, + ), + ( + "ColumnMigrationPlanEntry", + "blocks_build", + column_migration_blocks_build, + ), + ( + "PlanOutput", + "python_source_entries", + plan_python_source_entries, + ), + ("SqlReferenceKind", "function_name", reference_function_name), + ( + "SqlReferenceKind", + "fixture_cte_prefix", + reference_fixture_cte_prefix, + ), + ("ConstantDeclaration", "logical_type", constant_logical_type), + ("ConstantDeclaration", "scalar_type", constant_scalar_type), + ("ProjectGraph", "upstream_deps", graph_upstream_deps), + ("ProjectGraph", "downstream_deps", graph_downstream_deps), + ("ProjectGraph", "tag_index", graph_tag_index), + ("ProjectGraph", "path_index", graph_path_index), + ("ProjectGraph", "all_keys", graph_all_keys), + ("CompilerDiagnostic", "is_error", diagnostic_is_error), + ( + "ModelInputBuildContext", + "declaration_expansion", + build_context_declaration_expansion, + ), + ("CompileModelInput", "reference_rows", record_reference_rows), + ( + "CompileModelInput", + "relation_key_row", + input_relation_key_row, + ), + ( + "CompileModelInput", + "attached_schema_name", + input_attached_schema_name, + ), + ("CompiledModel", "reference_rows", record_reference_rows), + ( + "CompiledModel", + "relation_key_row", + model_relation_key_row_property, + ), + ("CompiledModel", "key_row", model_key_row), + ("CompiledModel", "dep_rows", model_dep_rows), + ("CompiledModel", "destination_row", model_destination_row), + ("CompiledModel", "inferred_rows", model_inferred_rows), + ( + "CompiledModel", + "schema_column_rows", + model_schema_column_rows, + ), + ( + "CompiledModel", + "has_dynamic_columns", + model_has_dynamic_columns, + ), + ( + "CompiledModel", + "output_column_count", + model_output_column_count, + ), + ("ProjectPath", "parts", project_path_parts), + ("ProjectPath", "name", project_path_name), + ("ProjectPath", "depth", project_path_depth), + ("Rule", "project_wide", rule_project_wide), + ("RuleResult", "finding_count", rule_result_finding_count), +]; +/// Computed attributes of compiler shapes that a record computes once and keeps. +pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ + ("ProjectGraph", "lineage_views", graph_lineage_views), + ("LoadedMacro", "injects_context", macro_injects_context), + ("CompactLineageFacts", "facts", facts_projected), + ("CompiledProject", "lineage_graph", project_lineage_graph), +]; +/// Methods of compiler shapes. +pub(crate) const BEHAVIOUR_METHODS: &[(&str, &str, ClassMethod)] = &[ + ( + "SqlLexicalSyntax", + "reads_differently_from_generic", + lexical_reads_differently, + ), + ("Duration", "parse", duration_parse), + ( + "PythonNodeGraph", + "selected_sql_refs", + graph_selected_sql_refs, + ), + ("Duration", "is_single_unit_in", duration_is_single_unit_in), + ("Duration", "add_to", duration_add_to), + ("MigrationRelation", "matches", relation_matches), + ("MigrationEvent", "mentions", event_mentions), + ("ColumnMigrationEvent", "mentions", column_event_mentions), + ("ColumnMigrationEvent", "renamed", column_event_renamed), + ("OldNameViewHistory", "status", history_status), + ( + "MaterializationType", + "is_table_backed", + materialization_is_table_backed, + ), + ("CursorOverrides", "typed_as", overrides_typed_as), + ("SqlReferenceKind", "example_call", reference_example_call), + ( + "SqlReferenceKind", + "placeholder_call", + reference_placeholder_call, + ), + ("MacroArgumentError", "__init__", macro_argument_error_init), + ( + "MacroArgumentError", + "shifted", + macro_argument_error_shifted, + ), + ( + "MacroArgumentError", + "located", + macro_argument_error_located, + ), + ( + "SqlTestReferenceError", + "__init__", + sql_test_reference_error_init, + ), + ("RulesConfigError", "__init__", rules_config_error_init), + ("Duration", "subtract_from", duration_subtract_from), + ("MacroContext", "render_constant", context_render_constant), + ( + "MacroContext", + "render_enum_member", + context_render_enum_member, + ), + ("CompactLineageFacts", "__len__", facts_len), + ("CompactLineageFacts", "__getitem__", facts_getitem), + ("CompactLineageFacts", "__iter__", facts_iter), + ("CompactLineageFacts", "__eq__", facts_eq), + ("CompactLineageFacts", "resource_name", facts_resource_name), + ("RuleOption", "__class_getitem__", rule_option_class_getitem), + ("RuleOption", "boolean", rule_option_boolean), + ("RuleOption", "integer", rule_option_integer), + ("RuleOption", "string", rule_option_string), + ("RuleOption", "string_list", rule_option_string_list), + ("RuleOption", "integer_list", rule_option_integer_list), +]; +/// Compiler records registered as `collections.abc.Sequence` virtual subclasses. +pub(crate) const BEHAVIOUR_SEQUENCES: &[&str] = &["CompactLineageFacts"]; +/// Compiler records ordered by their compared fields. +pub(crate) const BEHAVIOUR_ORDERED: &[&str] = &[ + "ResourceIdentity", + "DeclarationIdentity", + "OwnershipRoot", + "ProjectPath", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs new file mode 100644 index 0000000000..9fc69a029e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs @@ -0,0 +1,184 @@ +//! Cursor value normalization of the natively hosted planner cursor records. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyString, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// One cursor value conversion. +type Conversion = for<'py> fn(&Bound<'py, PyAny>) -> PyResult>; + +const SCALAR_CLASSES: [&str; 3] = ["TimestampValue", "DateValue", "IntegerValue"]; +const CURSOR_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; +const BOUND_SENTINEL: &str = "BoundSentinel"; +const CURSOR_ERRORS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; +const CURSOR_ALGEBRA_ERROR: &str = "CursorAlgebraError"; + +/// A cursor scalar from `raw`: typed values unchanged, text read as an integer, else a timestamp. +pub(crate) fn inferred<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = raw.py(); + for class in SCALAR_CLASSES { + if raw.is_instance(&imported(py, NATIVE_MODULE, class)?)? { + return Ok(raw.clone()); + } + } + let text: Bound<'py, PyString> = raw.str()?; + match parse_value(text.as_any(), &PyString::new(py, "integer").into_any()) { + Ok(value) => Ok(value), + Err(error) + if error.is_instance( + py, + &imported(py, CURSOR_ERRORS_MODULE, CURSOR_ALGEBRA_ERROR)?, + ) => + { + parse_value(text.as_any(), &PyString::new(py, "timestamp").into_any()) + } + Err(error) => Err(error), + } +} + +/// `inferred(raw)`, keeping `None`. +pub(crate) fn optional_inferred<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + if raw.is_none() { + return Ok(raw.clone()); + } + inferred(raw) +} + +/// A cursor bound: a sentinel or its token text as the sentinel, else an inferred scalar. +fn bound<'py>(raw: &Bound<'py, PyAny>) -> PyResult> { + let sentinel: Bound<'py, PyAny> = imported(raw.py(), CURSOR_TYPES_MODULE, BOUND_SENTINEL)?; + if raw.is_instance(&sentinel)? { + return Ok(raw.clone()); + } + for member in sentinel.try_iter()? { + let member: Bound<'py, PyAny> = member?; + if raw.eq(member.getattr("value")?)? { + return Ok(member); + } + } + inferred(raw) +} + +/// `values` with `convert` applied at each of `indexes`. +fn converted( + py: Python<'_>, + mut values: Vec>, + (indexes, convert): (&[usize], Conversion), +) -> PyResult>> { + for index in indexes { + let value: Bound<'_, PyAny> = convert(values[*index].bind(py))?; + values[*index] = value.unbind(); + } + Ok(values) +} + +/// `values` with the items at each of `indexes` read as a tuple of cursor scalars. +fn converted_tuples( + py: Python<'_>, + mut values: Vec>, + indexes: &[usize], +) -> PyResult>> { + for index in indexes { + let items: Vec> = values[*index] + .bind(py) + .try_iter()? + .map(|item| inferred(&item?)) + .collect::>()?; + values[*index] = PyTuple::new(py, items)?.into_any().unbind(); + } + Ok(values) +} + +/// `ModelCursorSnapshot`: watermark values parsed as cursor scalars. +pub(crate) fn finish_cursor_snapshot( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let mut values: Vec> = converted(py, values, (&[0, 3, 4], optional_inferred))?; + values = converted_tuples(py, values, &[1, 2, 11, 12, 14])?; + let pairs: Vec> = values[13] + .bind(py) + .try_iter()? + .map(|pair| { + let pair: Bound<'_, PyAny> = pair?; + PyTuple::new( + py, + [ + optional_inferred(&pair.get_item(0)?)?, + optional_inferred(&pair.get_item(1)?)?, + ], + ) + }) + .collect::>()?; + values[13] = PyTuple::new(py, pairs)?.into_any().unbind(); + let mode: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "CursorWatermarkMode")?.call1((values[10].bind(py),))?; + values[10] = mode.unbind(); + Ok(values) +} + +/// `ModelCursorSnapshot.watermarks_available`: whether every physical watermark was read. +pub(crate) fn snapshot_watermarks_available(instance: &Bound<'_, PyAny>) -> PyResult> { + let available: bool = !instance + .getattr("unavailable_watermark_tags")? + .is_truthy()?; + Ok(pyo3::types::PyBool::new(instance.py(), available) + .to_owned() + .into_any() + .unbind()) +} + +/// `CursorBounds`: start and end read as bounds. +pub(crate) fn finish_cursor_bounds( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + converted(py, values, (&[0, 1], bound)) +} + +/// `CursorInputEvidence`: an optional minimum and a maximum scalar. +pub(crate) fn finish_input_evidence( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let values: Vec> = converted(py, values, (&[2], optional_inferred))?; + converted(py, values, (&[3], inferred)) +} + +/// `FutureCursorSafetyEvidence`: its seven cursor values read as scalars. +pub(crate) fn finish_future_evidence( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + converted(py, values, (&[2, 3, 4, 5, 6, 7, 8], inferred)) +} + +/// `MaximumStartSafetyEvidence`: its cursor values read as scalars. +pub(crate) fn finish_maximum_evidence( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let values: Vec> = converted(py, values, (&[2, 3, 5, 6], inferred))?; + converted(py, values, (&[4], optional_inferred)) +} + +/// `CursorInputRelation`: its grain as a `CursorGrain` and its terminal bounds as scalars. +pub(crate) fn finish_input_relation( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + if !values[2].bind(py).is_none() { + let grain: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "CursorGrain")?.call1((values[2].bind(py),))?; + values[2] = grain.unbind(); + } + converted(py, values, (&[5, 6], optional_inferred)) +} + +/// `CursorInputRelation.is_runtime_owned`: whether bounds are read after upstream execution. +pub(crate) fn input_relation_runtime_owned(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(instance.getattr("is_runtime_produced")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs new file mode 100644 index 0000000000..b2bdc75cf3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs @@ -0,0 +1,73 @@ +//! Computed attributes of the natively hosted discovery and pipeline records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; + +use crate::bindings::_helpers::graph::views::lineage_graph_views; + +/// SQL scalar type labels by logical value kind. +const SCALAR_TYPE_LABELS: [(&str, &str); 6] = [ + ("string", "VARCHAR"), + ("integer", "INTEGER"), + ("boolean", "BOOLEAN"), + ("float", "FLOAT"), + ("decimal", "DECIMAL"), + ("null", "NULL"), +]; + +/// `ConstantDeclaration.logical_type`: the normalized logical type. +pub(crate) fn constant_logical_type(declaration: &Bound<'_, PyAny>) -> PyResult> { + Ok(declaration + .getattr("value")? + .getattr("logical_type")? + .unbind()) +} + +/// `ConstantDeclaration.scalar_type`: the SQL scalar type label, or the upper-cased logical type. +pub(crate) fn constant_scalar_type(declaration: &Bound<'_, PyAny>) -> PyResult> { + let value: Bound<'_, PyAny> = declaration.getattr("value")?; + let kind: String = value.getattr("kind")?.str()?.extract()?; + if let Some((_, label)) = SCALAR_TYPE_LABELS.iter().find(|(name, _)| *name == kind) { + return label.into_py_any(declaration.py()); + } + let display: String = value + .getattr("logical_type")? + .getattr("display_name")? + .extract()?; + display.to_uppercase().into_py_any(declaration.py()) +} + +/// `ProjectGraph.lineage_views`: lineage edges and selector indexes, converted once. +pub(crate) fn graph_lineage_views(graph: &Bound<'_, PyAny>) -> PyResult> { + let native = graph.getattr("native")?; + Ok(lineage_graph_views(graph.py(), native.extract()?)?.unbind()) +} + +fn view(graph: &Bound<'_, PyAny>, name: &str) -> PyResult> { + Ok(graph.getattr("lineage_views")?.getattr(name)?.unbind()) +} + +/// `ProjectGraph.upstream_deps`. +pub(crate) fn graph_upstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "upstream_deps") +} + +/// `ProjectGraph.downstream_deps`. +pub(crate) fn graph_downstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "downstream_deps") +} + +/// `ProjectGraph.tag_index`. +pub(crate) fn graph_tag_index(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "tag_index") +} + +/// `ProjectGraph.path_index`. +pub(crate) fn graph_path_index(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "path_index") +} + +/// `ProjectGraph.all_keys`. +pub(crate) fn graph_all_keys(graph: &Bound<'_, PyAny>) -> PyResult> { + view(graph, "all_keys") +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs new file mode 100644 index 0000000000..c0aa28ca6d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/durations.rs @@ -0,0 +1,147 @@ +//! Calendar arithmetic and validation of the natively hosted cursor algebra records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::cursor::durations::{ + parse_cursor_duration, shift_by_cursor_duration, +}; +use crate::bindings::_helpers::cursor::intervals::interval_validation; + +/// `(field, unit suffix)` of each duration part, in declaration order. +const DURATION_UNITS: [(&str, &str); 6] = [ + ("years", "y"), + ("months", "mo"), + ("days", "d"), + ("hours", "h"), + ("minutes", "m"), + ("seconds", "s"), +]; +const MONTHS_PER_YEAR: i64 = 12; +const SECONDS_PER_DAY: i64 = 86_400; +const SECONDS_PER_HOUR: i64 = 3_600; +const SECONDS_PER_MINUTE: i64 = 60; + +/// The one positional or keyword argument `name` of a native method call. +fn argument<'py>( + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, + name: &str, +) -> PyResult> { + if let Some(kwargs) = kwargs + && let Some(value) = kwargs.get_item(name)? + { + return Ok(value); + } + args.get_item(0) +} + +fn part(duration: &Bound<'_, PyAny>, name: &str) -> PyResult { + duration.getattr(name)?.extract() +} + +fn total_months(duration: &Bound<'_, PyAny>) -> PyResult { + Ok(part(duration, "years")? * MONTHS_PER_YEAR + part(duration, "months")?) +} + +fn fixed_seconds(duration: &Bound<'_, PyAny>) -> PyResult { + Ok(part(duration, "days")? * SECONDS_PER_DAY + + part(duration, "hours")? * SECONDS_PER_HOUR + + part(duration, "minutes")? * SECONDS_PER_MINUTE + + part(duration, "seconds")?) +} + +fn units<'py>(duration: &Bound<'py, PyAny>) -> PyResult> { + let mut used: Vec<&str> = Vec::new(); + for (field, unit) in DURATION_UNITS { + if part(duration, field)? != 0 { + used.push(unit); + } + } + PyFrozenSet::new(duration.py(), used) +} + +/// `Duration.parse(value)`: the duration a text like `1y6mo` or `30d` spells, or `None`. +pub(crate) fn duration_parse( + class: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = class.py(); + let value: String = argument(args, kwargs, "value")?.extract()?; + let Some(parts) = parse_cursor_duration(py, &value)? else { + return Ok(py.None()); + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + for ((field, _), amount) in DURATION_UNITS.iter().zip(parts.iter()) { + keywords.set_item(*field, amount)?; + } + Ok(class.call((), Some(&keywords))?.unbind()) +} + +/// `Duration.is_zero`. +pub(crate) fn duration_is_zero(instance: &Bound<'_, PyAny>) -> PyResult> { + (total_months(instance)? == 0 && fixed_seconds(instance)? == 0).into_py_any(instance.py()) +} + +/// `Duration.units`: the unit suffixes with non-zero values. +pub(crate) fn duration_units(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(units(instance)?.into_any().unbind()) +} + +/// `Duration.has_calendar_component`: whether years or months are used. +pub(crate) fn duration_has_calendar_component(instance: &Bound<'_, PyAny>) -> PyResult> { + (total_months(instance)? != 0).into_py_any(instance.py()) +} + +/// `Duration.fixed_seconds`: days and below as whole seconds. +pub(crate) fn duration_fixed_seconds(instance: &Bound<'_, PyAny>) -> PyResult> { + fixed_seconds(instance)?.into_py_any(instance.py()) +} + +/// `Duration.total_months`: years folded into months. +pub(crate) fn duration_total_months(instance: &Bound<'_, PyAny>) -> PyResult> { + total_months(instance)?.into_py_any(instance.py()) +} + +/// `Duration.is_single_unit_in(allowed_units)`. +pub(crate) fn duration_is_single_unit_in( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let allowed: Bound<'_, PyAny> = argument(args, kwargs, "allowed_units")?; + let used: Bound<'_, PyFrozenSet> = units(instance)?; + let single: bool = used.len()? == 1 && used.as_any().le(&allowed)?; + single.into_py_any(instance.py()) +} + +/// `Duration.add_to(moment)`. +pub(crate) fn duration_add_to( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let moment: Bound<'_, PyAny> = argument(args, kwargs, "moment")?; + Ok(shift_by_cursor_duration(&moment, instance, 1)?.unbind()) +} + +/// `Duration.subtract_from(moment)`. +pub(crate) fn duration_subtract_from( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let moment: Bound<'_, PyAny> = argument(args, kwargs, "moment")?; + Ok(shift_by_cursor_duration(&moment, instance, -1)?.unbind()) +} + +/// `AlignedInterval`: its start, end and grain must form a valid aligned interval. +pub(crate) fn finish_aligned_interval( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + interval_validation(values[0].bind(py), values[1].bind(py), values[2].bind(py))?; + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs new file mode 100644 index 0000000000..46314aded6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/error_inits.rs @@ -0,0 +1,174 @@ +//! Constructors and derived copies of the natively hosted expected-error classes. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; + +const HELP_FIELD: &str = "help"; +const LOCATION_FIELD: &str = "location"; +/// `MacroArgumentError` keyword fields, in constructor order. +const MACRO_ARGUMENT_FIELDS: [&str; 7] = [ + "detail", + "help", + "macro_name", + "file_path", + "offset", + "relative_position", + "location", +]; + +/// Set the expected-error attributes every native error carries. +fn initialized( + error: &Bound<'_, PyAny>, + message: &Bound<'_, PyAny>, + help: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = error.py(); + error.setattr("args", PyTuple::new(py, [message])?)?; + error.setattr("message", message)?; + if let Ok(code) = error.get_type().getattr("code") { + error.setattr("code", code)?; + } + error.setattr("help", help) +} + +/// A keyword argument, or `None` when absent. +fn keyword<'py>( + py: Python<'py>, + kwargs: Option<&Bound<'py, PyDict>>, + name: &str, +) -> PyResult> { + let value: Option> = match kwargs { + Some(kwargs) => kwargs.get_item(name)?, + None => None, + }; + Ok(value.unwrap_or_else(|| py.None().into_bound(py))) +} + +/// `MacroArgumentError(detail=..., help=..., macro_name=..., file_path=..., offset=..., relative_position=..., location=None)`. +pub(crate) fn macro_argument_error_init( + error: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = error.py(); + let field = |name: &str| keyword(py, kwargs, name); + let location: Bound<'_, PyAny> = field("location")?; + let file_path: String = field("file_path")?.str()?.extract()?; + let macro_name: String = field("macro_name")?.str()?.extract()?; + let detail: String = field("detail")?.str()?.extract()?; + let (line, column): (Bound<'_, PyAny>, Bound<'_, PyAny>) = if location.is_none() { + let position: Bound<'_, PyAny> = field("relative_position")?; + (position.get_item(0)?, position.get_item(1)?) + } else { + (location.getattr("line")?, location.getattr("column")?) + }; + let (scope, suffix): (String, String) = if location.is_none() { + ( + format!("in '{file_path}'"), + format!(" of the '@{macro_name}' arguments"), + ) + } else { + ( + format!("of '@{macro_name}' in '{file_path}'"), + String::new(), + ) + }; + let message: String = format!( + "Macro arguments {scope} {detail} at line {}, column {}{suffix}", + line.str()?, + column.str()? + ); + initialized(error, PyString::new(py, &message).as_any(), &field("help")?)?; + for name in MACRO_ARGUMENT_FIELDS { + if name != HELP_FIELD { + error.setattr(name, field(name)?)?; + } + } + Ok(py.None()) +} + +/// A copy of `error` with `changes` applied to its constructor fields. +fn macro_argument_copy<'py>( + error: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = error.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for name in MACRO_ARGUMENT_FIELDS { + if name == LOCATION_FIELD { + continue; + } + keywords.set_item(name, error.getattr(name)?)?; + } + let help: Bound<'py, PyAny> = error.getattr("help")?; + if !help.is_truthy()? { + keywords.set_item("help", "")?; + } + for (name, value) in changes { + keywords.set_item(*name, value)?; + } + error.get_type().call((), Some(&keywords)) +} + +/// `MacroArgumentError.shifted(by)`: the same error, its offset moved into the enclosing SQL. +pub(crate) fn macro_argument_error_shifted( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let by: Bound<'_, PyAny> = call_argument((args, kwargs), "by", 0)?; + let offset: Bound<'_, PyAny> = error.getattr("offset")?.add(by)?; + Ok(macro_argument_copy(error, &[("offset", offset)])?.unbind()) +} + +/// `MacroArgumentError.located(location)`: the same error at its authored file location. +pub(crate) fn macro_argument_error_located( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let location: Bound<'_, PyAny> = call_argument((args, kwargs), "location", 0)?; + Ok(macro_argument_copy(error, &[("location", location)])?.unbind()) +} + +/// `SqlTestReferenceError(message, *, location, help)`: the message prefixed with its location. +pub(crate) fn sql_test_reference_error_init( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = error.py(); + let message: Bound<'_, PyAny> = call_argument((args, kwargs), "message", 0)?; + let location: Bound<'_, PyAny> = keyword(py, kwargs, "location")?; + let path: Bound<'_, PyAny> = location.getattr("path")?.call_method0("as_posix")?; + let text: String = format!( + "{}:{}:{}: {}", + path.str()?, + location.getattr("line")?.str()?, + location.getattr("column")?.str()?, + message.str()? + ); + initialized( + error, + PyString::new(py, &text).as_any(), + &keyword(py, kwargs, "help")?, + )?; + error.setattr("location", location)?; + Ok(py.None()) +} + +/// `RulesConfigError(message, *, help)`. +pub(crate) fn rules_config_error_init( + error: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = error.py(); + let message: Bound<'_, PyAny> = call_argument((args, kwargs), "message", 0)?; + error.setattr("args", PyTuple::new(py, [&message])?)?; + error.setattr("message", message)?; + error.setattr("help", keyword(py, kwargs, "help")?)?; + Ok(py.None()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs new file mode 100644 index 0000000000..7264226e08 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/freshness.rs @@ -0,0 +1,40 @@ +//! Validation and identity of the natively hosted source freshness state record. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::freshness::observation::require_aware_utc_datetime; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// `SourceFreshnessRecord.observed_at` position. +const OBSERVED_AT_FIELD: usize = 9; +const IDENTITY_FIELDS: [&str; 4] = [ + "source_name", + "target_database", + "target_schema", + "target_name", +]; + +/// `SourceFreshnessRecord`: its observed time made aware UTC, a naive time an error. +pub(crate) fn finish_freshness_record( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let observed_at: Bound<'_, PyAny> = + require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; + values[OBSERVED_AT_FIELD] = observed_at.unbind(); + Ok(values) +} + +/// `SourceFreshnessRecord.identity`: the source and target the observation belongs to. +pub(crate) fn freshness_record_identity(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + for name in IDENTITY_FIELDS { + pyo3::types::PyDictMethods::set_item(&keywords, name, instance.getattr(name)?)?; + } + Ok(imported(py, NATIVE_MODULE, "SourceFreshnessIdentity")? + .call((), Some(&keywords))? + .unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs new file mode 100644 index 0000000000..dd5e7855fb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lexical.rs @@ -0,0 +1,103 @@ +//! Validation and derived views of the natively hosted `SqlLexicalSyntax` record. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple, PyTupleMethods}; + +use sqlbuild_sqltext::sql_scan::constants::SUPPORTED_LINE_COMMENT_PREFIXES; +use sqlbuild_sqltext::sql_scan::main::lexical_syntax_key::lexical_syntax_key; +use sqlbuild_sqltext::sql_scan::main::reads_differently_from_generic::reads_differently_from_generic; +use sqlbuild_sqltext::sql_scan::main::unsupported_line_comment_prefixes::unsupported_line_comment_prefixes; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// `SqlLexicalSyntax.line_comment_prefixes` position. +const LINE_COMMENT_PREFIXES_FIELD: usize = 5; + +/// The sorted texts of a set of strings. +fn sorted_texts(values: &Bound<'_, PyAny>) -> PyResult> { + let mut texts: Vec = values + .try_iter()? + .map(|value| value?.extract::()) + .collect::>()?; + texts.sort_unstable(); + Ok(texts) +} + +/// The lexical rules a `SqlLexicalSyntax` record holds. +fn record_syntax(instance: &Bound<'_, PyAny>) -> PyResult { + let flag = |name: &str| -> PyResult { instance.getattr(name)?.is_truthy() }; + Ok(LexicalSyntax { + backslash_escape_quotes: sorted_texts(&instance.getattr("backslash_escape_quotes")?)?, + escape_string_prefix: flag("escape_string_prefix")?, + raw_string_prefix: flag("raw_string_prefix")?, + triple_quoted_strings: flag("triple_quoted_strings")?, + nested_block_comments: flag("nested_block_comments")?, + line_comment_prefixes: sorted_texts(&instance.getattr("line_comment_prefixes")?)?, + }) +} + +/// `SqlLexicalSyntax`: reject line comment prefixes the scanners cannot read. +pub(crate) fn finish_lexical_syntax( + py: Python<'_>, + values: Vec>, +) -> PyResult>> { + let prefixes: Vec = sorted_texts(values[LINE_COMMENT_PREFIXES_FIELD].bind(py))?; + let unsupported: Vec = unsupported_line_comment_prefixes(&prefixes); + if unsupported.is_empty() { + return Ok(values); + } + let supported: Vec<&str> = SUPPORTED_LINE_COMMENT_PREFIXES.to_vec(); + let message: String = format!( + "SQL line comment prefixes {} are not supported; SQLBuild scans only {}", + PyList::new(py, &unsupported)?.repr()?, + PyList::new(py, &supported)?.repr()? + ); + let error: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "UnsupportedLexicalSyntaxError")?.call1((message,))?; + Err(PyErr::from_value(error)) +} + +/// `SqlLexicalSyntax.cache_key`: a process-stable text key for these rules. +pub(crate) fn lexical_cache_key(instance: &Bound<'_, PyAny>) -> PyResult> { + lexical_syntax_key(&record_syntax(instance)?).into_py_any(instance.py()) +} + +/// `SqlLexicalSyntax.native_mapping`: these rules as the mapping native scanners read. +pub(crate) fn lexical_native_mapping(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let syntax: LexicalSyntax = record_syntax(instance)?; + let mapping: Bound<'_, PyDict> = PyDict::new(py); + mapping.set_item( + "backslash_escape_quotes", + PyList::new(py, &syntax.backslash_escape_quotes)?, + )?; + mapping.set_item("escape_string_prefix", syntax.escape_string_prefix)?; + mapping.set_item("raw_string_prefix", syntax.raw_string_prefix)?; + mapping.set_item("triple_quoted_strings", syntax.triple_quoted_strings)?; + mapping.set_item("nested_block_comments", syntax.nested_block_comments)?; + mapping.set_item( + "line_comment_prefixes", + PyList::new(py, &syntax.line_comment_prefixes)?, + )?; + Ok(mapping.into_any().unbind()) +} + +/// `SqlLexicalSyntax.reads_differently_from_generic(sql)`. +pub(crate) fn lexical_reads_differently( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let keyword: Option> = match kwargs { + Some(kwargs) => kwargs.get_item("sql")?, + None => None, + }; + let sql: String = match keyword { + Some(sql) => sql.extract()?, + None => args.get_item(0)?.extract()?, + }; + reads_differently_from_generic(&record_syntax(instance)?, &sql).into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs new file mode 100644 index 0000000000..21d5d48e4a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs @@ -0,0 +1,190 @@ +//! Sequence behaviour of the natively hosted `CompactLineageFacts` record. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PySlice, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// `ColumnTransformKind` members by native transform code. +const TRANSFORM_KINDS: [&str; 6] = [ + "DIRECT", + "CAST", + "EXPRESSION", + "AGGREGATION", + "STAR", + "CONSTANT", +]; +/// `ColumnLineageConfidence` members by native confidence code. +const CONFIDENCES: [&str; 3] = ["UNKNOWN", "HIGH", "MEDIUM"]; + +/// The `ColumnTransformKind` member of a native transform code. +pub(crate) fn transform_kind_member(py: Python<'_>, code: usize) -> PyResult> { + coded_member(py, ("ColumnTransformKind", &TRANSFORM_KINDS), code) +} + +/// The `ColumnLineageConfidence` member of a native confidence code. +pub(crate) fn confidence_member(py: Python<'_>, code: usize) -> PyResult> { + coded_member(py, ("ColumnLineageConfidence", &CONFIDENCES), code) +} + +/// The enum member at `code` of `members`. +fn coded_member( + py: Python<'_>, + (class, members): (&str, &[&str]), + code: usize, +) -> PyResult> { + let Some(member) = members.get(code) else { + return Err(PyIndexError::new_err("tuple index out of range")); + }; + Ok(imported(py, NATIVE_MODULE, class)? + .getattr(*member)? + .unbind()) +} + +/// `CompactLineageFacts.resource_name(index)`: the model's name for a native relation index. +fn resource_name<'py>( + facts: &Bound<'py, PyAny>, + index: &Bound<'py, PyAny>, +) -> PyResult> { + let names: Bound<'py, PyAny> = facts.getattr("resource_names")?; + let name: Bound<'py, PyAny> = names.call_method1("get", (index,))?; + if !name.is_none() { + return Ok(name); + } + let pooled: Bound<'py, PyAny> = facts + .getattr("resource_name_indexes")? + .call_method1("get", (index, index))?; + facts.getattr("string_pool")?.get_item(pooled) +} + +/// `CompactLineageFacts.resource_name(index)`. +pub(crate) fn facts_resource_name( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let index: Bound<'_, PyAny> = call_argument((args, kwargs), "index", 0)?; + Ok(resource_name(instance, &index)?.unbind()) +} + +/// One compact row projected to a `CompiledLineageColumnFact`. +fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = facts.py(); + let pool: Bound<'_, PyAny> = facts.getattr("string_pool")?; + let (name_index, transform_code, confidence_code, sources): ( + Bound<'_, PyAny>, + usize, + usize, + Bound<'_, PyAny>, + ) = row.extract()?; + let source_class: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "CompiledLineageSourceFact")?; + let mut upstream: Vec> = Vec::new(); + for source in sources.try_iter()? { + let source: Bound<'_, PyTuple> = source?.cast_into::()?; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("resource_type", pool.get_item(source.get_item(0)?)?)?; + keywords.set_item("resource_name", resource_name(facts, &source.get_item(1)?)?)?; + keywords.set_item("column_name", pool.get_item(source.get_item(2)?)?)?; + upstream.push(source_class.call((), Some(&keywords))?); + } + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("output_column", pool.get_item(name_index)?)?; + keywords.set_item("upstream_columns", PyTuple::new(py, upstream)?)?; + keywords.set_item( + "transform_kind", + coded_member( + py, + ("ColumnTransformKind", &TRANSFORM_KINDS), + transform_code, + )?, + )?; + keywords.set_item( + "confidence", + coded_member( + py, + ("ColumnLineageConfidence", &CONFIDENCES), + confidence_code, + )?, + )?; + Ok(imported(py, NATIVE_MODULE, "CompiledLineageColumnFact")? + .call((), Some(&keywords))? + .unbind()) +} + +/// `CompactLineageFacts.facts`: every row projected once, in order. +pub(crate) fn facts_projected(instance: &Bound<'_, PyAny>) -> PyResult> { + let projected: Vec> = instance + .getattr("rows")? + .try_iter()? + .map(|row| project_row(instance, &row?)) + .collect::>()?; + Ok(PyTuple::new(instance.py(), projected)?.into_any().unbind()) +} + +/// `len(facts)`. +pub(crate) fn facts_len( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + instance.getattr("rows")?.len()?.into_py_any(instance.py()) +} + +/// `facts[index]`: one fact, or a tuple of facts for a slice. +pub(crate) fn facts_getitem( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let index: Bound<'_, PyAny> = args.get_item(0)?; + let projected: Bound<'_, PyAny> = instance.getattr("facts")?; + if index.is_instance_of::() { + return Ok(projected.get_item(index)?.unbind()); + } + let length: isize = isize::try_from(projected.len()?)?; + let requested: isize = index.extract()?; + let normalized: isize = if requested >= 0 { + requested + } else { + length + requested + }; + if normalized < 0 || normalized >= length { + return Err(PyIndexError::new_err(index.unbind())); + } + Ok(projected.get_item(normalized)?.unbind()) +} + +/// `iter(facts)`. +pub(crate) fn facts_iter( + instance: &Bound<'_, PyAny>, + _args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + Ok(instance.getattr("facts")?.try_iter()?.into_any().unbind()) +} + +/// `facts == other`: item-wise equality with any sequence. +pub(crate) fn facts_eq( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let other: Bound<'_, PyAny> = args.get_item(0)?; + let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; + if !other.is_instance(&sequence)? { + return Ok(py.NotImplemented()); + } + let mine: Bound<'_, PyAny> = instance.getattr("facts")?; + let theirs: Bound<'_, PyTuple> = PyTuple::new( + py, + other + .try_iter()? + .collect::>>>()?, + )?; + mine.eq(theirs)?.into_py_any(py) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs new file mode 100644 index 0000000000..2ba6bba172 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -0,0 +1,197 @@ +//! Behaviour of the natively hosted macro records and the compiled project's lineage graph. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyList, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::graph::project_graph::{ResourceRow, project_graph}; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::record_behaviours::compiled::compile_input_error; +use crate::bindings::_helpers::render::declaration_values::{ + constant_declaration_sql, render_enum_member_value, +}; + +/// The parameter name through which a macro receives its context. +const MACRO_CONTEXT_PARAMETER_NAME: &str = "ctx"; +/// Parameter kinds that can receive the context positionally. +const POSITIONAL_KINDS: [&str; 2] = ["POSITIONAL_ONLY", "POSITIONAL_OR_KEYWORD"]; + +/// `LoadedMacro.injects_context`: whether the first positional parameter is the macro context. +pub(crate) fn macro_injects_context(instance: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = instance.py(); + let inspect: Bound<'_, PyAny> = py.import("inspect")?.into_any(); + let parameters: Bound<'_, PyTuple> = PyTuple::new( + py, + inspect + .getattr("signature")? + .call1((instance.getattr("function")?,))? + .getattr("parameters")? + .call_method0("values")? + .try_iter()? + .collect::>>>()?, + )?; + let Ok(first) = parameters.get_item(0) else { + return false.into_py_any(py); + }; + let kind: Bound<'_, PyAny> = first.getattr("kind")?; + let mut positional: bool = false; + for name in POSITIONAL_KINDS { + positional |= kind.eq(inspect.getattr("Parameter")?.getattr(name)?)?; + } + let named: bool = first.getattr("name")?.eq(MACRO_CONTEXT_PARAMETER_NAME)?; + (positional && named).into_py_any(py) +} + +/// `MacroContext.render_constant(name)`: one visible constant as an adapter-safe SQL value. +pub(crate) fn context_render_constant( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + let name: Bound<'_, PyAny> = call_argument((args, kwargs), "name", 0)?; + let value_renderer: Bound<'_, PyAny> = instance.getattr("_value_renderer")?; + if value_renderer.is_none() { + return Err(compile_input_error( + py, + "Constant SQL rendering is available only during a project macro invocation", + )?); + } + let _ = instance.getattr("constants")?.get_item(&name)?; + let declaration: Bound<'_, PyAny> = instance + .getattr("_constant_declarations")? + .call_method1("get", (&name,))?; + if declaration.is_none() { + return Err(compile_input_error( + py, + "Constant SQL rendering requires a declaration resolved for this macro invocation", + )?); + } + constant_declaration_sql( + &declaration, + &value_renderer, + (&instance.getattr("_collection_rendering")?, None), + )? + .into_py_any(py) +} + +/// `MacroContext.render_enum_member(*, enum_name, member_name)`: one member as a SQL literal. +pub(crate) fn context_render_enum_member( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + if !args.is_empty() { + return Err(pyo3::exceptions::PyTypeError::new_err( + "render_enum_member() takes keyword arguments enum_name and member_name only", + )); + } + let enum_name: Bound<'_, PyAny> = call_argument((args, kwargs), "enum_name", 0)?; + let member_name: Bound<'_, PyAny> = call_argument((args, kwargs), "member_name", 1)?; + let value: Bound<'_, PyAny> = instance + .getattr("enums")? + .get_item(enum_name)? + .get_item(member_name)?; + render_enum_member_value(&value)?.into_py_any(instance.py()) +} + +/// `(resource type, name)` of one compiled object key. +fn graph_key(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + key.getattr("resource_type")?.str()?.extract()?, + key.getattr("name")?.extract()?, + )) +} + +/// `(resource type, name)` of each compiled object key. +fn graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| graph_key(&key?)).collect() +} + +/// The texts of a list or tuple of tags; anything else has none. +fn graph_tags(value: &Bound<'_, PyAny>) -> PyResult> { + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(Vec::new()); + } + value + .try_iter()? + .map(|item| item?.str()?.extract()) + .collect() +} + +/// A resource's tags and model folder. +type ResourceTags = (Vec, Option); +/// Reads one resource's tags and model folder. +type DescribeResource = fn(&Bound<'_, PyAny>) -> PyResult; + +/// The graph rows of each resource in `resources`, with tags and folder from `describe`. +fn resource_rows( + resources: &Bound<'_, PyAny>, + describe: DescribeResource, +) -> PyResult> { + let mut rows: Vec = Vec::new(); + for resource in resources.try_iter()? { + let resource: Bound<'_, PyAny> = resource?; + let (tags, folder) = describe(&resource)?; + rows.push(( + graph_key(&resource.getattr("key")?)?, + graph_keys(&resource.getattr("deps")?)?, + tags, + folder, + )); + } + Ok(rows) +} + +/// A model's configured tags and folder. +fn model_tags(model: &Bound<'_, PyAny>) -> PyResult { + let tags: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("tags",))?; + let folder: String = model + .getattr("relative_path")? + .getattr("parent")? + .str()? + .extract()?; + Ok((graph_tags(&tags)?, Some(folder))) +} + +/// A source has neither tags nor folder. +fn source_tags(_source: &Bound<'_, PyAny>) -> PyResult { + Ok((Vec::new(), None)) +} + +/// A seed's schema entry tags. +fn seed_tags(seed: &Bound<'_, PyAny>) -> PyResult { + let tags: Vec = seed + .getattr("schema_entry")? + .getattr("tags")? + .try_iter()? + .map(|tag| tag?.extract()) + .collect::>()?; + Ok((tags, None)) +} + +/// A function's tags. +fn function_tags(function: &Bound<'_, PyAny>) -> PyResult { + let tags: Vec = function + .getattr("tags")? + .try_iter()? + .map(|tag| tag?.extract()) + .collect::>()?; + Ok((tags, None)) +} + +/// `CompiledProject.lineage_graph`: lineage edges, tags, folders and names indexed natively. +pub(crate) fn project_lineage_graph(instance: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + rows.extend(resource_rows(&instance.getattr("models")?, model_tags)?); + rows.extend(resource_rows(&instance.getattr("sources")?, source_tags)?); + rows.extend(resource_rows(&instance.getattr("seeds")?, seed_tags)?); + rows.extend(resource_rows( + &instance.getattr("functions")?, + function_tags, + )?); + project_graph(rows)?.into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs new file mode 100644 index 0000000000..86cba9622e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/migrations.rs @@ -0,0 +1,210 @@ +//! Behaviour of the natively hosted migration decisions and migration state records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; + +use sqlbuild_core::migration_values::main::column_decision_traits::column_decision_traits; +use sqlbuild_core::migration_values::main::migration_decision_traits::migration_decision_traits; +use sqlbuild_core::migration_values::main::plan_label::plan_label; +use sqlbuild_core::migration_values::models::{ColumnDecisionTraits, DecisionTraits}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +const VIEW_STATUS: &str = "OldNameViewStatus"; + +fn value(member: &Bound<'_, PyAny>) -> PyResult { + member.str()?.extract() +} + +fn model_traits(member: &Bound<'_, PyAny>) -> PyResult { + Ok(migration_decision_traits(&value(member)?)) +} + +fn column_traits(member: &Bound<'_, PyAny>) -> PyResult { + Ok(column_decision_traits(&value(member)?)) +} + +/// `MigrationDecision.moves_data`. +pub(crate) fn decision_moves_data(member: &Bound<'_, PyAny>) -> PyResult> { + model_traits(member)?.moves_data.into_py_any(member.py()) +} + +/// `MigrationDecision.blocks_build`. +pub(crate) fn decision_blocks_build(member: &Bound<'_, PyAny>) -> PyResult> { + model_traits(member)?.blocks_build.into_py_any(member.py()) +} + +/// `MigrationDecision.checks_compatibility`. +pub(crate) fn decision_checks_compatibility(member: &Bound<'_, PyAny>) -> PyResult> { + model_traits(member)? + .checks_compatibility + .into_py_any(member.py()) +} + +/// `MigrationDecision.label` and `MigrationPromotion.label`. +pub(crate) fn value_plan_label(member: &Bound<'_, PyAny>) -> PyResult> { + plan_label(&value(member)?).into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.renames`. +pub(crate) fn column_decision_renames(member: &Bound<'_, PyAny>) -> PyResult> { + column_traits(member)?.renames.into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.records_event`. +pub(crate) fn column_decision_records_event(member: &Bound<'_, PyAny>) -> PyResult> { + column_traits(member)? + .records_event + .into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.blocks_build`. +pub(crate) fn column_decision_blocks_build(member: &Bound<'_, PyAny>) -> PyResult> { + column_traits(member)?.blocks_build.into_py_any(member.py()) +} + +/// `ColumnMigrationDecision.label`. +pub(crate) fn column_decision_label(member: &Bound<'_, PyAny>) -> PyResult> { + match column_traits(member)?.label { + Some(label) => label.into_py_any(member.py()), + None => Err(pyo3::exceptions::PyKeyError::new_err(value(member)?)), + } +} + +/// `text.lower()`, keeping `None`. +fn lowered<'py>(text: &Bound<'py, PyAny>) -> PyResult> { + if text.is_none() { + return Ok(text.clone()); + } + text.call_method0("lower") +} + +/// `(database.lower() or None, (schema or "").lower(), name.lower())`. +fn relation_identity<'py>(relation: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = relation.py(); + let schema: Bound<'py, PyAny> = relation.getattr("schema")?; + let schema: Bound<'py, PyAny> = if schema.is_none() { + "".into_bound_py_any(py)? + } else { + schema.call_method0("lower")? + }; + PyTuple::new( + py, + [ + lowered(&relation.getattr("database")?)?, + schema, + relation.getattr("name")?.call_method0("lower")?, + ], + ) +} + +/// Whether two relations match, an unknown database matching any database. +fn relations_match(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult { + let left: Bound<'_, PyTuple> = relation_identity(left)?; + let right: Bound<'_, PyTuple> = relation_identity(right)?; + if !left.get_item(1)?.eq(right.get_item(1)?)? || !left.get_item(2)?.eq(right.get_item(2)?)? { + return Ok(false); + } + let (left, right) = (left.get_item(0)?, right.get_item(0)?); + Ok(left.is_none() || right.is_none() || left.eq(right)?) +} + +/// `MigrationRelation.identity`: the case-insensitive relation identity. +pub(crate) fn relation_identity_property(relation: &Bound<'_, PyAny>) -> PyResult> { + Ok(relation_identity(relation)?.into_any().unbind()) +} + +/// `MigrationRelation.matches(other)`. +pub(crate) fn relation_matches( + relation: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let other: Bound<'_, PyAny> = call_argument((args, kwargs), "other", 0)?; + relations_match(relation, &other)?.into_py_any(relation.py()) +} + +/// `MigrationEvent.mentions(relation)`: the relation is the event's origin or destination. +pub(crate) fn event_mentions( + event: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let relation: Bound<'_, PyAny> = call_argument((args, kwargs), "relation", 0)?; + let mentions: bool = relations_match(&event.getattr("origin")?, &relation)? + || relations_match(&event.getattr("destination")?, &relation)?; + mentions.into_py_any(event.py()) +} + +/// `ColumnMigrationEvent.mentions(relation=..., columns=...)`. +pub(crate) fn column_event_mentions( + event: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let relation: Bound<'_, PyAny> = call_argument((args, kwargs), "relation", 0)?; + let columns: Bound<'_, PyAny> = call_argument((args, kwargs), "columns", 1)?; + let mentions: bool = relations_match(&event.getattr("relation")?, &relation)? + && (columns.contains(event.getattr("origin_column")?.call_method0("lower")?)? + || columns.contains(event.getattr("destination_column")?.call_method0("lower")?)?); + mentions.into_py_any(event.py()) +} + +/// `ColumnMigrationEvent.renamed(origin_column=..., destination_column=...)`. +pub(crate) fn column_event_renamed( + event: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let same = |field: &str, name: &str, position: usize| -> PyResult { + let given: Bound<'_, PyAny> = call_argument((args, kwargs), name, position)?; + event + .getattr(field)? + .call_method0("lower")? + .eq(given.call_method0("lower")?) + }; + let renamed: bool = same("origin_column", "origin_column", 0)? + && same("destination_column", "destination_column", 1)?; + renamed.into_py_any(event.py()) +} + +/// `OldNameViewHistory.status(now=...)`: the current state of the move's old-name steps. +pub(crate) fn history_status( + history: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = history.py(); + let now: Bound<'_, PyAny> = call_argument((args, kwargs), "now", 0)?; + let created: Bound<'_, PyAny> = history.getattr("created")?; + let status: &str = if !history.getattr("dropped")?.is_none() { + "DROPPED" + } else if !created.is_none() { + let expires_at: Bound<'_, PyAny> = created.getattr("expires_at")?; + if !expires_at.is_none() && expires_at.le(&now)? { + "EXPIRED" + } else { + "LIVE" + } + } else if !history.getattr("archived")?.is_none() { + "PENDING_VIEW" + } else { + "PENDING_ARCHIVE" + }; + Ok(imported(py, NATIVE_MODULE, VIEW_STATUS)? + .getattr(status)? + .unbind()) +} + +/// `OldNameViewHistory.old`: the relation name the compatibility view occupies. +pub(crate) fn history_old(history: &Bound<'_, PyAny>) -> PyResult> { + Ok(history.getattr("required")?.getattr("old")?.unbind()) +} + +/// `OldNameViewHistory.new`: the relation the compatibility view reads. +pub(crate) fn history_new(history: &Bound<'_, PyAny>) -> PyResult> { + Ok(history.getattr("required")?.getattr("new")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs new file mode 100644 index 0000000000..76a8bf272e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/mod.rs @@ -0,0 +1,20 @@ +pub(crate) mod auditing; +pub(crate) mod call_arguments; +pub(crate) mod compiled; +pub(crate) mod config; +pub(crate) mod constants; +pub(crate) mod cursors; +pub(crate) mod discovery; +pub(crate) mod durations; +pub(crate) mod error_inits; +pub(crate) mod freshness; +pub(crate) mod lexical; +pub(crate) mod lineage_facts; +pub(crate) mod macros; +pub(crate) mod migrations; +pub(crate) mod planner; +pub(crate) mod python_nodes; +pub(crate) mod references; +pub(crate) mod rule_shapes; +pub(crate) mod scopes; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs new file mode 100644 index 0000000000..8c6fbbaffd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/planner.rs @@ -0,0 +1,353 @@ +//! Computed attributes of the natively hosted plan entry records and cursor overrides. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use sqlbuild_core::migration_values::main::migration_decision_traits::migration_decision_traits; +use sqlbuild_core::migration_values::main::model_migration_blocks_build::model_migration_blocks_build as blocks_build; +use sqlbuild_core::migration_values::main::storage_transition::storage_transition; +use sqlbuild_core::migration_values::models::{DecisionTraits, ModelMigrationFacts}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::scalars::parse_value; +use crate::bindings::_helpers::record_behaviours::call_arguments::call_argument; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +const DECREASING_DIRECTIONS: [&str; 2] = ["decrease", "mixed"]; +const STEP_ACTIONS: [&str; 2] = ["archive_and_view", "view_only"]; +const TABLE_BACKED: [&str; 3] = ["table", "incremental", "snapshot"]; +const ORIGIN_MISSING: &str = "origin_missing"; +const SOURCE_MISSING: &str = "source_missing"; +const DENY_POLICY: &str = "deny"; +const INCOMPATIBLE: &str = "incompatible"; +const DEFAULT_TARGET_LABEL: &str = "default"; +const PLANNER_INPUT_ERROR: &str = "PlannerInputError"; +const INTEGER_CURSOR: &str = "integer"; +/// `CursorOverrides` field positions: `start_ts`, `end_ts`, `start_int`, `end_int`, `start`, `end`. +const OVERRIDE_FIELDS: [&str; 6] = ["start_ts", "end_ts", "start_int", "end_int", "start", "end"]; +const TIMESTAMP_FLAGS: [&str; 2] = ["--start-cursor-ts", "--end-cursor-ts"]; +const INTEGER_FLAGS: [&str; 2] = ["--start-cursor-int", "--end-cursor-int"]; +const UNTYPED_FLAGS: [&str; 2] = ["--start-cursor", "--end-cursor"]; + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn flag(py: Python<'_>, value: bool) -> PyResult> { + value.into_py_any(py) +} + +/// `RetentionPlanEntry.decreases`: whether the entry lowers any live retention value. +pub(crate) fn retention_decreases(instance: &Bound<'_, PyAny>) -> PyResult> { + let direction: String = text(&instance.getattr("direction")?)?; + flag( + instance.py(), + DECREASING_DIRECTIONS.contains(&direction.as_str()), + ) +} + +/// `OldNameViewPlanEntry.runs_steps`: whether the build archives the old relation or creates the view. +pub(crate) fn old_name_runs_steps(instance: &Bound<'_, PyAny>) -> PyResult> { + let action: String = text(&instance.getattr("action")?)?; + flag(instance.py(), STEP_ACTIONS.contains(&action.as_str())) +} + +/// `PlanOutput.python_source_entries`: the source entries Python reads resolve. +pub(crate) fn plan_python_source_entries(instance: &Bound<'_, PyAny>) -> PyResult> { + for name in ["python_source_read_map", "source_read_map"] { + let entries: Bound<'_, PyAny> = instance.getattr(name)?; + if entries.is_truthy()? { + return Ok(entries.unbind()); + } + } + Ok(instance.getattr("source_map")?.unbind()) +} + +/// `MaterializationType.is_table_backed(materialized=...)`. +pub(crate) fn materialization_is_table_backed( + _class: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let materialized: Bound<'_, PyAny> = call_argument((args, kwargs), "materialized", 0)?; + let backed: bool = match materialized.downcast::() { + Ok(value) => TABLE_BACKED.contains(&value.extract::()?.as_str()), + Err(_) => false, + }; + flag(args.py(), backed) +} + +fn decision_traits(entry: &Bound<'_, PyAny>) -> PyResult<(String, DecisionTraits)> { + let decision: String = text(&entry.getattr("decision")?)?; + let traits: DecisionTraits = migration_decision_traits(&decision); + Ok((decision, traits)) +} + +fn model_facts(entry: &Bound<'_, PyAny>) -> PyResult { + let (decision, traits) = decision_traits(entry)?; + Ok(ModelMigrationFacts { + origin_missing: decision == ORIGIN_MISSING, + origin_tracked: entry.getattr("origin_tracked")?.is_truthy()?, + deny_missing_origin: text(&entry.getattr("missing_origin_policy")?)? == DENY_POLICY, + decision: traits, + incompatible: text(&entry.getattr("compatibility")?)? == INCOMPATIBLE, + }) +} + +/// `ModelMigrationPlanEntry.origin_missing`. +pub(crate) fn model_migration_origin_missing(entry: &Bound<'_, PyAny>) -> PyResult> { + flag(entry.py(), model_facts(entry)?.origin_missing) +} + +/// `ModelMigrationPlanEntry.origin_hidden`: a missing origin with build history in this target. +pub(crate) fn model_migration_origin_hidden(entry: &Bound<'_, PyAny>) -> PyResult> { + let facts: ModelMigrationFacts = model_facts(entry)?; + flag(entry.py(), facts.origin_missing && facts.origin_tracked) +} + +/// `ModelMigrationPlanEntry.storage_transition`: the table-type transition label, when known. +pub(crate) fn model_migration_storage_transition(entry: &Bound<'_, PyAny>) -> PyResult> { + let origin: bool = entry.getattr("origin_is_transient")?.is_truthy()?; + let stage: Bound<'_, PyAny> = entry.getattr("stage_is_transient")?; + let stage: Option = if stage.is_none() { + None + } else { + Some(stage.is_truthy()?) + }; + storage_transition(origin, stage).into_py_any(entry.py()) +} + +/// `ModelMigrationPlanEntry.target_label`: the target name, naming the unnamed target `default`. +pub(crate) fn model_migration_target_label(entry: &Bound<'_, PyAny>) -> PyResult> { + let target: Bound<'_, PyAny> = entry.getattr("target_name")?; + if target.is_truthy()? { + return Ok(target.unbind()); + } + DEFAULT_TARGET_LABEL.into_py_any(entry.py()) +} + +/// `ModelMigrationPlanEntry.blocks_build`: whether the migration stops a build before execution. +pub(crate) fn model_migration_blocks_build(entry: &Bound<'_, PyAny>) -> PyResult> { + flag(entry.py(), blocks_build(&model_facts(entry)?)) +} + +/// `ColumnMigrationPlanEntry.origin_missing`. +pub(crate) fn column_migration_origin_missing(entry: &Bound<'_, PyAny>) -> PyResult> { + flag( + entry.py(), + text(&entry.getattr("decision")?)? == SOURCE_MISSING, + ) +} + +/// `ColumnMigrationPlanEntry.blocks_build`. +pub(crate) fn column_migration_blocks_build(entry: &Bound<'_, PyAny>) -> PyResult> { + let decision: Bound<'_, PyAny> = entry.getattr("decision")?; + let blocks: bool = if text(&decision)? == SOURCE_MISSING { + text(&entry.getattr("missing_origin_policy")?)? == DENY_POLICY + } else { + decision.getattr("blocks_build")?.is_truthy()? + }; + flag(entry.py(), blocks) +} + +/// The `PlannerInputError` carrying `message`. +fn planner_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, NATIVE_MODULE, PLANNER_INPUT_ERROR).and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// A `--*-cursor-ts` value: text checked as an ISO timestamp and parsed, others unchanged. +fn override_timestamp<'py>( + flag_name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() || !value.is_instance_of::() { + return Ok(value.clone()); + } + let datetime: Bound<'py, PyAny> = imported(py, "datetime", "datetime")?; + if let Err(error) = datetime.call_method1("fromisoformat", (value,)) { + let shown: String = error.value(py).str()?.extract()?; + return Err(planner_error( + py, + format!( + "{flag_name} value '{}' is not a valid ISO timestamp: {shown}", + text(value)? + ), + )); + } + parse_value(value, &PyString::new(py, "timestamp").into_any()) +} + +/// A `--*-cursor-int` value: text checked as a whole number and parsed, others unchanged. +fn override_integer<'py>( + flag_name: &str, + value: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() || !value.is_instance_of::() { + return Ok(value.clone()); + } + let shown: String = text(value)?; + let invalid = || { + planner_error( + py, + format!("{flag_name} value '{shown}' is not a valid integer"), + ) + }; + let Ok(decimal) = imported(py, "decimal", "Decimal")?.call1((value,)) else { + return Err(invalid()); + }; + let whole: Bound<'py, PyAny> = py.get_type::().call1((&decimal,))?; + if !decimal.eq(&whole)? { + return Err(planner_error( + py, + format!("{flag_name} value '{shown}' is not a whole number"), + )); + } + let parsed: Bound<'py, PyAny> = + parse_value(value, &PyString::new(py, INTEGER_CURSOR).into_any())?; + if parsed.is_instance(&imported(py, NATIVE_MODULE, "IntegerValue")?)? { + return Ok(parsed); + } + Err(invalid()) +} + +/// `CursorOverrides`: typed flag values parsed, untyped `start`/`end` kept as text. +pub(crate) fn finish_cursor_overrides( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + for (index, flag_name) in TIMESTAMP_FLAGS.iter().enumerate() { + let parsed: Bound<'_, PyAny> = override_timestamp(flag_name, values[index].bind(py))?; + values[index] = parsed.unbind(); + } + for (offset, flag_name) in INTEGER_FLAGS.iter().enumerate() { + let parsed: Bound<'_, PyAny> = override_integer(flag_name, values[2 + offset].bind(py))?; + values[2 + offset] = parsed.unbind(); + } + Ok(values) +} + +/// `CursorOverrides.has_untyped`: whether `--start-cursor` or `--end-cursor` awaits a type. +pub(crate) fn overrides_have_untyped(instance: &Bound<'_, PyAny>) -> PyResult> { + let untyped: bool = + !instance.getattr("start")?.is_none() || !instance.getattr("end")?.is_none(); + flag(instance.py(), untyped) +} + +/// `parsed or fallback`. +fn either<'py>( + parsed: Bound<'py, PyAny>, + fallback: Bound<'py, PyAny>, +) -> PyResult> { + if parsed.is_truthy()? { + return Ok(parsed); + } + Ok(fallback) +} + +/// `CursorOverrides.typed_as(cursor_type=...)`: `--start-cursor`/`--end-cursor` read as that type. +pub(crate) fn overrides_typed_as( + instance: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = instance.py(); + if !overrides_have_untyped(instance)?.bind(py).is_truthy()? { + return Ok(instance.clone().unbind()); + } + let cursor_type: String = text(&call_argument((args, kwargs), "cursor_type", 0)?)?; + let field = |name: &str| instance.getattr(name); + let (start, end) = (field(OVERRIDE_FIELDS[4])?, field(OVERRIDE_FIELDS[5])?); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + if cursor_type == INTEGER_CURSOR { + keywords.set_item("start_ts", field("start_ts")?)?; + keywords.set_item("end_ts", field("end_ts")?)?; + keywords.set_item( + "start_int", + either( + override_integer(UNTYPED_FLAGS[0], &start)?, + field("start_int")?, + )?, + )?; + keywords.set_item( + "end_int", + either(override_integer(UNTYPED_FLAGS[1], &end)?, field("end_int")?)?, + )?; + } else { + keywords.set_item( + "start_ts", + either( + override_timestamp(UNTYPED_FLAGS[0], &start)?, + field("start_ts")?, + )?, + )?; + keywords.set_item( + "end_ts", + either( + override_timestamp(UNTYPED_FLAGS[1], &end)?, + field("end_ts")?, + )?, + )?; + keywords.set_item("start_int", field("start_int")?)?; + keywords.set_item("end_int", field("end_int")?)?; + } + Ok(instance.get_type().call((), Some(&keywords))?.unbind()) +} + +/// `ModelPlanEntry.backfill`'s default: a forward-only backfill. +pub(crate) fn forward_only_backfill(py: Python<'_>) -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item( + "action", + imported(py, NATIVE_MODULE, "BackfillAction")?.getattr("FORWARD_ONLY")?, + )?; + imported(py, NATIVE_MODULE, "BackfillResult")?.call((), Some(&keywords)) +} + +/// `ScenarioArtifactIdentity.kind` coerced to `ScenarioArtifactKind`. +pub(crate) fn finish_scenario_artifact_kind( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + if let Some(kind) = values.first_mut() { + *kind = imported(py, NATIVE_MODULE, "ScenarioArtifactKind")? + .call1((kind.bind(py),))? + .unbind(); + } + Ok(values) +} + +/// `ColumnRenameHint.message`: the one-line hint shown beside the added column. +pub(crate) fn rename_hint_message(hint: &Bound<'_, PyAny>) -> PyResult> { + let relation: &str = if hint.getattr("identical")?.is_truthy()? { + "same expression as" + } else { + "similar to" + }; + let candidates: Vec = hint + .getattr("candidate_columns")? + .try_iter()? + .map(|candidate| Ok(candidate?.str()?.to_string())) + .collect::>()?; + let origin: &str = match candidates.as_slice() { + [only] => only, + _ => "", + }; + format!( + "{relation} {}; if this is a rename, add {} (migrate_from {origin})", + candidates.join(", "), + hint.getattr("added_column")?.str()? + ) + .into_py_any(hint.py()) +} + +/// `QueryProjection.key`: the case-insensitive output name. +pub(crate) fn projection_key(projection: &Bound<'_, PyAny>) -> PyResult> { + Ok(projection.getattr("name")?.call_method0("lower")?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs new file mode 100644 index 0000000000..24d78cf850 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/python_nodes.rs @@ -0,0 +1,50 @@ +//! Graph walks over the natively hosted Python-node graph record. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTuple, PyTupleMethods, +}; + +/// `PythonNodeGraph.selected_sql_refs(selected_names=...)`: SQL references of the selected closure. +pub(crate) fn graph_selected_sql_refs( + graph: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py = graph.py(); + let keyword: Option> = match kwargs { + Some(kwargs) => kwargs.get_item("selected_names")?, + None => None, + }; + let selected: Bound<'_, PyAny> = match keyword { + Some(selected) => selected, + None => args.get_item(0)?, + }; + let nodes_by_name: Bound<'_, PyAny> = graph.getattr("nodes_by_name")?; + let upstream: Bound<'_, PyAny> = graph.getattr("upstream_deps")?; + let refs: Bound<'_, PySet> = PySet::empty(py)?; + let mut pending: Vec = selected + .try_iter()? + .map(|name| name?.extract::()) + .collect::>()?; + let mut visited: HashSet = HashSet::new(); + while let Some(name) = pending.pop() { + if !visited.insert(name.clone()) { + continue; + } + let node: Bound<'_, PyAny> = nodes_by_name.call_method1("get", (name.as_str(),))?; + if !node.is_none() { + for reference in node.getattr("sql_deps")?.try_iter()? { + refs.add(reference?)?; + } + } + let parents: Bound<'_, PyAny> = + upstream.call_method1("get", (name.as_str(), PyTuple::empty(py)))?; + for parent in parents.try_iter()? { + pending.push(parent?.extract()?); + } + } + Ok(PyFrozenSet::new(py, refs.iter())?.into_any().unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs new file mode 100644 index 0000000000..3fb813183d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/references.rs @@ -0,0 +1,59 @@ +//! Reference call spellings of the natively hosted `SqlReferenceKind`. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; + +fn function_name(kind: &Bound<'_, PyAny>) -> PyResult { + let value: String = kind.str()?.extract()?; + Ok(format!("__{value}")) +} + +/// `SqlReferenceKind.function_name`: the reference function, such as `__ref`. +pub(crate) fn reference_function_name(kind: &Bound<'_, PyAny>) -> PyResult> { + function_name(kind)?.into_py_any(kind.py()) +} + +/// `SqlReferenceKind.fixture_cte_prefix`: the prefix of fixture CTEs for this kind. +pub(crate) fn reference_fixture_cte_prefix(kind: &Bound<'_, PyAny>) -> PyResult> { + format!("{}__", function_name(kind)?).into_py_any(kind.py()) +} + +/// `SqlReferenceKind.example_call(*args, quote="'")`: a call with quoted arguments. +pub(crate) fn reference_example_call( + kind: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let quote: String = match kwargs { + Some(kwargs) => match kwargs.get_item("quote")? { + Some(quote) => quote.extract()?, + None => "'".to_owned(), + }, + None => "'".to_owned(), + }; + let doubled: String = format!("{quote}{quote}"); + let mut quoted: Vec = Vec::with_capacity(args.len()); + for argument in args.iter() { + let text: String = argument.extract()?; + quoted.push(format!("{quote}{}{quote}", text.replace("e, &doubled))); + } + format!("{}({})", function_name(kind)?, quoted.join(", ")).into_py_any(kind.py()) +} + +/// `SqlReferenceKind.placeholder_call(placeholder="")`. +pub(crate) fn reference_placeholder_call( + kind: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let placeholder: String = match kwargs + .map(|kwargs| kwargs.get_item("placeholder")) + .transpose()? + { + Some(Some(value)) => value.extract()?, + _ if !args.is_empty() => args.get_item(0)?.extract()?, + _ => String::new(), + }; + format!("{}({placeholder})", function_name(kind)?).into_py_any(kind.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs new file mode 100644 index 0000000000..a535e8be08 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/rule_shapes.rs @@ -0,0 +1,190 @@ +//! Derived views and constructors of the natively hosted custom-rule records. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::NATIVE_MODULE; + +/// Keyword fields every `RuleOption` constructor requires. +const REQUIRED_OPTION_FIELDS: [&str; 3] = ["name", "default", "description"]; + +/// `pathlib.Path(value)` of a `ProjectPath`. +fn project_path<'py>(instance: &Bound<'py, PyAny>) -> PyResult> { + imported(instance.py(), "pathlib", "Path")?.call1((instance.getattr("value")?,)) +} + +/// `ProjectPath.parts`. +pub(crate) fn project_path_parts(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(project_path(instance)?.getattr("parts")?.unbind()) +} + +/// `ProjectPath.name`. +pub(crate) fn project_path_name(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(project_path(instance)?.getattr("name")?.unbind()) +} + +/// `ProjectPath.depth`: the number of path parts. +pub(crate) fn project_path_depth(instance: &Bound<'_, PyAny>) -> PyResult> { + project_path(instance)? + .getattr("parts")? + .len()? + .into_py_any(instance.py()) +} + +/// `Rule.project_wide`: whether the rule runs once for the compiled project. +pub(crate) fn rule_project_wide(instance: &Bound<'_, PyAny>) -> PyResult> { + let project: Bound<'_, PyAny> = + imported(instance.py(), NATIVE_MODULE, "RuleSubject")?.getattr("PROJECT")?; + instance + .getattr("subject")? + .is(&project) + .into_py_any(instance.py()) +} + +/// `RuleResult.finding_count`. +pub(crate) fn rule_result_finding_count(instance: &Bound<'_, PyAny>) -> PyResult> { + instance + .getattr("findings")? + .len()? + .into_py_any(instance.py()) +} + +/// `RuleOption[T]`: the generic alias annotations use. +pub(crate) fn rule_option_class_getitem( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + _kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = receiver.py(); + let class: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "RuleOption")?; + Ok(imported(py, "types", "GenericAlias")? + .call1((class, args.get_item(0)?))? + .unbind()) +} + +/// A `RuleOption` of `value_type` from the keyword fields named in `names`, plus `extra`. +fn rule_option( + (args, kwargs): (&Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>), + (value_type, names): (Bound<'_, PyAny>, &[&str]), + extra: &[(&str, Bound<'_, PyAny>)], +) -> PyResult> { + let py: Python<'_> = value_type.py(); + if !args.is_empty() { + return Err(PyTypeError::new_err( + "RuleOption constructors take keyword arguments only", + )); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("value_type", &value_type)?; + if let Some(kwargs) = kwargs { + for (name, value) in kwargs.iter() { + let name: String = name.extract()?; + if !names.contains(&name.as_str()) { + return Err(PyTypeError::new_err(format!( + "RuleOption constructor got an unexpected keyword argument '{name}'" + ))); + } + fields.set_item(name, value)?; + } + } + for name in REQUIRED_OPTION_FIELDS { + if !fields.contains(name)? { + return Err(PyTypeError::new_err(format!( + "RuleOption constructor missing required keyword-only argument: '{name}'" + ))); + } + } + for (name, value) in extra { + fields.set_item(*name, value)?; + } + Ok(imported(py, NATIVE_MODULE, "RuleOption")? + .call((), Some(&fields))? + .unbind()) +} + +/// The builtin type `name`. +fn builtin<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, "builtins", name) +} + +/// `RuleOption.boolean(*, name, default, description)`. +pub(crate) fn rule_option_boolean( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let value_type: Bound<'_, PyAny> = builtin(receiver.py(), "bool")?; + rule_option( + (args, kwargs), + (value_type, &["name", "default", "description"]), + &[], + ) +} + +/// `RuleOption.integer(*, name, default, description, minimum=None, maximum=None)`. +pub(crate) fn rule_option_integer( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let value_type: Bound<'_, PyAny> = builtin(receiver.py(), "int")?; + rule_option( + (args, kwargs), + ( + value_type, + &["name", "default", "description", "minimum", "maximum"], + ), + &[], + ) +} + +/// `RuleOption.string(*, name, default, description, choices=())`. +pub(crate) fn rule_option_string( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let value_type: Bound<'_, PyAny> = builtin(receiver.py(), "str")?; + rule_option( + (args, kwargs), + (value_type, &["name", "default", "description", "choices"]), + &[], + ) +} + +/// `RuleOption.string_list(*, name, default, description, minimum_items=None)`. +pub(crate) fn rule_option_string_list( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = receiver.py(); + rule_option( + (args, kwargs), + ( + builtin(py, "tuple")?, + &["name", "default", "description", "minimum_items"], + ), + &[("item_type", builtin(py, "str")?)], + ) +} + +/// `RuleOption.integer_list(*, name, default, description, minimum_items=None)`. +pub(crate) fn rule_option_integer_list( + receiver: &Bound<'_, PyAny>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py: Python<'_> = receiver.py(); + rule_option( + (args, kwargs), + ( + builtin(py, "tuple")?, + &["name", "default", "description", "minimum_items"], + ), + &[("item_type", builtin(py, "int")?)], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs new file mode 100644 index 0000000000..9634a620d9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/scopes.rs @@ -0,0 +1,30 @@ +//! Computed attributes of the natively hosted scope records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; + +/// The completeness sections of a `ScopeCompleteness`. +const COMPLETENESS_SECTIONS: [&str; 6] = [ + "discovery", + "static_visibility", + "runtime_usage", + "relationships", + "placement", + "promotion_impact", +]; + +/// `ScopeCompleteness.complete`: whether every index section is complete. +pub(crate) fn completeness_complete(instance: &Bound<'_, PyAny>) -> PyResult> { + let mut complete: bool = true; + for section in COMPLETENESS_SECTIONS { + complete = complete && instance.getattr(section)?.is_truthy()?; + } + complete.into_py_any(instance.py()) +} + +/// `ScopeTargetQuery.unknown`: whether the target is absent rather than merely inaccessible. +pub(crate) fn target_query_unknown(instance: &Bound<'_, PyAny>) -> PyResult> { + let unknown: bool = !instance.getattr("matches")?.is_truthy()? + && !instance.getattr("declaration_matches")?.is_truthy()?; + unknown.into_py_any(instance.py()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs new file mode 100644 index 0000000000..a0b1704ce5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/values.rs @@ -0,0 +1,45 @@ +//! Computed attributes of the natively hosted SQL value and fingerprint records. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +/// `SqlLogicalType.display_name`: the kind, with its element type in angle brackets. +pub(crate) fn logical_type_display_name(instance: &Bound<'_, PyAny>) -> PyResult> { + let kind: String = instance.getattr("kind")?.getattr("value")?.extract()?; + let element: Bound<'_, PyAny> = instance.getattr("element_type")?; + if element.is_none() { + return kind.into_py_any(instance.py()); + } + let inner: String = element.getattr("display_name")?.extract()?; + format!("{kind}<{inner}>").into_py_any(instance.py()) +} + +/// `SqlValue.kind`: the outer logical kind. +pub(crate) fn sql_value_kind(instance: &Bound<'_, PyAny>) -> PyResult> { + Ok(instance.getattr("logical_type")?.getattr("kind")?.unbind()) +} + +/// `FingerprintSet`: index the fingerprints by `(node_type, node_name)` when not given. +pub(crate) fn finish_fingerprint_set( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + if !values[2].bind(py).is_none() { + return Ok(values); + } + let by_identity: Bound<'_, PyDict> = PyDict::new(py); + for fingerprint in values[1].bind(py).call_method0("values")?.try_iter()? { + let fingerprint: Bound<'_, PyAny> = fingerprint?; + let identity: Bound<'_, PyTuple> = PyTuple::new( + py, + [ + fingerprint.getattr("node_type")?, + fingerprint.getattr("node_name")?, + ], + )?; + by_identity.set_item(identity, fingerprint)?; + } + values[2] = by_identity.into_any().unbind(); + Ok(values) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index f86c402d4d..09870bba21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1,14 +1,24 @@ //! The layouts of every native record class. use crate::bindings::_helpers::compile_records::constants::COMPILE_VALUE_RECORDS; -use crate::bindings::_helpers::compiler_records::constants::{ - COMPILER_ENUMS, COMPILER_ERRORS, COMPILER_RECORDS, -}; +use crate::bindings::_helpers::compiler_enums::constants::COMPILER_ENUMS; +use crate::bindings::_helpers::compiler_errors::constants::COMPILER_ERRORS; +use crate::bindings::_helpers::compiler_records::constants::COMPILER_RECORDS; +use crate::bindings::_helpers::contract_records::constants::CONTRACT_RECORDS; use crate::bindings::_helpers::freshness::observation::normalize_observation; +use crate::bindings::_helpers::planner_records::constants::{ + PLANNER_ENUMS, PLANNER_RECORDS, PLANNER_VALUE_RECORDS, +}; +use crate::bindings::_helpers::record_behaviours::constants::{ + BEHAVIOUR_CACHED_PROPERTIES, BEHAVIOUR_FINISHERS, BEHAVIOUR_METHODS, BEHAVIOUR_ORDERED, + BEHAVIOUR_PROPERTIES, BEHAVIOUR_SEQUENCES, +}; use crate::bindings::_helpers::records::models::{ ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, }; -use crate::bindings::_helpers::records::types::RecordFinish; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFinish}; +use crate::bindings::_helpers::rule_records::constants::RULE_RECORDS; +use crate::bindings::_helpers::scope_records::constants::SCOPE_RECORDS; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; /// The class attribute holding a record class's index into `RECORDS`. @@ -43,6 +53,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Expected CLI-facing error that should be rendered without a traceback.", class_attributes: &[("code", ClassValue::Text("C000"))], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -51,6 +62,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Staged compile artifacts changed before publication, so nothing stale is deleted.", class_attributes: &[], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -62,6 +74,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ ("exit_code", ClassValue::Int(2)), ], required: &[], + optional: &[], fixed: &[], }, ErrorSpec { @@ -70,6 +83,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Expected query-diff failure with an explicit outcome and exit code.", class_attributes: &[], required: &["status", "exit_code"], + optional: &[], fixed: &[], }, ErrorSpec { @@ -78,6 +92,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Safety or preflight prevented complete value evidence.", class_attributes: &[], required: &[], + optional: &[], fixed: &[ ("status", ClassValue::Text("incomplete")), ("exit_code", ClassValue::Int(2)), @@ -89,6 +104,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Setup, execution, ownership publication, or cleanup failed.", class_attributes: &[], required: &[], + optional: &[], fixed: &[ ("status", ClassValue::Text("execution_failed")), ("exit_code", ClassValue::Int(3)), @@ -103,6 +119,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ ("exit_code", ClassValue::Int(2)), ], required: &["details"], + optional: &[], fixed: &[], }, ErrorSpec { @@ -111,6 +128,7 @@ const CLI_ERRORS: &[ErrorSpec] = &[ doc: "Raised when model migration state inputs or rows are invalid.", class_attributes: &[("code", ClassValue::Text("M000"))], required: &[], + optional: &[], fixed: &[], }, ]; @@ -122,6 +140,7 @@ pub(crate) const ENUMS: &[&[EnumSpec]] = &[ DBT_ENUMS, PRESENTATION_ENUMS, COMPILER_ENUMS, + PLANNER_ENUMS, ]; /// Every native record group, in registration order. @@ -137,11 +156,29 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ DIAGNOSTIC_RECORDS, COMPILER_RECORDS, COMPILE_VALUE_RECORDS, + SCOPE_RECORDS, + PLANNER_RECORDS, + PLANNER_VALUE_RECORDS, + RULE_RECORDS, + CONTRACT_RECORDS, ]; -/// Records whose bound values are normalized before the record is created. -pub(crate) const RECORD_FINISHERS: &[(&str, RecordFinish)] = - &[("SourceFreshnessObservation", normalize_observation)]; +/// Records whose bound values are normalized or validated before the record is created. +pub(crate) const RECORD_FINISHERS: &[&[(&str, RecordFinish)]] = &[ + &[("SourceFreshnessObservation", normalize_observation)], + BEHAVIOUR_FINISHERS, +]; +/// `(class, attribute, getter)` computed attributes set on native classes after registration. +pub(crate) const CLASS_PROPERTIES: &[&[(&str, &str, ClassProperty)]] = &[BEHAVIOUR_PROPERTIES]; +/// `(class, attribute, getter)` computed attributes a record computes once and keeps. +pub(crate) const CACHED_PROPERTIES: &[&[(&str, &str, ClassProperty)]] = + &[BEHAVIOUR_CACHED_PROPERTIES]; +/// `(class, attribute, method)` methods set on native classes after registration. +pub(crate) const CLASS_METHODS: &[&[(&str, &str, ClassMethod)]] = &[BEHAVIOUR_METHODS]; +/// Records ordered by their compared fields, as `order=True` dataclasses are. +pub(crate) const ORDERED_RECORDS: &[&[&str]] = &[BEHAVIOUR_ORDERED]; +/// Native records registered as `collections.abc.Sequence` virtual subclasses. +pub(crate) const SEQUENCE_RECORDS: &[&[&str]] = &[BEHAVIOUR_SEQUENCES]; /// `SourceFreshnessObservation` field positions. pub(crate) const DATA_VERSION_FIELD: usize = 2; pub(crate) const OBSERVED_AT_FIELD: usize = 4; @@ -246,7 +283,7 @@ const BUILD_INVOCATION: RecordSpec = RecordSpec { FieldSpec::new("effective_target_name", FieldDefault::None), FieldSpec::new( "execution_limits", - FieldDefault::Call("sqlbuild.spec.contracts.models", "ExecutionLimitsConfig"), + FieldDefault::Call("sqlbuild._native", "ExecutionLimitsConfig"), ), ], }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs index c5d83e70b9..67c2079c95 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs @@ -115,7 +115,7 @@ fn initialize( if let Some(fields) = fields { for key in fields.keys() { let key: String = key.extract()?; - if !spec.required.contains(&key.as_str()) { + if !spec.required.contains(&key.as_str()) && !spec.optional.contains(&key.as_str()) { return Err(PyTypeError::new_err(format!( "{name}.__init__() got an unexpected keyword argument '{key}'" ))); @@ -148,6 +148,13 @@ fn initialize( }; error.setattr(*required, value)?; } + for optional in spec.optional { + let value = match fields { + Some(fields) => fields.get_item(optional)?, + None => None, + }; + error.setattr(*optional, value.unwrap_or_else(|| py.None().into_bound(py)))?; + } for (attribute, value) in spec.fixed { error.setattr(*attribute, class_value(py, *value)?)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index def82d50c4..02c32155fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -1,8 +1,9 @@ //! Construction, comparison and display of native record field values. use pyo3::IntoPyObjectExt; -use pyo3::exceptions::{PyAttributeError, PyTypeError}; +use pyo3::exceptions::{PyAttributeError, PyRuntimeError, PyTypeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyclass::CompareOp; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods, PyType, PyTypeMethods, @@ -10,7 +11,7 @@ use pyo3::types::{ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::records::constants::{ - FROZEN_INSTANCE_ERROR, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, + FROZEN_INSTANCE_ERROR, ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, }; use crate::bindings::_helpers::records::models::{FieldDefault, NativeRecord, RecordSpec}; @@ -81,7 +82,11 @@ pub(crate) fn build_values( }; values.push(value.unbind()); } - match RECORD_FINISHERS.iter().find(|(record, _)| *record == name) { + match RECORD_FINISHERS + .iter() + .flat_map(|group| group.iter()) + .find(|(record, _)| *record == name) + { Some((_, finish)) => finish(py, values), None => Ok(values), } @@ -102,15 +107,19 @@ fn default_value<'py>( FieldDefault::True => Ok(PyBool::new(py, true).to_owned().into_any()), FieldDefault::Int(value) => value.into_bound_py_any(py), FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), + FieldDefault::TextTuple(texts) => Ok(PyTuple::new(py, texts)?.into_any()), + FieldDefault::TextFrozenset(texts) => Ok(PyFrozenSet::new(py, texts)?.into_any()), FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), FieldDefault::EmptyFrozenset => Ok(PyFrozenSet::empty(py)?.into_any()), FieldDefault::Member(module, class, member) => imported(py, module, class)?.getattr(member), FieldDefault::Call(module, name) => imported(py, module, name)?.call0(), + FieldDefault::Factory(factory) => factory(py), + FieldDefault::CallText(module, name, text) => imported(py, module, name)?.call1((text,)), } } -/// The values of the fields that take part in equality, hashing and repr. +/// The values of the fields that take part in equality and hashing. pub(crate) fn compared_values<'py>( record: &Bound<'py, NativeRecord>, ) -> PyResult> { @@ -126,14 +135,14 @@ pub(crate) fn compared_values<'py>( PyTuple::new(py, values) } -/// `Name(field=value!r, ...)` over the compared fields, as a dataclass repr reads. +/// `Name(field=value!r, ...)` over the shown fields, as a dataclass repr reads. pub(crate) fn repr_text(record: &Bound<'_, NativeRecord>) -> PyResult { let py: Python<'_> = record.py(); let cls: Bound<'_, PyType> = record.get_type(); let spec: &RecordSpec = record_spec(&cls)?; let mut parts: Vec = Vec::with_capacity(spec.fields.len()); for (field, value) in spec.fields.iter().zip(record.get().values.iter()) { - if field.compared { + if field.shown { parts.push(format!("{}={}", field.name, value.bind(py).repr()?)); } } @@ -150,3 +159,57 @@ pub(crate) fn frozen_error(py: Python<'_>, name: &str) -> PyErr { Err(_) => PyAttributeError::new_err(message), } } + +/// `slf other` over the compared fields, as an `order=True` dataclass compares. +pub(crate) fn ordered_comparison( + slf: &Bound<'_, NativeRecord>, + other: &Bound<'_, PyAny>, + op: CompareOp, +) -> PyResult> { + let py: Python<'_> = slf.py(); + let name: &str = record_spec(&slf.get_type())?.name; + let ordered: bool = ORDERED_RECORDS.iter().any(|group| group.contains(&name)); + if !ordered || !other.get_type().is(slf.get_type()) { + return Ok(py.NotImplemented()); + } + let other: &Bound<'_, NativeRecord> = other.cast::()?; + compared_values(slf)? + .as_any() + .rich_compare(compared_values(other)?, op)? + .into_py_any(py) +} + +/// The cached value of `name` on `record`, if computed before. +pub(crate) fn cached_value( + record: &Bound<'_, NativeRecord>, + name: &str, +) -> PyResult>> { + let py: Python<'_> = record.py(); + let cached = record + .get() + .cached + .lock() + .map_err(|_| PyRuntimeError::new_err("native record cache is poisoned"))?; + Ok(cached + .iter() + .find(|(cached_name, _)| *cached_name == name) + .map(|(_, value)| value.clone_ref(py))) +} + +/// Keep `value` as the cached value of `name` on `record`. +pub(crate) fn store_cached( + record: &Bound<'_, NativeRecord>, + name: &'static str, + value: &Py, +) -> PyResult<()> { + let py: Python<'_> = record.py(); + let mut cached = record + .get() + .cached + .lock() + .map_err(|_| PyRuntimeError::new_err("native record cache is poisoned"))?; + if !cached.iter().any(|(cached_name, _)| *cached_name == name) { + cached.push((name, value.clone_ref(py))); + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 4d5df04eb9..4c2b8657de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -1,13 +1,18 @@ //! Native frozen value records and the field layouts they are built from. +use std::sync::Mutex; + use pyo3::exceptions::PyAttributeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyclass::CompareOp; use pyo3::types::{PyDict, PyTuple, PyType}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::records::fields::{ - build_values, compared_values, frozen_error, record_spec, repr_text, + build_values, cached_value, compared_values, frozen_error, ordered_comparison, record_spec, + repr_text, store_cached, }; +use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFactory}; /// How a record field is filled when the caller does not pass it. #[derive(Debug, Clone, Copy)] @@ -18,6 +23,10 @@ pub(crate) enum FieldDefault { True, Int(i64), Text(&'static str), + /// A tuple of texts. + TextTuple(&'static [&'static str]), + /// A frozenset of texts. + TextFrozenset(&'static [&'static str]), EmptyTuple, EmptyDict, EmptyFrozenset, @@ -25,14 +34,19 @@ pub(crate) enum FieldDefault { Member(&'static str, &'static str, &'static str), /// `getattr(import(module), name)()`, evaluated for every instance. Call(&'static str, &'static str), + /// A native factory, evaluated for every instance. + Factory(RecordFactory), + /// `getattr(import(module), name)(text)`, evaluated for every instance. + CallText(&'static str, &'static str, &'static str), } -/// One record field: its name, default and whether it takes part in equality and repr. +/// One record field: its name, default, and whether it takes part in equality and in repr. #[derive(Debug, Clone, Copy)] pub(crate) struct FieldSpec { pub(crate) name: &'static str, pub(crate) default: FieldDefault, pub(crate) compared: bool, + pub(crate) shown: bool, } /// The layout of one native record class. @@ -68,6 +82,8 @@ pub(crate) struct ErrorSpec { pub(crate) class_attributes: &'static [(&'static str, ClassValue)], /// Keyword-only constructor fields stored as attributes. pub(crate) required: &'static [&'static str], + /// Keyword-only constructor fields stored as attributes, `None` when not passed. + pub(crate) optional: &'static [&'static str], /// Attributes every instance is given. pub(crate) fixed: &'static [(&'static str, ClassValue)], } @@ -77,6 +93,8 @@ pub(crate) struct ErrorSpec { #[pyclass(module = "sqlbuild._native", subclass, frozen)] pub(crate) struct NativeRecord { pub(crate) values: Vec>, + /// Values of the record's cached computed attributes, by attribute name. + pub(crate) cached: Mutex)>>, } #[pymethods] @@ -92,6 +110,7 @@ impl NativeRecord { let spec: &RecordSpec = record_spec(cls)?; Ok(Self { values: build_values(spec, args, kwargs)?, + cached: Mutex::new(Vec::new()), }) } @@ -107,6 +126,22 @@ impl NativeRecord { .into_py_any(py) } + fn __lt__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Lt) + } + + fn __le__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Le) + } + + fn __gt__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Gt) + } + + fn __ge__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + ordered_comparison(slf, other, CompareOp::Ge) + } + fn __hash__(slf: &Bound<'_, Self>) -> PyResult { compared_values(slf)?.as_any().hash() } @@ -122,6 +157,90 @@ impl NativeRecord { } } +/// A computed attribute of a native class, evaluated by its native function. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeProperty { + pub(crate) property: ClassProperty, + pub(crate) name: &'static str, + /// Whether a record computes the attribute once and keeps it. + pub(crate) cached: bool, +} + +#[pymethods] +impl NativeProperty { + fn __get__( + slf: &Bound<'_, Self>, + instance: &Bound<'_, PyAny>, + _owner: &Bound<'_, PyAny>, + ) -> PyResult> { + if instance.is_none() { + return Ok(slf.clone().into_any().unbind()); + } + let descriptor: &Self = slf.get(); + if !descriptor.cached || !instance.is_instance_of::() { + return (descriptor.property)(instance); + } + let record: &Bound<'_, NativeRecord> = instance.cast::()?; + if let Some(value) = cached_value(record, descriptor.name)? { + return Ok(value); + } + let value: Py = (descriptor.property)(instance)?; + store_cached(record, descriptor.name, &value)?; + Ok(value) + } + + fn __set__(&self, instance: &Bound<'_, PyAny>, _value: &Bound<'_, PyAny>) -> PyResult<()> { + Err(frozen_error(instance.py(), self.name)) + } +} + +/// A method of a native class, run by its native function. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeMethod { + pub(crate) method: ClassMethod, +} + +#[pymethods] +impl NativeMethod { + fn __get__( + slf: &Bound<'_, Self>, + instance: &Bound<'_, PyAny>, + owner: &Bound<'_, PyAny>, + ) -> PyResult> { + let receiver: &Bound<'_, PyAny> = if instance.is_none() { owner } else { instance }; + Ok(Py::new( + slf.py(), + BoundNativeMethod { + method: slf.get().method, + receiver: receiver.clone().unbind(), + }, + )? + .into_any()) + } +} + +/// A `NativeMethod` bound to the instance or class it was read from. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct BoundNativeMethod { + method: ClassMethod, + receiver: Py, +} + +#[pymethods] +impl BoundNativeMethod { + #[pyo3(signature = (*args, **kwargs))] + fn __call__( + &self, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + (self.method)(self.receiver.bind(args.py()), args, kwargs) + } +} + /// Read-only access to one field of a native record. #[derive(Debug)] #[pyclass(module = "sqlbuild._native", frozen)] @@ -172,6 +291,27 @@ impl FieldSpec { name, default, compared: true, + shown: true, + } + } + + /// A field left out of equality, hashing and repr. + pub(crate) const fn uncompared(name: &'static str, default: FieldDefault) -> Self { + Self { + name, + default, + compared: false, + shown: false, + } + } + + /// A field left out of equality and hashing but shown in repr. + pub(crate) const fn uncompared_shown(name: &'static str, default: FieldDefault) -> Self { + Self { + name, + default, + compared: false, + shown: true, } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 3705508d4e..e26703adf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -5,22 +5,27 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ - BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, - DATACLASSES_MODULE, ENUM_MODULE, ENUMS, ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, - NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, VALUE_ERROR_PARENT, + BUILTIN_PARENT_PREFIX, BUILTINS_MODULE, CACHED_PROPERTIES, CLASS_METHODS, CLASS_PROPERTIES, + DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, + ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, + SEQUENCE_RECORDS, STR_ENUM, VALUE_ERROR_PARENT, }; use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, }; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::models::{ - EnumSpec, ErrorSpec, NativeRecord, RecordAttribute, RecordSpec, + BoundNativeMethod, EnumSpec, ErrorSpec, NativeMethod, NativeProperty, NativeRecord, + RecordAttribute, RecordSpec, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let py: Python<'_> = module.py(); module.add_class::()?; module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; let base: Bound<'_, PyAny> = py.get_type::().into_any(); let metaclass: Bound<'_, PyAny> = base.get_type().into_any(); let str_enum: Bound<'_, PyAny> = py.import(ENUM_MODULE)?.getattr(STR_ENUM)?; @@ -49,6 +54,36 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { )?; module.add(spec.name, class)?; } + for (class, name, property) in CLASS_PROPERTIES.iter().flat_map(|group| group.iter()) { + let descriptor: Py = Py::new( + py, + NativeProperty { + property: *property, + name, + cached: false, + }, + )?; + module.getattr(*class)?.setattr(*name, descriptor)?; + } + for (class, name, property) in CACHED_PROPERTIES.iter().flat_map(|group| group.iter()) { + let descriptor: Py = Py::new( + py, + NativeProperty { + property: *property, + name, + cached: true, + }, + )?; + module.getattr(*class)?.setattr(*name, descriptor)?; + } + for (class, name, method) in CLASS_METHODS.iter().flat_map(|group| group.iter()) { + let descriptor: Py = Py::new(py, NativeMethod { method: *method })?; + module.getattr(*class)?.setattr(*name, descriptor)?; + } + let sequence: Bound<'_, PyAny> = py.import("collections.abc")?.getattr("Sequence")?; + for class in SEQUENCE_RECORDS.iter().flat_map(|group| group.iter()) { + sequence.call_method1("register", (module.getattr(*class)?,))?; + } Ok(()) } @@ -97,7 +132,7 @@ fn dataclass_fields<'py>( for field in spec.fields { let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item("compare", field.compared)?; - options.set_item("repr", field.compared)?; + options.set_item("repr", field.shown)?; let declared: Bound<'py, PyAny> = dataclasses.getattr("field")?.call((), Some(&options))?; declared.setattr("name", field.name)?; declared.setattr("type", "object")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs index 4b90f1cc6a..6bd40b1bda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs @@ -1,6 +1,17 @@ //! Hooks applied while building native records. -use pyo3::prelude::{Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; /// Normalizes the bound field values of one record before it is created. pub(crate) type RecordFinish = fn(Python<'_>, Vec>) -> PyResult>>; + +/// Computes one read-only attribute of a native class instance. +pub(crate) type ClassProperty = fn(&Bound<'_, PyAny>) -> PyResult>; + +/// Runs one method of a native class with its receiver (the instance, or the class when unbound). +pub(crate) type ClassMethod = + fn(&Bound<'_, PyAny>, &Bound<'_, PyTuple>, Option<&Bound<'_, PyDict>>) -> PyResult>; + +/// Builds the default value of one record field. +pub(crate) type RecordFactory = fn(Python<'_>) -> PyResult>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index 1aaca63c39..24d25ddda6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -12,11 +12,11 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS: &str = "sqlbuild._native"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; -const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; -const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; +const COMPILE_ERRORS: &str = "sqlbuild._native"; +const PLANNER_ERRORS: &str = "sqlbuild._native"; +const RULES_ERRORS: &str = "sqlbuild._native"; const SPEC_ERRORS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; const COMMAND_TYPES: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index 205b8f1d28..4bc156e557 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -17,8 +17,8 @@ use crate::bindings::_helpers::scopes::scope_validation::scope_relocated_declara const NATIVE_MODULE: &str = "sqlbuild._native"; const CONFIG_MODULE: &str = "sqlbuild._native"; -const SCOPE_MODELS: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS: &str = "sqlbuild._native"; +const SCOPE_TYPES: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const MATERIALIZED_KEY: &str = "materialized"; const MIGRATE_FROM_KEY: &str = "migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index fcfa29b3a3..137c2a04a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -22,9 +22,9 @@ use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, inter const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs index 62cadc33e2..2c74115fff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 1d40296f1f..b2aca182b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -11,14 +11,13 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; -const BUILTINS_MODULE: &str = "builtins"; const NO_DECLARATION_KIND: u8 = 255; /// Declaration kinds by their native code: enum 0, constant 1, macro 2. const DECLARATION_KINDS: [&str; 3] = ["ENUM", "CONSTANT", "MACRO"]; @@ -255,10 +254,6 @@ pub(crate) fn native_declaration_contexts<'py>( "resolution_context", imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, )?; - classes.set_item( - "allocate", - imported(py, BUILTINS_MODULE, "object")?.getattr("__new__")?, - )?; classes.set_item("reasons", reasons)?; let keys: Bound<'py, PyDict> = PyDict::new(py); keys.set_item("identity_keys", identity_keys)?; @@ -442,7 +437,7 @@ fn render_constant_declaration<'py>( /// One enum member value as a SQL scalar literal. #[pyfunction] #[pyo3(signature = (*, value))] -fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn render_enum_member_value(value: &Bound<'_, PyAny>) -> PyResult { if value.is_instance_of::() { return value.str()?.extract(); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs index 277833513d..208e75a836 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_calls.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::render::macro_call_rows::{SiteRow, SpanRow, site_ use crate::bindings::_helpers::render::macro_expansion::raised; use crate::bindings::types::CompilerDetach; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; /// `(call start or None, call name or None, message)` of the error Python's scan raises. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 957e496407..230858dbbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -19,11 +19,11 @@ use crate::bindings::_helpers::sqltext::sql_references::extract_references; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const FRONTIER_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const SCOPES_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const SCOPES_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPES_MODELS_MODULE: &str = "sqlbuild._native"; +const SCOPES_TYPES_MODULE: &str = "sqlbuild._native"; const REF_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; const TYPED_REFERENCE_FUNCTIONS: [(&str, &str); 3] = [ @@ -43,8 +43,8 @@ const GENERATED_REFERENCE_MARKERS: [&str; 6] = [ "__table_fn(", ]; const DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const BUILTINS_MODULE: &str = "builtins"; const MACRO_TOKEN: &str = "@"; const GENERATED_ALLOWED: &str = "macro-generated references"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs index 41c6b55dab..6362c065ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::result_output::terminal_index::{ TerminalEventClaim, TerminalEventIndex, current_index_object, }; -const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; +pub(crate) const RESULT_PATH_ENV: &str = "SQLBUILD_INTEGRATION_RESULT_PATH"; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["result", "plan"], &["command"]); /// Thread-safe JSON Lines execution event writer. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs new file mode 100644 index 0000000000..7eb07bb0da --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -0,0 +1,430 @@ +//! Layouts of the rule engine value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_RULE_GUIDANCE: RecordSpec = RecordSpec { + name: "RuleGuidance", + doc: "Structured authoring guidance shared by diagnostics and generated documentation.", + fields: &[ + FieldSpec::required("good_example"), + FieldSpec::required("anti_tautology"), + FieldSpec::required("mutation_check"), + ], +}; +const R_RULE_EXEMPTION: RecordSpec = RecordSpec { + name: "RuleExemption", + doc: "", + fields: &[ + FieldSpec::required("rule"), + FieldSpec::required("path"), + FieldSpec::required("reason"), + ], +}; +const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { + name: "GraphEdgeExclusion", + doc: "One exact, temporary internal-layer dependency exception.", + fields: &[ + FieldSpec::required("consumer"), + FieldSpec::required("dependency"), + FieldSpec::required("reason"), + ], +}; +const R_RULE_IGNORE: RecordSpec = RecordSpec { + name: "RuleIgnore", + doc: "", + fields: &[ + FieldSpec::required("rules"), + FieldSpec::required("paths"), + FieldSpec::required("reason"), + FieldSpec::new("selectors", FieldDefault::EmptyTuple), + ], +}; +const R_SELECT_STAR_ALLOW: RecordSpec = RecordSpec { + name: "SelectStarAllow", + doc: "", + fields: &[FieldSpec::required("paths"), FieldSpec::required("reason")], +}; +const R_THRESHOLD_OVERRIDE: RecordSpec = RecordSpec { + name: "ThresholdOverride", + doc: "", + fields: &[ + FieldSpec::required("paths"), + FieldSpec::required("thresholds"), + FieldSpec::required("reason"), + ], +}; +const R_RULES_CACHE_CONFIG: RecordSpec = RecordSpec { + name: "RulesCacheConfig", + doc: "", + fields: &[FieldSpec::new("enabled", FieldDefault::True)], +}; +const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { + name: "SqlTestRulesConfig", + doc: "", + fields: &[FieldSpec::new( + "pipeline_directory", + FieldDefault::Text("pipelines"), + )], +}; +const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { + name: "PreparedSqlLintResult", + doc: "Reusable expansion context and completed preliminary SQL checks.", + fields: &[ + FieldSpec::required("result"), + FieldSpec::required("context"), + ], +}; +const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { + name: "SqlExpansionReuse", + doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", + fields: &[ + FieldSpec::required("discovered_inputs"), + FieldSpec::required("declaration_scope"), + ], +}; +const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { + name: "PreparedSqlLint", + doc: "Process-local SQL checks started after compiler expansion has completed.", + fields: &[ + FieldSpec::required("codes"), + FieldSpec::required("future"), + FieldSpec::new("project_dir", FieldDefault::None), + ], +}; +const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { + name: "SqlRulesEvaluation", + doc: "SQL rule findings of one run, their cache accounting and elapsed time.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::new("sql_ms", FieldDefault::Int(0)), + ], +}; +const R_RULE_RUN_PLAN: RecordSpec = RecordSpec { + name: "RuleRunPlan", + doc: "The catalogue, selected rules, project slice and settings one rules run evaluates.", + fields: &[ + FieldSpec::required("effective_config"), + FieldSpec::required("project_dir"), + FieldSpec::required("catalogue"), + FieldSpec::required("custom_payloads"), + FieldSpec::required("selected"), + FieldSpec::required("skipped_type_proof_rules"), + FieldSpec::required("model_paths"), + FieldSpec::required("project"), + FieldSpec::required("config"), + ], +}; +const R_RULES_RUN_RESULT: RecordSpec = RecordSpec { + name: "RulesRunResult", + doc: "Unified findings and phase measurements for configured rules.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("evaluated_models"), + FieldSpec::required("built_in_ms"), + FieldSpec::required("custom_ms"), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), + FieldSpec::new("cache_hits", FieldDefault::Int(0)), + FieldSpec::new("cache_misses", FieldDefault::Int(0)), + FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + ], +}; +const R_NATIVE_RULES_EVALUATION: RecordSpec = RecordSpec { + name: "NativeRulesEvaluation", + doc: "The built-in rules response, decoded natively.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("selected_codes"), + FieldSpec::required("evaluated_models"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::required("built_in_ms"), + FieldSpec::required("reused"), + ], +}; +const R_CUSTOM_RULES_OUTCOME: RecordSpec = RecordSpec { + name: "CustomRulesOutcome", + doc: "Custom-rule findings in canonical order and their cache accounting.", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("cache_hits"), + FieldSpec::required("cache_misses"), + FieldSpec::required("custom_ms"), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + ], +}; +const R_MODEL: RecordSpec = RecordSpec { + name: "Model", + doc: "Stable identity and common metadata for one compiled SQL model.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("path"), + FieldSpec::required("materialization"), + ], +}; +const R_PROJECT: RecordSpec = RecordSpec { + name: "Project", + doc: "Stable summary for one compiled project invocation.", + fields: &[ + FieldSpec::required("model_count"), + FieldSpec::required("target_name"), + ], +}; +const R_PROJECT_PATH: RecordSpec = RecordSpec { + name: "ProjectPath", + doc: "One normalized project-relative file or directory.", + fields: &[ + FieldSpec::required("value"), + FieldSpec::required("is_file"), + FieldSpec::required("is_dir"), + ], +}; +const R_SQL_NODE: RecordSpec = RecordSpec { + name: "SqlNode", + doc: "Stable source location for one common SQL structure fact.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("line"), + FieldSpec::required("column"), + ], +}; +const R_MODEL_SQL: RecordSpec = RecordSpec { + name: "ModelSql", + doc: "Authored and compiler-expanded SQL documents for one model.", + fields: &[ + FieldSpec::required("authored"), + FieldSpec::required("expanded"), + ], +}; +const R_RULE_FACT_VIEWS: RecordSpec = RecordSpec { + name: "RuleFactViews", + doc: "Views shared by every invocation in one custom-rule host evaluation.", + fields: &[ + FieldSpec::required("project"), + FieldSpec::required("sql"), + FieldSpec::required("graph"), + FieldSpec::required("columns"), + FieldSpec::required("contracts"), + FieldSpec::required("tests"), + FieldSpec::required("audits"), + FieldSpec::required("declarations"), + ], +}; +const R_RULE_OPTION: RecordSpec = RecordSpec { + name: "RuleOption", + doc: "One typed configuration option declared by a rule.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("value_type"), + FieldSpec::required("default"), + FieldSpec::required("description"), + FieldSpec::new("choices", FieldDefault::EmptyTuple), + FieldSpec::new("minimum", FieldDefault::None), + FieldSpec::new("maximum", FieldDefault::None), + FieldSpec::new("minimum_items", FieldDefault::None), + FieldSpec::new("item_type", FieldDefault::None), + ], +}; +const R_RULE: RecordSpec = RecordSpec { + name: "Rule", + doc: "Immutable metadata and implementation for one rule.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("family"), + FieldSpec::required("slug"), + FieldSpec::required("message"), + FieldSpec::required("remediation"), + FieldSpec::required("check"), + FieldSpec::new("options", FieldDefault::EmptyTuple), + FieldSpec::new("enabled_by_default", FieldDefault::False), + FieldSpec::new("custom", FieldDefault::False), + FieldSpec::new("source", FieldDefault::None), + FieldSpec::new("subject", FieldDefault::None), + FieldSpec::new("subject_parameter", FieldDefault::None), + FieldSpec::new("context_parameter", FieldDefault::None), + FieldSpec::new("guidance", FieldDefault::None), + ], +}; +const R_FINDING: RecordSpec = RecordSpec { + name: "Finding", + doc: "One deterministic compiler rules violation.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::required("message"), + FieldSpec::required("remediation"), + FieldSpec::new("unevaluated", FieldDefault::False), + FieldSpec::new("affected_rules", FieldDefault::EmptyTuple), + FieldSpec::new("fixable", FieldDefault::False), + ], +}; +const R_LAYOUT_CONFIG: RecordSpec = RecordSpec { + name: "LayoutConfig", + doc: "", + fields: &[ + FieldSpec::new( + "levels", + FieldDefault::TextTuple(&[ + "staging", + "intermediate/clean", + "intermediate/enriched", + "mart", + ]), + ), + FieldSpec::new("domain_roots", FieldDefault::EmptyTuple), + ], +}; +const R_RULES_CONFIG: RecordSpec = RecordSpec { + name: "RulesConfig", + doc: "", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("ignore", FieldDefault::EmptyTuple), + FieldSpec::new("thresholds", FieldDefault::EmptyDict), + FieldSpec::new("threshold_overrides", FieldDefault::EmptyTuple), + FieldSpec::new("rule_options", FieldDefault::EmptyDict), + FieldSpec::new("rule_exceptions", FieldDefault::EmptyTuple), + FieldSpec::new("graph_edge_exceptions", FieldDefault::EmptyTuple), + FieldSpec::new("rule_ignores", FieldDefault::EmptyTuple), + FieldSpec::new("allow_exceptions", FieldDefault::True), + FieldSpec::new("select_star_allow", FieldDefault::EmptyTuple), + FieldSpec::new("domains", FieldDefault::EmptyTuple), + FieldSpec::new("approved_source_tokens", FieldDefault::EmptyTuple), + FieldSpec::new("retired_source_tokens", FieldDefault::EmptyDict), + FieldSpec::new( + "sql_tests", + FieldDefault::Call("sqlbuild._native", "SqlTestRulesConfig"), + ), + FieldSpec::new( + "layout", + FieldDefault::Call("sqlbuild._native", "LayoutConfig"), + ), + FieldSpec::new( + "cache", + FieldDefault::Call("sqlbuild._native", "RulesCacheConfig"), + ), + ], +}; +const R_RULES_RESULT: RecordSpec = RecordSpec { + name: "RulesResult", + doc: "", + fields: &[ + FieldSpec::required("findings"), + FieldSpec::required("evaluated_models"), + FieldSpec::new("unevaluated_resources", FieldDefault::Int(0)), + FieldSpec::new("cache_hits", FieldDefault::Int(0)), + FieldSpec::new("cache_misses", FieldDefault::Int(0)), + FieldSpec::new("built_in_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_cpu_ms", FieldDefault::Int(0)), + ], +}; +const R_RULE_FILE: RecordSpec = RecordSpec { + name: "RuleFile", + doc: "", + fields: &[FieldSpec::required("path"), FieldSpec::required("source")], +}; +const R_RULE_CASE: RecordSpec = RecordSpec { + name: "RuleCase", + doc: "", + fields: &[ + FieldSpec::required("description"), + FieldSpec::required("source"), + FieldSpec::required("expected_finding_count"), + FieldSpec::new( + "path", + FieldDefault::Text("models/mart/example__mart__result.sql"), + ), + FieldSpec::new("files", FieldDefault::EmptyTuple), + FieldSpec::new("config", FieldDefault::EmptyDict), + ], +}; +const R_RULE_RESULT: RecordSpec = RecordSpec { + name: "RuleResult", + doc: "", + fields: &[FieldSpec::required("findings")], +}; +const R_CUSTOM_RULE_EVALUATION: RecordSpec = RecordSpec { + name: "CustomRuleEvaluation", + doc: "Findings and observed fact reads of one rule invocation for one subject.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("subject"), + FieldSpec::required("findings"), + FieldSpec::required("reads"), + ], +}; +const R_MODULE_STATE_TRACE: RecordSpec = RecordSpec { + name: "ModuleStateTrace", + doc: "What exact module-state walks touched: object and type identities and their decisions.", + fields: &[ + FieldSpec::required("touched"), + FieldSpec::required("descended_types"), + FieldSpec::required("checked_instances"), + ], +}; +const R_MODULE_STATE_FINGERPRINT: RecordSpec = RecordSpec { + name: "ModuleStateFingerprint", + doc: "One exact custom-rule module-state fingerprint and the trace of the walk behind it.", + fields: &[FieldSpec::required("token"), FieldSpec::required("trace")], +}; +const R_CUSTOM_RULE_RUN: RecordSpec = RecordSpec { + name: "CustomRuleRun", + doc: "Every invocation of one host run plus rules whose reads could not be attributed.", + fields: &[ + FieldSpec::required("evaluations"), + FieldSpec::required("untracked_codes"), + FieldSpec::new("uncacheable_codes", FieldDefault::EmptyFrozenset), + FieldSpec::new("observed", FieldDefault::EmptyTuple), + FieldSpec::new("stateful_codes", FieldDefault::EmptyFrozenset), + ], +}; +const R_CUSTOM_HOST_PARTITION: RecordSpec = RecordSpec { + name: "CustomHostPartition", + doc: "How one host of a split run detects module state and learns the run was cancelled.", + fields: &[FieldSpec::required("cancelled")], +}; + +/// The rule engine value records. +pub(crate) const RULE_RECORDS: &[RecordSpec] = &[ + R_RULE_GUIDANCE, + R_RULE_EXEMPTION, + R_GRAPH_EDGE_EXCLUSION, + R_RULE_IGNORE, + R_SELECT_STAR_ALLOW, + R_THRESHOLD_OVERRIDE, + R_RULES_CACHE_CONFIG, + R_SQL_TEST_RULES_CONFIG, + R_PREPARED_SQL_LINT_RESULT, + R_SQL_EXPANSION_REUSE, + R_PREPARED_SQL_LINT, + R_SQL_RULES_EVALUATION, + R_RULE_RUN_PLAN, + R_RULES_RUN_RESULT, + R_NATIVE_RULES_EVALUATION, + R_CUSTOM_RULES_OUTCOME, + R_MODEL, + R_PROJECT, + R_PROJECT_PATH, + R_SQL_NODE, + R_MODEL_SQL, + R_RULE_FACT_VIEWS, + R_RULE_OPTION, + R_RULE, + R_FINDING, + R_LAYOUT_CONFIG, + R_RULES_CONFIG, + R_RULES_RESULT, + R_RULE_FILE, + R_RULE_CASE, + R_RULE_RESULT, + R_CUSTOM_RULE_EVALUATION, + R_MODULE_STATE_TRACE, + R_MODULE_STATE_FINGERPRINT, + R_CUSTOM_RULE_RUN, + R_CUSTOM_HOST_PARTITION, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index aef30876a5..3a0cfaa261 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -12,8 +12,8 @@ use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const TYPES_MODULE: &str = "sqlbuild.rule_engine.types"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const TYPES_MODULE: &str = "sqlbuild._native"; const FAMILY_SUFFIX_LENGTH: usize = 3; fn keywords<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs index 7b74700c6e..d872186f32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -15,11 +15,11 @@ use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSp use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; +use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules as verify_rule_sources; use crate::bindings::_helpers::rules::selection::{rules_error, selected_rule_codes}; const CUSTOM_CATALOGUE_MODULE: &str = "sqlbuild.rule_engine._helpers.engine._custom_catalogue"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; -const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; const PLANNER_SCOPE_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; @@ -264,13 +264,7 @@ pub(crate) fn verify_custom_rules<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult<()> { let py: Python<'py> = project_dir.py(); - let _ = call_with( - &imported(py, HERMETICITY_MODULE, "verify_custom_rules")?, - &[ - ("rules", PyTuple::new(py, rules)?.into_any()), - ("project_dir", project_dir.clone()), - ], - )?; + verify_rule_sources(PyTuple::new(py, rules)?.as_any(), project_dir)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index d535039dd1..613e955c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -16,3 +16,16 @@ pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; pub(crate) const RULES_NATIVE_API_VERSION: i64 = 1; +/// File name of the custom-rule result cache under the project cache directory. +pub(crate) const CUSTOM_RULES_CACHE_FILE: &str = "custom-rules.json"; +/// Suffix of a custom host scratch directory abandoned by a cancelled run. +pub(crate) const CUSTOM_HOST_ABANDONED_SUFFIX: &str = ".abandoned"; +/// Marker file whose presence tells split custom hosts the run was cancelled. +pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; +/// Rules that need proven column types and are skipped without SQL analysis. +pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"]; +/// Runtime identity custom-rule hosts and their cache entries record. +pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v2"; +/// Module a custom-rule host process runs. +pub(crate) const CUSTOM_HOST_LAUNCH_MODULE: &str = + "sqlbuild.rule_engine._helpers.host.custom_host_launch"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 76232eeaa2..6908890048 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -19,6 +19,9 @@ use sqlbuild_rules::custom_rule_cache::types::ReadSet; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::rules::constants::{ + CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULES_CACHE_FILE, +}; use crate::bindings::_helpers::rules::custom_hosts::{ HostCancel, HostRequest, host_payload_project, run_custom_hosts, }; @@ -27,11 +30,10 @@ use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const RULE_CONTEXT_MODULE: &str = "sqlbuild.rule_engine.classes.rule_context"; const FACT_REPLAY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.fact_replay"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_SUBJECT: &str = "project"; const MODEL_SUBJECT: &str = "model"; /// Project fields no rule fact reads, cleared before digests and host payloads use the project. @@ -112,7 +114,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( ("project_dir", run.project_dir.clone()), ( "file_name", - imported(py, RULES_CONSTANTS_MODULE, "CUSTOM_RULES_CACHE_FILE")?, + pyo3::types::PyString::new(py, CUSTOM_RULES_CACHE_FILE).into_any(), ), ], )? @@ -472,7 +474,7 @@ fn rule_identity( ), ( "runtime_version", - imported(py, RULES_CONSTANTS_MODULE, "CUSTOM_HOST_RUNTIME_VERSION")?, + pyo3::types::PyString::new(py, CUSTOM_HOST_RUNTIME_VERSION).into_any(), ), ("code", rule.getattr("code")?), ("family", rule.getattr("family")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs index 93fe2b427e..a72344175d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_hosts.rs @@ -26,14 +26,17 @@ use sqlbuild_rules::custom_host_pool::types::{HostInvocation, HostPlan}; use sqlbuild_rules::engine::main::custom_host::run_custom_host_json; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::rules::constants::{ + CUSTOM_HOST_ABANDONED_SUFFIX, CUSTOM_HOST_CANCELLED_MARKER, CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_RUNTIME_VERSION, +}; use crate::bindings::_helpers::rules::custom_host_observation::{ LaunchLog, ObservedHosts, before_publish, observed_hosts, record_launch, }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const HOST_INPUTS: [&str; 3] = ["target", "rules-cache", "host-inputs"]; /// The model fields hosts decode; every other model field is dropped from their payload. const PAYLOAD_MODEL_FIELDS: [&str; 18] = [ @@ -118,10 +121,7 @@ pub(crate) fn run_custom_hosts( let input_dir: PathBuf = input_dir.canonicalize()?; let stem: String = format!("project-{}", uuid_hex(py)?); let input_path: PathBuf = input_dir.join(format!("{stem}.pickle")); - let abandoned: PathBuf = input_dir.join(format!( - "{stem}{}", - constant::(py, "CUSTOM_HOST_ABANDONED_SUFFIX")? - )); + let abandoned: PathBuf = input_dir.join(format!("{stem}{}", CUSTOM_HOST_ABANDONED_SUFFIX)); let mut payload: Map = Map::new(); payload.insert("project_pickle_path".to_owned(), json!(text(&input_path))); payload.insert("abandoned_marker".to_owned(), json!(text(&abandoned))); @@ -169,8 +169,8 @@ impl Launch { ) -> PyResult { Ok(Self { program: imported(py, "sys", "executable")?.extract()?, - arguments: vec!["-m".to_owned(), constant(py, "CUSTOM_HOST_LAUNCH_MODULE")?], - runtime_version: constant(py, "CUSTOM_HOST_RUNTIME_VERSION")?, + arguments: vec!["-m".to_owned(), CUSTOM_HOST_LAUNCH_MODULE.to_owned()], + runtime_version: CUSTOM_HOST_RUNTIME_VERSION.to_owned(), payload, launches, }) @@ -221,7 +221,7 @@ fn run_partitions<'py>( .collect(); } let cancel_dir: PathBuf = unique_directory(input_dir, "hosts-")?; - let marker: PathBuf = cancel_dir.join(constant::(py, "CUSTOM_HOST_CANCELLED_MARKER")?); + let marker: PathBuf = cancel_dir.join(CUSTOM_HOST_CANCELLED_MARKER); let specs: Vec = plans .iter() .map(|partition| launch.spec(partition, Some(&marker))) @@ -432,10 +432,6 @@ fn cores(py: Python<'_>) -> PyResult { )) } -fn constant<'py, T: pyo3::FromPyObject<'py>>(py: Python<'py>, name: &str) -> PyResult { - imported(py, RULES_CONSTANTS_MODULE, name)?.extract() -} - fn uuid_hex(py: Python<'_>) -> PyResult { imported(py, "uuid", "uuid4")? .call0()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs index 211128b211..d935f4bc93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_cache.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, invalid_finding}; -const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_ERRORS_MODULE: &str = "sqlbuild._native"; const RULES_ERROR: &str = "RulesError"; /// One custom rule's cache identity from its metadata, source fingerprint and settings. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index fe1ab554a6..3c61e9b37e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -1,20 +1,30 @@ //! Custom-rule import closures, implementation fingerprints and statically found test cases. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyTuple; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_pysource::rule_evidence::main::allowed_custom_rule_import::allowed_custom_rule_import; +use sqlbuild_pysource::rule_evidence::main::disallowed_import::disallowed_import; use sqlbuild_pysource::rule_evidence::main::implementation_fingerprint::implementation_fingerprint; use sqlbuild_pysource::rule_evidence::main::import_closure::import_closure as import_closure_of; use sqlbuild_pysource::rule_evidence::main::test_evidence::test_evidence; -use sqlbuild_pysource::rule_evidence::models::RuleTestEvidence; +use sqlbuild_pysource::rule_evidence::models::{ + DisallowedImport, RuleTestEvidence, UnparsedSource, +}; use crate::bindings::_helpers::compile_driver::python_calls::imported; const PATHLIB_MODULE: &str = "pathlib"; const INSPECT_MODULE: &str = "inspect"; const HASHLIB_MODULE: &str = "hashlib"; +const AST_MODULE: &str = "ast"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULES_ERROR: &str = "RulesError"; +const NON_HERMETIC_RULE_ERROR: &str = "NonHermeticRuleError"; fn python_version(py: Python<'_>) -> (u8, u8) { let version = py.version_info(); @@ -120,7 +130,108 @@ fn custom_rule_test_evidence<'py>( .collect()) } +/// `error_class(message)` from the native module. +fn rules_error(py: Python<'_>, error_class: &str, message: String) -> PyErr { + match imported(py, NATIVE_MODULE, error_class).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Whether custom-rule code may import one absolute module name. +#[pyfunction] +#[pyo3(name = "allowed_custom_rule_import")] +fn allowed_custom_rule_import_binding(module: &str) -> bool { + allowed_custom_rule_import(module) +} + +/// Reject a repository-owned rule file that imports outside the allowlist. +fn verify_source(path: &Bound<'_, PyAny>, root: &Path) -> PyResult<()> { + let py: Python<'_> = path.py(); + if !native_path(path)?.starts_with(root) { + return Err(rules_error( + py, + RULES_ERROR, + format!( + "custom rule source must be repository-owned: {}", + path.str()? + ), + )); + } + let source: Bound<'_, PyAny> = path.call_method("read_text", (), Some(&utf8_encoding(py)?))?; + let text: String = source.extract()?; + match disallowed_import(&text, python_version(py)) { + Ok(None) => Ok(()), + Ok(Some(DisallowedImport { line, module })) => Err(rules_error( + py, + NON_HERMETIC_RULE_ERROR, + format!( + "non-hermetic custom rule at {}:{line}: import {} is not allowed", + path.str()?, + PyString::new(py, &module).repr()? + ), + )), + Err(UnparsedSource) => { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("filename", path.str()?)?; + imported(py, AST_MODULE, "parse")?.call((source,), Some(&keywords))?; + Err(rules_error( + py, + RULES_ERROR, + format!( + "custom rule source could not be checked for imports: {}", + path.str()? + ), + )) + } + } +} + +fn utf8_encoding(py: Python<'_>) -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("encoding", "utf-8")?; + Ok(keywords) +} + +/// Reject imports outside the deterministic allowlist in every fingerprinted rule file. +#[pyfunction] +#[pyo3(signature = (*, rules, project_dir))] +pub(crate) fn verify_custom_rules( + rules: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = rules.py(); + let root_path: Bound<'_, PyAny> = project_dir.call_method0("resolve")?; + let root: PathBuf = native_path(&root_path)?; + let checked: Bound<'_, PySet> = PySet::empty(py)?; + for rule in rules.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + if !rule.getattr("custom")?.is_truthy()? { + continue; + } + let Some(source) = resolved_source(&rule)? else { + continue; + }; + let closure: Vec = + import_closure_of(&native_path(&source)?, &root, python_version(py)); + let paths: Bound<'_, PyTuple> = closure_paths(py, closure)?; + for path in std::iter::once(source).chain(paths.iter()) { + if checked.contains(&path)? { + continue; + } + checked.add(&path)?; + verify_source(&path, &root)?; + } + } + Ok(()) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + allowed_custom_rule_import_binding, + module + )?)?; + module.add_function(wrap_pyfunction!(verify_custom_rules, module)?)?; module.add_function(wrap_pyfunction!(custom_rule_import_closure, module)?)?; module.add_function(wrap_pyfunction!( custom_rule_implementation_fingerprint, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs index 93fd0e3775..2fd567c7ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/findings.rs @@ -5,11 +5,11 @@ use pyo3::types::{PyDict, PyDictMethods}; use serde_json::Value; use sqlbuild_rules::models::Fault; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const FINDING_CLASS: &str = "Finding"; const PATHLIB_MODULE: &str = "pathlib"; const PATH_CLASS: &str = "Path"; -const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const RULES_ERROR: &str = "RulesError"; const INVALID_FINDING: &str = "native rules engine returned an invalid finding"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 3e9a741619..261eb0bf41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild._native"; const ADAPTER: &str = "sqlbuild.adapter.contract.classes.base_adapter"; -const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS: &str = "sqlbuild._native"; const LINT_ERRORS: &str = "sqlbuild._native"; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index 3e835c3457..9fde98d0f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const ADAPTER_EXCEPTIONS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; -const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index 44023b8dd8..4adab35a94 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -16,7 +16,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild._native"; -const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS: &str = "sqlbuild._native"; const LINT_ERRORS: &str = "sqlbuild._native"; /// Expand a single body and finish its native SQL preparation. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index a748bb5224..ccc75757da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -28,7 +28,7 @@ use crate::bindings::types::CompilerDetach; const LINT_MODELS_MODULE: &str = "sqlbuild._native"; const LINT_TYPES_MODULE: &str = "sqlbuild._native"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_ENGINE: &str = "native"; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 4b10fcadae..5c3ad0aa26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -23,10 +23,12 @@ pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; pub(crate) mod lint_stop; pub(crate) mod lint_text; +pub(crate) mod project_tree_queries; pub(crate) mod registration; pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; +pub(crate) mod rule_api; pub(crate) mod rule_cache; pub(crate) mod rule_dispatch; pub(crate) mod rule_plan; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs new file mode 100644 index 0000000000..62a6f2b4dd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs @@ -0,0 +1,180 @@ +//! Project-tree queries custom rules run over the discovered paths. + +use std::ffi::OsString; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::project_tree::errors::RelativePartsError; +use sqlbuild_rules::project_tree::main::project_relative_parts::project_relative_parts; +use sqlbuild_rules::project_tree::main::project_tree_children::project_tree_children; +use sqlbuild_rules::project_tree::main::project_tree_descendants::project_tree_descendants; +use sqlbuild_rules::project_tree::main::project_tree_glob::project_tree_glob; +use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::rules::rule_api::{usage_error_for, validate_project_relative}; + +const PATHLIB_MODULE: &str = "pathlib"; +const RULE_USAGE_ERROR: &str = "RuleUsageError"; +const NATIVE_MODULE: &str = "sqlbuild._native"; + +/// `pathlib.Path(value)`, raising exactly as `Path` does for an unsupported value. +fn python_path<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + imported(value.py(), PATHLIB_MODULE, "Path")?.call1((value,)) +} + +/// The nodes and their `value`s as raw paths. +fn node_values<'py>(nodes: &Bound<'py, PyAny>) -> PyResult<(Vec>, Vec)> { + let nodes: Vec> = nodes.try_iter()?.collect::>()?; + let values: Vec = nodes + .iter() + .map(|node| node.getattr("value")?.extract()) + .collect::>()?; + Ok((nodes, values)) +} + +/// The nodes at `indexes`, in order. +fn selected<'py>( + py: Python<'py>, + nodes: &[Bound<'py, PyAny>], + indexes: Vec, +) -> PyResult> { + PyTuple::new(py, indexes.into_iter().map(|index| nodes[index].clone())) +} + +/// The nodes directly inside the project directory `path`. +#[pyfunction] +fn project_tree_child_nodes<'py>( + nodes: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, +) -> PyResult> { + let directory: PathBuf = python_path(path)?.extract()?; + let (nodes, values) = node_values(nodes)?; + let values: Vec<&std::path::Path> = values.iter().map(PathBuf::as_path).collect(); + selected( + path.py(), + &nodes, + project_tree_children(&values, &directory), + ) +} + +/// The nodes below the project directory `path`, or every node for an empty path. +#[pyfunction] +fn project_tree_descendant_nodes<'py>( + nodes: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, +) -> PyResult> { + let directory: Option = if path.is_truthy()? { + Some(python_path(path)?.extract()?) + } else { + None + }; + let (nodes, values) = node_values(nodes)?; + let values: Vec<&std::path::Path> = values.iter().map(PathBuf::as_path).collect(); + selected( + path.py(), + &nodes, + project_tree_descendants(&values, directory.as_deref()), + ) +} + +/// The nodes a project-relative shell-style pattern matches. +#[pyfunction] +fn project_tree_glob_nodes<'py>( + nodes: &Bound<'py, PyAny>, + pattern: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = pattern.py(); + validate_project_relative(pattern)?; + let pattern: PathBuf = pattern.extract()?; + let (nodes, values) = node_values(nodes)?; + let values: Vec<&std::path::Path> = values.iter().map(PathBuf::as_path).collect(); + selected(py, &nodes, project_tree_glob(&values, &pattern)) +} + +/// The values of the file nodes rules may read as tracked text, in order. +#[pyfunction] +fn tracked_text_file_values<'py>(nodes: &Bound<'py, PyAny>) -> PyResult> { + let mut values: Vec> = Vec::new(); + for node in nodes.try_iter()? { + let node: Bound<'py, PyAny> = node?; + if !node.getattr("is_file")?.is_truthy()? { + continue; + } + let value: Bound<'py, PyAny> = node.getattr("value")?; + if tracked_text_path(&value.extract::()?) { + values.push(value); + } + } + PyTuple::new(nodes.py(), values) +} + +/// `RuleUsageError(f"project path {path} is not beneath {under}")`, spelled by Python. +fn not_beneath(path: &Bound<'_, PyAny>, under: &Bound<'_, PyAny>) -> PyErr { + let py: Python<'_> = path.py(); + let message = path.str().and_then(|path_text| { + let under_text = under.str()?; + PyString::new(py, "project path ") + .as_any() + .add(path_text)? + .add(" is not beneath ")? + .add(under_text) + }); + match message.and_then(|message| { + imported(py, NATIVE_MODULE, RULE_USAGE_ERROR).and_then(|class| class.call1((message,))) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `path`'s parts relative to the resolved project `root`, then to `under` when given. +#[pyfunction] +#[pyo3(signature = (*, root, path, under))] +fn project_tree_relative_parts<'py>( + root: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, + under: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = path.py(); + let relative: Bound<'py, PyAny> = python_path(path)?; + let under_path: Option = if under.is_truthy()? { + Some(python_path(under)?.extract()?) + } else { + None + }; + let parts: Vec = match project_relative_parts( + &relative.extract::()?, + &root.extract::()?, + under_path.as_deref(), + |_| relative.call_method0("resolve")?.extract::(), + ) { + Ok(parts) => parts, + Err(RelativePartsError::Resolve(error)) if !error.is_instance_of::(py) => { + return Err(error); + } + Err(RelativePartsError::Resolve(_) | RelativePartsError::Escapes) => { + return Err(usage_error_for("project path escapes the project: ", path)); + } + Err(RelativePartsError::NotBeneath) => return Err(not_beneath(path, under)), + }; + let parts: Vec> = parts + .into_iter() + .map(|part| Ok(part.into_pyobject(py)?.into_any())) + .collect::>()?; + PyTuple::new(py, parts) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(project_tree_child_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_descendant_nodes, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_glob_nodes, module)?)?; + module.add_function(wrap_pyfunction!(tracked_text_file_values, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_relative_parts, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index efb60e8500..802295076a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -7,11 +7,13 @@ use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_stop, - lint_text, results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, - sql_rules, + lint_text, results, rule_api, rule_cache, rule_plan, rule_run, rules_config, run_lint, + selection, sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + rule_api::register(module)?; + crate::bindings::_helpers::rules::project_tree_queries::register(module)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::lint_headers::lint_file_role, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index ec95af0ea3..dce80ccc23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -123,7 +123,7 @@ fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { /// The key row of a seed, source or hand-built model from its config values and schema entry. #[pyfunction(name = "relation_key_row")] #[pyo3(signature = (kind, name, values, entry))] -fn entry_relation_key_row( +pub(crate) fn entry_relation_key_row( kind: &str, name: String, values: &Bound<'_, PyDict>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs new file mode 100644 index 0000000000..415d36751c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs @@ -0,0 +1,257 @@ +//! Custom-rule API decisions: finding positions, harness option TOML and project-tree paths. + +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::{PyAttributeError, PyOSError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, +}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_config::toml::main::toml_literal::toml_literal; +use sqlbuild_config::toml::models::TomlLiteral; +use sqlbuild_rules::project_tree::errors::ProjectTreeFileError; +use sqlbuild_rules::project_tree::main::project_path_escapes::project_path_escapes; +use sqlbuild_rules::project_tree::main::project_tree_entries::project_tree_entries; +use sqlbuild_rules::project_tree::main::project_tree_file::project_tree_file; +use sqlbuild_rules::project_tree::main::tracked_text_suffixes::tracked_text_suffixes; +use sqlbuild_rules::project_tree::models::ProjectTreeEntry; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULE_USAGE_ERROR: &str = "RuleUsageError"; +const PROJECT_PATH_CLASS: &str = "ProjectPath"; +const BUILTINS_MODULE: &str = "builtins"; +const DEFAULT_POSITION: i64 = 1; +#[cfg(unix)] +const OS_MODULE: &str = "os"; +/// The error `os.lstat` raises for a path holding a NUL byte. +const EMBEDDED_NUL_MESSAGE: &str = "lstat: embedded null character in path"; + +/// `RuleUsageError(message)`. +fn usage_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, NATIVE_MODULE, RULE_USAGE_ERROR).and_then(|class| class.call1((message,))) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// `RuleUsageError(prefix + str(value))`, keeping the value's text exactly as Python spells it. +pub(crate) fn usage_error_for(prefix: &str, value: &Bound<'_, PyAny>) -> PyErr { + let py: Python<'_> = value.py(); + let message = value + .str() + .and_then(|text| PyString::new(py, prefix).as_any().add(text)); + match message.and_then(|message| { + imported(py, NATIVE_MODULE, RULE_USAGE_ERROR).and_then(|class| class.call1((message,))) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// An explicit position, or `int(getattr(node, attribute, 1) or 1)`. +fn position<'py>( + explicit: &Bound<'py, PyAny>, + node: &Bound<'py, PyAny>, + attribute: &str, +) -> PyResult> { + if !explicit.is_none() { + return Ok(explicit.clone()); + } + let py: Python<'py> = node.py(); + let value: Bound<'py, PyAny> = match node.getattr(attribute) { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + DEFAULT_POSITION.into_pyobject(py)?.into_any() + } + Err(error) => return Err(error), + }; + let value: Bound<'py, PyAny> = if value.is_truthy()? { + value + } else { + DEFAULT_POSITION.into_pyobject(py)?.into_any() + }; + imported(py, BUILTINS_MODULE, "int")?.call1((value,)) +} + +/// The `(line, column)` of a finding at the project-relative `path`, from `node` when unset. +#[pyfunction] +#[pyo3(signature = (*, path, line, column, node))] +fn finding_position<'py>( + path: &Bound<'py, PyAny>, + line: &Bound<'py, PyAny>, + column: &Bound<'py, PyAny>, + node: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = path.py(); + if escapes(path)? { + return Err(usage_error_for( + "finding path must be project-relative: ", + path, + )); + } + let line: Bound<'py, PyAny> = position(line, node, "line")?; + let column: Bound<'py, PyAny> = position(column, node, "column")?; + if line.lt(DEFAULT_POSITION)? || column.lt(DEFAULT_POSITION)? { + return Err(usage_error( + py, + "finding line and column must be positive".to_owned(), + )); + } + Ok((line, column)) +} + +/// One rule-option value as a TOML literal, or the harness's usage error. +fn option_literal(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(flag) = value.downcast::() { + return Ok(TomlLiteral::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(TomlLiteral::Integer(value.str()?.to_string())); + } + if let Ok(text) = value.downcast::() { + return Ok(TomlLiteral::Text(text.to_str()?.to_owned())); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(TomlLiteral::Array( + value + .try_iter()? + .map(|item| option_literal(&item?)) + .collect::>()?, + )); + } + Err(usage_error( + value.py(), + format!( + "RuleCase config value {} cannot be represented as a rule option", + value.repr()? + ), + )) +} + +/// The TOML text the rule harness writes for one `RuleCase` option value. +#[pyfunction] +fn rule_option_toml_value(value: &Bound<'_, PyAny>) -> PyResult { + Ok(toml_literal(&option_literal(value)?)) +} + +/// Reject a project path that is absolute or climbs out of the project. +#[pyfunction] +pub(crate) fn validate_project_relative(path: &Bound<'_, PyAny>) -> Result<(), PyErr> { + if escapes(path)? { + return Err(usage_error_for("project path escapes the project: ", path)); + } + Ok(()) +} + +/// Whether `str(path)` is absolute or climbs out of the project with `..`. +fn escapes(path: &Bound<'_, PyAny>) -> PyResult { + Ok(project_path_escapes(Path::new( + &*path.str()?.to_string_lossy(), + ))) +} + +/// The `OSError` `Path.is_file()` raises for a stat failure it does not ignore. +#[cfg(unix)] +fn unreadable(py: Python<'_>, path: PathBuf, errno: i32) -> PyResult { + let message: String = imported(py, OS_MODULE, "strerror")? + .call1((errno,))? + .extract()?; + let Ok(path) = path.into_os_string().into_pyobject(py); + Ok(PyOSError::new_err((errno, message, path.unbind()))) +} + +/// The `OSError` `Path.is_file()` raises for a Windows stat failure it does not ignore. +#[cfg(not(unix))] +fn unreadable(py: Python<'_>, path: PathBuf, code: i32) -> PyResult { + let detail: String = std::io::Error::from_raw_os_error(code).to_string(); + let detail: &str = detail + .strip_suffix(&format!(" (os error {code})")) + .unwrap_or(&detail) + .trim_end_matches(|character: char| character <= ' ' || character == '.'); + let Ok(path) = path.into_os_string().into_pyobject(py); + Ok(PyOSError::new_err(( + code, + detail.to_owned(), + path.unbind(), + code, + ))) +} + +/// Every supported path under the resolved project `root`, as sorted `ProjectPath`s. +#[pyfunction] +fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = root.py(); + let root: PathBuf = root.extract()?; + let class: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, PROJECT_PATH_CLASS)?; + let mut paths: Vec<(Bound<'py, PyAny>, bool)> = project_tree_entries(&root) + .into_iter() + .map(|ProjectTreeEntry { relative, is_file }| { + Ok(( + relative.into_os_string().into_pyobject(py)?.into_any(), + is_file, + )) + }) + .collect::>()?; + let mut failure: Option = None; + paths.sort_by(|left, right| { + left.0.compare(&right.0).unwrap_or_else(|error| { + failure.get_or_insert(error); + std::cmp::Ordering::Equal + }) + }); + if let Some(error) = failure { + return Err(error); + } + let nodes: Vec> = paths + .into_iter() + .map(|(value, is_file)| { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("value", value)?; + keywords.set_item("is_file", is_file)?; + keywords.set_item("is_dir", !is_file)?; + class.call((), Some(&keywords)) + }) + .collect::>()?; + PyTuple::new(py, nodes) +} + +/// The real file a rule may read at the project-relative `path` under the resolved `root`. +#[pyfunction] +fn project_tree_file_path<'py>( + root: &Bound<'py, PyAny>, + path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = root.py(); + let relative: PathBuf = path.str()?.extract()?; + let prefix: String = match project_tree_file(&root.extract::()?, &relative) { + Ok(file) => return Ok(file.into_os_string().into_pyobject(py)?.into_any()), + Err(ProjectTreeFileError::Escapes) => "project path escapes the project: ".to_owned(), + Err(ProjectTreeFileError::EmbeddedNul) => { + return Err(PyValueError::new_err(EMBEDDED_NUL_MESSAGE)); + } + Err(ProjectTreeFileError::Missing) => "project file does not exist: ".to_owned(), + Err(ProjectTreeFileError::Untracked) => format!( + "compiler rules input must be a tracked file type ({}): ", + tracked_text_suffixes().join(", ") + ), + Err(ProjectTreeFileError::Symlink) => "project file must not be a symlink: ".to_owned(), + Err(ProjectTreeFileError::Unreadable { path, errno }) => { + return Err(unreadable(py, path, errno)?); + } + }; + Err(usage_error_for(&prefix, path)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finding_position, module)?)?; + module.add_function(wrap_pyfunction!(rule_option_toml_value, module)?)?; + module.add_function(wrap_pyfunction!(validate_project_relative, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_paths, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_file_path, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 0d6f2b81a1..9ef1718718 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -18,18 +18,18 @@ use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; use crate::bindings::_helpers::rules::custom_evaluation::{ CustomRun, evaluate_custom_rules_cached, outcome, }; use crate::bindings::_helpers::rules::custom_hosts::HostCancel; +use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const RULES_MODELS_MODULE: &str = "sqlbuild._native"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; -const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; const CUSTOM_THREAD: &str = "sqlbuild-custom-rules"; const GENERIC_DIALECT: &str = "generic"; const PROJECT_RULES_OPTIONS: (&[&str], &[&str]) = ( @@ -164,10 +164,7 @@ fn evaluate_project_rules<'py>( ("project_dir", project_dir.clone()), ], )?; - call_with( - &imported(py, HERMETICITY_MODULE, "verify_custom_rules")?, - &[("rules", rules), ("project_dir", project_dir.clone())], - )?; + verify_custom_rules(&rules, &project_dir)?; } let run: NativeRun<'py> = NativeRun { project, @@ -532,11 +529,14 @@ fn native_request<'py>( )?, }; let mut include_type_proof: bool = false; - for code in imported(py, RULES_CONSTANTS_MODULE, "TYPE_PROOF_RULE_CODES")?.try_iter()? { + for code in TYPE_PROOF_RULE_CODES { include_type_proof = include_type_proof || call_with( &native(py, "rule_selected")?, - &[("config", run.config.clone()), ("code", code?)], + &[ + ("config", run.config.clone()), + ("code", pyo3::types::PyString::new(py, code).into_any()), + ], )? .is_truthy()?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index b7f202a1c5..9efa1652f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -10,13 +10,13 @@ use crate::bindings::_helpers::rules::catalogue::{ references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, verify_custom_rules, }; +use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const PATHLIB_MODULE: &str = "pathlib"; @@ -57,12 +57,10 @@ fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult = - imported(py, RULES_CONSTANTS_MODULE, "TYPE_PROOF_RULE_CODES")?; let mut kept: Vec> = Vec::with_capacity(selected.len()); for rule in selected { let code: Bound<'py, PyAny> = rule.getattr("code")?; - if type_proof.contains(&code)? { + if TYPE_PROOF_RULE_CODES.contains(&code.extract::()?.as_str()) { skipped_type_proof_rules.push(code); } else { kept.push(rule); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index f5de4a16c1..02667e3faf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const MODELS_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const CTYPES_MODULE: &str = "ctypes"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index a428a3a4c3..b6c0de35f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -14,8 +14,8 @@ use sqlbuild_rules::models::RulesConfig; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; /// The rules configuration models that stay Python for the custom-rule API. const PYTHON_CLASSES: &[&str] = &["RulesConfig", "LayoutConfig"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index 289a085c0e..1e98140149 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -11,7 +11,7 @@ use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::rules::constants::RULES_NATIVE_API_VERSION; -const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; +const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 6c483d569b..d8cf5699fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const RULES_MODELS_MODULE: &str = "sqlbuild._native"; const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs index 486212aaf4..67c07cfe32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs @@ -25,7 +25,7 @@ const FAIL_STATUS: &str = "FAIL"; const RUN_DIR: &str = "run"; const SCENARIOS_DIR: &str = "scenarios"; const LOCAL_DIR: &str = "local"; -const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES: &str = "sqlbuild._native"; const QUALIFIED_NAME_MODULE: &str = "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs new file mode 100644 index 0000000000..1e77649b84 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/constants.rs @@ -0,0 +1,451 @@ +//! Layouts of the declaration scope value records hosted natively. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +const R_RESOURCE_IDENTITY: RecordSpec = RecordSpec { + name: "ResourceIdentity", + doc: "Stable kind-qualified identity of one authored resource.", + fields: &[FieldSpec::required("kind"), FieldSpec::required("name")], +}; +const R_DECLARATION_IDENTITY: RecordSpec = RecordSpec { + name: "DeclarationIdentity", + doc: "Stable declaration identity, including a private declaration owner.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("name"), + FieldSpec::new("owner", FieldDefault::None), + ], +}; +const R_OWNERSHIP_ROOT: RecordSpec = RecordSpec { + name: "OwnershipRoot", + doc: "Normalized project-relative directory that owns resource scope.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::new( + "kind", + FieldDefault::Member("sqlbuild._native", "OwnershipRootKind", "RESOURCE"), + ), + FieldSpec::new("resource_kind", FieldDefault::None), + ], +}; +const R_RESOURCE_RECORD: RecordSpec = RecordSpec { + name: "ResourceRecord", + doc: "One resource participating in lexical declaration scope.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("path"), + FieldSpec::required("ownership_root"), + ], +}; +const R_MACRO_METADATA: RecordSpec = RecordSpec { + name: "MacroMetadata", + doc: "Safe macro metadata required by scope and dependency inspection.", + fields: &[ + FieldSpec::new("parameters", FieldDefault::EmptyTuple), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("source_digest", FieldDefault::Text("")), + ], +}; +const R_ENUM_MEMBER_METADATA: RecordSpec = RecordSpec { + name: "EnumMemberMetadata", + doc: "One value-free authored enum member identity.", + fields: &[FieldSpec::required("name")], +}; +const R_ENUM_METADATA: RecordSpec = RecordSpec { + name: "EnumMetadata", + doc: "Safe enum metadata containing member names and their shared scalar type.", + fields: &[ + FieldSpec::required("members"), + FieldSpec::required("scalar_type"), + ], +}; +const R_CONSTANT_METADATA: RecordSpec = RecordSpec { + name: "ConstantMetadata", + doc: "Value-free normalized metadata for one typed constant.", + fields: &[ + FieldSpec::required("logical_type"), + FieldSpec::new("collection_kind", FieldDefault::None), + FieldSpec::new("item_count", FieldDefault::None), + FieldSpec::new("nullable", FieldDefault::False), + FieldSpec::new("render_as", FieldDefault::None), + ], +}; +const R_DECLARATION_RECORD: RecordSpec = RecordSpec { + name: "DeclarationRecord", + doc: "One declaration definition and its compiler-owned scope metadata.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + FieldSpec::required("scope"), + FieldSpec::required("ownership_root"), + FieldSpec::new("owning_path", FieldDefault::None), + FieldSpec::new("macro", FieldDefault::None), + FieldSpec::new("enum", FieldDefault::None), + FieldSpec::new("constant", FieldDefault::None), + ], +}; +const R_USAGE_RECORD: RecordSpec = RecordSpec { + name: "UsageRecord", + doc: "A runtime or declaration dependency edge.", + fields: &[ + FieldSpec::required("consumer"), + FieldSpec::required("declaration"), + FieldSpec::new( + "kind", + FieldDefault::Member("sqlbuild._native", "UsageKind", "RUNTIME"), + ), + FieldSpec::new("through", FieldDefault::None), + FieldSpec::new("enum_member", FieldDefault::None), + ], +}; +const R_GRANT_RECORD: RecordSpec = RecordSpec { + name: "GrantRecord", + doc: "A compiler relationship granting a public declaration to a resource.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("declaration"), + FieldSpec::required("through"), + FieldSpec::new( + "kind", + FieldDefault::Member("sqlbuild._native", "GrantKind", "EXPECTED_MODEL"), + ), + ], +}; +const R_VISIBILITY_RECORD: RecordSpec = RecordSpec { + name: "VisibilityRecord", + doc: "A resolved positive visibility fact with stable provenance.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("declaration"), + FieldSpec::required("reason"), + FieldSpec::new("through", FieldDefault::None), + ], +}; +const R_INACCESSIBLE_RECORD: RecordSpec = RecordSpec { + name: "InaccessibleRecord", + doc: "A resolved negative visibility fact with stable provenance.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("declaration"), + FieldSpec::required("reason"), + ], +}; +const R_SCOPE_DIAGNOSTIC: RecordSpec = RecordSpec { + name: "ScopeDiagnostic", + doc: "Stable diagnostic retained even when an index is partial.", + fields: &[ + FieldSpec::required("code"), + FieldSpec::required("message"), + FieldSpec::new( + "severity", + FieldDefault::Member("sqlbuild._native", "ScopeDiagnosticSeverity", "ERROR"), + ), + FieldSpec::new("path", FieldDefault::None), + FieldSpec::new("line", FieldDefault::None), + FieldSpec::new("column", FieldDefault::None), + FieldSpec::new("declaration", FieldDefault::None), + FieldSpec::new("resource", FieldDefault::None), + ], +}; +const R_SCOPE_COMPLETENESS: RecordSpec = RecordSpec { + name: "ScopeCompleteness", + doc: "Independent completeness facts for partially invalid projects.", + fields: &[ + FieldSpec::new("discovery", FieldDefault::True), + FieldSpec::new("static_visibility", FieldDefault::True), + FieldSpec::new("runtime_usage", FieldDefault::True), + FieldSpec::new("relationships", FieldDefault::True), + FieldSpec::new("placement", FieldDefault::True), + FieldSpec::new("promotion_impact", FieldDefault::True), + ], +}; +const R_SCOPE_INDEX: RecordSpec = RecordSpec { + name: "ScopeIndex", + doc: "Canonical deterministic facts used by every future scope consumer.", + fields: &[ + FieldSpec::new("ownership_roots", FieldDefault::EmptyTuple), + FieldSpec::new("resources", FieldDefault::EmptyTuple), + FieldSpec::new("declarations", FieldDefault::EmptyTuple), + FieldSpec::new("usages", FieldDefault::EmptyTuple), + FieldSpec::new("grants", FieldDefault::EmptyTuple), + FieldSpec::new("visibility", FieldDefault::EmptyTuple), + FieldSpec::new("inaccessible", FieldDefault::EmptyTuple), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new( + "completeness", + FieldDefault::Call("sqlbuild._native", "ScopeCompleteness"), + ), + ], +}; +const R_DECLARATION_VISIBILITY_INDEX: RecordSpec = RecordSpec { + name: "DeclarationVisibilityIndex", + doc: "Declaration positions grouped by the scope facts that make them visible.", + fields: &[ + FieldSpec::required("identities"), + FieldSpec::required("global_positions"), + FieldSpec::required("private_positions"), + FieldSpec::required("local_positions"), + FieldSpec::required("inherited_positions"), + ], +}; +const R_SCOPE_LOOKUP: RecordSpec = RecordSpec { + name: "ScopeLookup", + doc: "Immutable indexes over one canonical ``ScopeIndex``.", + fields: &[ + FieldSpec::required("index"), + FieldSpec::required("resources"), + FieldSpec::required("resources_by_path"), + FieldSpec::required("declarations"), + FieldSpec::required("usages_by_consumer"), + FieldSpec::required("usages_by_declaration"), + FieldSpec::required("grants_by_resource"), + FieldSpec::required("visibility_index"), + FieldSpec { + name: "native", + default: FieldDefault::None, + compared: false, + shown: false, + }, + ], +}; +const R_SCOPE_TARGET_QUERY: RecordSpec = RecordSpec { + name: "ScopeTargetQuery", + doc: "Non-throwing qualified-identity or path lookup result.", + fields: &[ + FieldSpec::required("value"), + FieldSpec::new("matches", FieldDefault::EmptyTuple), + FieldSpec::new("declaration_matches", FieldDefault::EmptyTuple), + ], +}; +const R_VISIBILITY_RESOLUTION: RecordSpec = RecordSpec { + name: "VisibilityResolution", + doc: "Complete static visibility classification for one queried target.", + fields: &[ + FieldSpec::required("target"), + FieldSpec::new("visible", FieldDefault::EmptyTuple), + FieldSpec::new("inaccessible", FieldDefault::EmptyTuple), + ], +}; +const R_DECLARATION_VISIBILITY: RecordSpec = RecordSpec { + name: "DeclarationVisibility", + doc: "Visible facts and inaccessible declaration identities for one queried target.", + fields: &[ + FieldSpec::required("target"), + FieldSpec::new("visible", FieldDefault::EmptyTuple), + FieldSpec::new("inaccessible", FieldDefault::EmptyTuple), + ], +}; +const R_SCOPE_REPORT_FILTERS: RecordSpec = RecordSpec { + name: "ScopeReportFilters", + doc: "Orthogonal declaration filters applied in their documented fixed order.", + fields: &[ + FieldSpec::new( + "sections", + FieldDefault::TextTuple(&["available", "used", "relationship_scope"]), + ), + FieldSpec::new("include_nearby", FieldDefault::False), + FieldSpec::new("defined_under", FieldDefault::None), + FieldSpec::new("kinds", FieldDefault::EmptyTuple), + FieldSpec::new("glob", FieldDefault::None), + FieldSpec::new("used_only", FieldDefault::False), + FieldSpec::new("dependency_depth", FieldDefault::Int(0)), + FieldSpec::new("sort", FieldDefault::Text("identity")), + FieldSpec::new("cursor", FieldDefault::None), + FieldSpec::new("page_size", FieldDefault::Int(100)), + FieldSpec::new("nearby_depth", FieldDefault::Int(1)), + FieldSpec::new("globals", FieldDefault::Text("summary")), + ], +}; +const R_SCOPE_SOURCE_LOCATION: RecordSpec = RecordSpec { + name: "ScopeSourceLocation", + doc: "Safe project-relative source location.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("line"), + FieldSpec::required("column"), + ], +}; +const R_VISIBILITY_PROVENANCE: RecordSpec = RecordSpec { + name: "VisibilityProvenance", + doc: "Why a declaration is visible or inaccessible.", + fields: &[ + FieldSpec::required("reason"), + FieldSpec::new("through", FieldDefault::None), + ], +}; +const R_DECLARATION_REPORT: RecordSpec = RecordSpec { + name: "DeclarationReport", + doc: "Value-free declaration projection used by every report section.", + fields: &[ + FieldSpec::required("identity"), + FieldSpec::required("kind"), + FieldSpec::required("name"), + FieldSpec::required("owner"), + FieldSpec::required("definition"), + FieldSpec::required("scope"), + FieldSpec::required("owning_path"), + FieldSpec::new("visibility", FieldDefault::None), + FieldSpec::new("inaccessible_reason", FieldDefault::None), + FieldSpec::new("metadata", FieldDefault::EmptyTuple), + FieldSpec::new("consumers", FieldDefault::EmptyTuple), + FieldSpec::new("dependencies", FieldDefault::EmptyTuple), + FieldSpec::new("grants", FieldDefault::EmptyTuple), + FieldSpec::new("required_scope", FieldDefault::None), + FieldSpec::new("required_path", FieldDefault::None), + FieldSpec::new("promotion_impact", FieldDefault::EmptyTuple), + ], +}; +const R_SCOPE_CHAIN_ENTRY: RecordSpec = RecordSpec { + name: "ScopeChainEntry", + doc: "One exact or inherited lexical scope in deterministic path order.", + fields: &[ + FieldSpec::required("kind"), + FieldSpec::required("path"), + FieldSpec::required("declaration_count"), + ], +}; +const R_SCOPE_SECTION: RecordSpec = RecordSpec { + name: "ScopeSection", + doc: "Pagination and completeness metadata for one report section.", + fields: &[ + FieldSpec::required("name"), + FieldSpec::required("total"), + FieldSpec::required("returned"), + FieldSpec::new("collapsed", FieldDefault::False), + FieldSpec::new("collapsed_count", FieldDefault::Int(0)), + FieldSpec::new("truncated", FieldDefault::False), + FieldSpec::new("complete", FieldDefault::True), + FieldSpec::new("next_cursor", FieldDefault::None), + FieldSpec::new("cursor", FieldDefault::None), + FieldSpec::new("page_size", FieldDefault::Int(100)), + ], +}; +const R_SCOPE_RESOURCE_REPORT: RecordSpec = RecordSpec { + name: "ScopeResourceReport", + doc: "Resolved existing resource or prospective authored path.", + fields: &[ + FieldSpec::required("target"), + FieldSpec::required("identity"), + FieldSpec::required("path"), + FieldSpec::new("prospective", FieldDefault::False), + FieldSpec::new("directory", FieldDefault::False), + FieldSpec::new("duplicate_count", FieldDefault::Int(0)), + ], +}; +const R_DECLARATION_EXPLANATION: RecordSpec = RecordSpec { + name: "DeclarationExplanation", + doc: "Detailed explanation for one qualified declaration.", + fields: &[ + FieldSpec::required("declaration"), + FieldSpec::required("complete"), + ], +}; +const R_MOVE_PREVIEW: RecordSpec = RecordSpec { + name: "MovePreview", + doc: "Pure visibility delta for moving one existing resource.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::required("destination"), + FieldSpec::required("new_ownership_root"), + FieldSpec::new("retained", FieldDefault::EmptyTuple), + FieldSpec::new("gained", FieldDefault::EmptyTuple), + FieldSpec::new("lost", FieldDefault::EmptyTuple), + FieldSpec::new("invalidated_usages", FieldDefault::EmptyTuple), + FieldSpec::new("private_retained", FieldDefault::EmptyTuple), + FieldSpec::new("relationship_retained", FieldDefault::EmptyTuple), + FieldSpec::new("complete", FieldDefault::True), + ], +}; +const R_SCOPE_FOLDER: RecordSpec = RecordSpec { + name: "ScopeFolder", + doc: "One declaration-definition folder with exact recursive counts.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("name"), + FieldSpec::required("descendant_count"), + FieldSpec::required("child_count"), + FieldSpec::required("used_count"), + FieldSpec::required("kind_counts"), + ], +}; +const R_SCOPE_BROWSE_RESULT: RecordSpec = RecordSpec { + name: "ScopeBrowseResult", + doc: "Folder-first browse result.", + fields: &[ + FieldSpec::required("folder"), + FieldSpec::required("folders"), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("complete", FieldDefault::True), + ], +}; +const R_SCOPE_LIST_RESULT: RecordSpec = RecordSpec { + name: "ScopeListResult", + doc: "Recursive declaration list beneath one definition folder.", + fields: &[ + FieldSpec::required("folder"), + FieldSpec::required("declarations"), + FieldSpec::required("section"), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + ], +}; +const R_SCOPE_REPORT: RecordSpec = RecordSpec { + name: "ScopeReport", + doc: "Canonical payload shared by all non-rendering scope consumers.", + fields: &[ + FieldSpec::required("resource"), + FieldSpec::new("scope_chain", FieldDefault::EmptyTuple), + FieldSpec::new("available", FieldDefault::EmptyTuple), + FieldSpec::new("used", FieldDefault::EmptyTuple), + FieldSpec::new("relationship_scope", FieldDefault::EmptyTuple), + FieldSpec::new("nearby_unavailable", FieldDefault::EmptyTuple), + FieldSpec::new( + "filters", + FieldDefault::Call("sqlbuild._native", "ScopeReportFilters"), + ), + FieldSpec::new("sections", FieldDefault::EmptyTuple), + FieldSpec::new("explanation", FieldDefault::None), + FieldSpec::new("move_preview", FieldDefault::None), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + FieldSpec::new("complete", FieldDefault::True), + ], +}; + +/// The declaration scope value records. +pub(crate) const SCOPE_RECORDS: &[RecordSpec] = &[ + R_RESOURCE_IDENTITY, + R_DECLARATION_IDENTITY, + R_OWNERSHIP_ROOT, + R_RESOURCE_RECORD, + R_MACRO_METADATA, + R_ENUM_MEMBER_METADATA, + R_ENUM_METADATA, + R_CONSTANT_METADATA, + R_DECLARATION_RECORD, + R_USAGE_RECORD, + R_GRANT_RECORD, + R_VISIBILITY_RECORD, + R_INACCESSIBLE_RECORD, + R_SCOPE_DIAGNOSTIC, + R_SCOPE_COMPLETENESS, + R_SCOPE_INDEX, + R_DECLARATION_VISIBILITY_INDEX, + R_SCOPE_LOOKUP, + R_SCOPE_TARGET_QUERY, + R_VISIBILITY_RESOLUTION, + R_DECLARATION_VISIBILITY, + R_SCOPE_REPORT_FILTERS, + R_SCOPE_SOURCE_LOCATION, + R_VISIBILITY_PROVENANCE, + R_DECLARATION_REPORT, + R_SCOPE_CHAIN_ENTRY, + R_SCOPE_SECTION, + R_SCOPE_RESOURCE_REPORT, + R_DECLARATION_EXPLANATION, + R_MOVE_PREVIEW, + R_SCOPE_FOLDER, + R_SCOPE_BROWSE_RESULT, + R_SCOPE_LIST_RESULT, + R_SCOPE_REPORT, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs new file mode 100644 index 0000000000..61d449b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scope_records/mod.rs @@ -0,0 +1 @@ +pub(crate) mod constants; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 40e1cd71cf..ec33f64641 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -21,9 +21,9 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_message; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; -const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs index c8fe07474e..293558c237 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs @@ -34,7 +34,6 @@ struct PositionRow(Py, Py, Py, u8, Py, u8, u32); pub(crate) struct ContextClasses { visibility_record: Py, pub(crate) resolution_context: Py, - allocate: Py, reasons: Py, } @@ -293,7 +292,7 @@ impl NativeDeclarationContexts { ) } - /// A `VisibilityRecord` built without running its dataclass initializer. + /// A `VisibilityRecord` of `resource`, `declaration`, `reason` and `through`. pub(crate) fn visibility_record_for<'py>( &self, resource: &Bound<'py, PyAny>, @@ -303,13 +302,7 @@ impl NativeDeclarationContexts { ) -> PyResult> { let py: Python<'py> = resource.py(); let class: &Bound<'py, PyAny> = self.classes.visibility_record.bind(py); - let record: Bound<'py, PyAny> = self.classes.allocate.bind(py).call1((class,))?; - let attributes: Bound<'py, PyDict> = - record.getattr("__dict__")?.downcast_into::()?; - attributes.set_item("resource", resource)?; - attributes.set_item("declaration", declaration)?; - attributes.set_item("reason", reason)?; - attributes.set_item("through", through)?; + let record: Bound<'py, PyAny> = class.call1((resource, declaration, reason, through))?; Ok(record) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs index dcfde4fa8c..9ac4b234e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_names.rs @@ -18,7 +18,7 @@ use crate::bindings::types::CompilerDetach; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; /// One scan's error, or its values. type Scanned = (Option, Vec); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index b73006f8a1..817073f23a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::scopes::scope_reports::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const TYPES_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs index 76dcf00193..56c42b1674 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; const COLLECTOR_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const ERROR_SEVERITY: &str = "error"; /// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index a0b8ce015d..53daffc840 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -25,7 +25,7 @@ const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; -const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index 7ac27f2348..ffeafab18c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::scopes::placement::{ }; use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; -const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; -const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const MODELS_MODULE: &str = "sqlbuild._native"; +const TYPES_MODULE: &str = "sqlbuild._native"; const ENUM_MODULE: &str = "enum"; /// The scope model and enum modules every built object comes from. @@ -670,7 +670,7 @@ pub(crate) fn diagnostic_object<'py>( classes.model("ScopeDiagnostic")?.call1(( classes.member("ScopeDiagnosticCode", &item.code)?, item.message.as_str(), - classes.member("DiagnosticSeverity", &item.severity)?, + classes.member("ScopeDiagnosticSeverity", &item.severity)?, item.path.as_deref(), item.line, item.column, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs index 4198c9a939..bbfbf1599f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -224,7 +224,7 @@ fn declaration_object<'py>( report.kind.into_bound_py_any(py)?, report.name.into_bound_py_any(py)?, report.owner.into_bound_py_any(py)?, - classes.model("SourceLocation")?.call1(( + classes.model("ScopeSourceLocation")?.call1(( report.definition.path, report.definition.line, report.definition.column, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index c63490bee2..292be2f619 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::scopes::scope_objects::{ use crate::bindings::types::CompilerDetach; const DATACLASSES_MODULE: &str = "dataclasses"; -const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild._native"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild._native"; const RUNTIME_USAGE: usize = 2; /// Compile input groups whose items carry declaration usages, after the model inputs. @@ -219,7 +219,7 @@ fn placement_validated_index<'py>( } let classes: ScopeClasses<'py> = ScopeClasses::new(py)?; let severity: Bound<'py, PyAny> = classes.member( - "DiagnosticSeverity", + "ScopeDiagnosticSeverity", if enforce_placement { "error" } else { @@ -379,7 +379,7 @@ pub(crate) fn validate_scope_index(index: &Bound<'_, PyAny>) -> Result<(), PyErr let py: Python<'_> = index.py(); let error: Bound<'_, PyAny> = py .import(SCOPE_TYPES_MODULE)? - .getattr("DiagnosticSeverity")? + .getattr("ScopeDiagnosticSeverity")? .getattr("ERROR")?; let mut errors: Vec> = Vec::new(); for item in index.getattr("diagnostics")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index d781c15a56..974a648c5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -17,9 +17,9 @@ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; const BUILT_IN_AUDIT_ROOT: &str = ""; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index fe679be210..7c2a2137b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -6,17 +6,17 @@ use pyo3::types::{PyDict, PyDictMethods, PyString}; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: &str = "P009"; const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; -const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild._native"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const TYPE_RECOVERY_MODULE: &str = "sqlbuild._native"; const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const NORMALIZE_TYPE_MODULE: &str = "sqlbuild.adapter.type_system.main.normalize_type"; const DATACLASSES_MODULE: &str = "dataclasses"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; /// Imported once per stage run. pub(crate) struct StageClasses<'py> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 0b3d10ea2e..468a2995fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -10,13 +10,15 @@ use crate::bindings::_helpers::analysis::binding_catalog::{ use crate::bindings::_helpers::boundary::keyword_request::Request; const ADAPTER_MODELS_MODULE: &str = "sqlbuild.adapter.contract.models"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const UNKNOWN: &str = "UNKNOWN"; const GENERIC_DIALECT: &str = "generic"; -const VALID_FROM_DEFAULT: &str = "valid_from"; -const VALID_TO_DEFAULT: &str = "valid_to"; +/// Snapshot validity start column when the model names none. +pub(crate) const VALID_FROM_DEFAULT: &str = "valid_from"; +/// Snapshot validity end column when the model names none. +pub(crate) const VALID_TO_DEFAULT: &str = "valid_to"; const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; const UDF_KIND: &str = "udf"; const TABLE_FUNCTION_KIND: &str = "table_fn"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index db1e80a2bd..f05b78d6f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -24,7 +24,7 @@ const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const DATACLASSES_MODULE: &str = "dataclasses"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 2649457aa3..f9711d0da0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -38,7 +38,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const TEST_PLAN_MODULE: &str = "sqlbuild._native"; -const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; +const PLANNER_ERRORS: &str = "sqlbuild._native"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const DECIMAL_KIND: &str = "decimal"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs index d0adb4dbcd..f1b7ed40f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs @@ -14,7 +14,7 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; const MODEL_MODE: &str = "model"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 96bbfa3179..fd84264b42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -10,10 +10,10 @@ use crate::bindings::_helpers::sql_test_glue::assembly::{ AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, }; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const GLUE_MODELS_MODULE: &str = "sqlbuild._native"; -const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SPEC_MODELS_MODULE: &str = "sqlbuild._native"; /// Fields a direct-logic compiled payload copies from its input payload, in order. const DIRECT_PAYLOAD_FIELDS: [&str; 5] = [ "mode", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index 5b41fbb0ec..463bd380c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -4,7 +4,7 @@ pub(super) const UDF: &str = "udf"; pub(super) const TABLE_FN: &str = "table_fn"; pub(super) const ERROR_SEVERITY: &str = "error"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -pub(super) const PLAN_MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(super) const PLAN_MODELS: &str = "sqlbuild._native"; pub(super) const NATIVE_PLANNING: &str = "sqlbuild._native"; pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ ("model", "__ref__"), @@ -13,7 +13,7 @@ pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ ("sql_test", "__expected__"), ]; pub(super) const CURSOR_ALGEBRA_EXCEPTIONS: &str = "sqlbuild.cursor_algebra.exceptions"; -pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +pub(super) const PLANNER_TYPES: &str = "sqlbuild._native"; pub(super) const INTEGER_CURSOR_TYPE: &str = "integer"; pub(super) const TIMESTAMP_CURSOR_TYPE: &str = "timestamp"; pub(super) const DEFAULT_TIMESTAMP_START: &str = "1900-01-01 00:00:00"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index 92028ee3d5..4652f20271 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -151,12 +151,8 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult( @@ -79,25 +79,16 @@ pub(super) fn planning_error( "CompileInputError" }; return Ok(PyErr::from_value( - imported(py, "sqlbuild.compiler.compile.exceptions", class)? - .call((message,), Some(&kwargs))?, + imported(py, "sqlbuild._native", class)?.call((message,), Some(&kwargs))?, )); } let (module, class, message) = if let Some(message) = message.strip_prefix("compile_input:") { - ( - "sqlbuild.compiler.compile.exceptions", - "CompileInputError", - message.to_owned(), - ) + ("sqlbuild._native", "CompileInputError", message.to_owned()) } else if let Some(message) = message.strip_prefix("planner_input:") { - ( - "sqlbuild.compiler.planner.exceptions", - "PlannerInputError", - message.to_owned(), - ) + ("sqlbuild._native", "PlannerInputError", message.to_owned()) } else { ( - "sqlbuild.compiler.planner.exceptions", + "sqlbuild._native", "NativeSqlTestPlanningError", format!("native SQL-test planning failed: {message}"), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs index f39de53e1e..1a4a6e7599 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rendering.rs @@ -160,12 +160,8 @@ pub(super) fn functions<'py>( .filter(|(_, suffix)| !suffix.contains(SENTINEL)) else { return Err(PyErr::from_value( - imported( - py, - "sqlbuild.compiler.planner.exceptions", - "NativeSqlTestPlanningError", - )? - .call1(("adapter SQL function call template is not deterministic",))?, + imported(py, "sqlbuild._native", "NativeSqlTestPlanningError")? + .call1(("adapter SQL function call template is not deterministic",))?, )); }; fields.push(string(py, prefix)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index f8a5141d1d..77659c1912 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -7,7 +7,7 @@ use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; use sqlbuild_sqltext::compiler::models::HeaderWord; const SQL_VALUES_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild._native"; fn word_to_python(py: Python<'_>, word: String) -> PyResult> { match header_word_value(&word) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs index cf4ea638ed..594654873d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/code_scanner.rs @@ -12,7 +12,7 @@ use sqlbuild_sqltext::sql_scan::models::Unclosed; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; const COMPILE_INPUT_ERROR: &str = "CompileInputError"; /// One SQL text scanned under an adapter's lexical rules, or the generic compiler rules. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs new file mode 100644 index 0000000000..3b48467554 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/ddl_effects.rs @@ -0,0 +1,73 @@ +//! Executed-statement effects on the executor's cached relation metadata. + +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBool, PyFrozenSet}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_sqltext::ddl_effects::main::qualified_relation_name_key::qualified_relation_name_key as native_qualified_relation_name_key; +use sqlbuild_sqltext::ddl_effects::main::relation_name_key::relation_name_key as native_relation_name_key; +use sqlbuild_sqltext::ddl_effects::main::statement_metadata_effect::statement_metadata_effect as native_statement_metadata_effect; +use sqlbuild_sqltext::ddl_effects::models::StatementMetadataEffect; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const EFFECT_MODEL: (&str, &str) = ( + "sqlbuild.adapter.relations.models", + "StatementMetadataEffect", +); + +static EFFECT_CLASS: PyOnceLock> = PyOnceLock::new(); + +/// Which cached relation names `sql` may have changed, as a `StatementMetadataEffect`. +#[pyfunction] +#[pyo3(signature = (sql, /))] +fn statement_metadata_effect<'py>(py: Python<'py>, sql: &str) -> PyResult> { + let effect: StatementMetadataEffect = py.detach(|| native_statement_metadata_effect(sql)); + let class: &Bound<'py, PyAny> = EFFECT_CLASS + .get_or_try_init(py, || { + imported(py, EFFECT_MODEL.0, EFFECT_MODEL.1).map(Bound::unbind) + })? + .bind(py); + call_with( + class, + &[ + ( + "relation_names", + PyFrozenSet::new(py, &effect.relation_names)?.into_any(), + ), + ( + "invalidates_all", + PyBool::new(py, effect.invalidates_all) + .to_owned() + .into_any(), + ), + ( + "ends_transaction", + PyBool::new(py, effect.ends_transaction) + .to_owned() + .into_any(), + ), + ], + ) +} + +/// The invalidation key for one unquoted or quoted unqualified relation name. +#[pyfunction] +#[pyo3(signature = (name, /))] +fn relation_name_key(name: &str) -> String { + native_relation_name_key(name) +} + +/// The key for the last part of a qualified name, or `None` when it cannot be parsed. +#[pyfunction] +#[pyo3(signature = (qualified, /))] +fn qualified_relation_name_key(qualified: &str) -> Option { + native_qualified_relation_name_key(qualified) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(statement_metadata_effect, module)?)?; + module.add_function(wrap_pyfunction!(relation_name_key, module)?)?; + module.add_function(wrap_pyfunction!(qualified_relation_name_key, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs index 52ccc56e39..670af7fcaa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod authored_values; pub(crate) mod code_scanner; +pub(crate) mod ddl_effects; pub(crate) mod declaration_references; pub(crate) mod lexical_syntax; pub(crate) mod model_headers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs index 2b9fdf77b5..655c754951 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::sqltext::{ - code_scanner, model_headers, placeholders, sql_references, + code_scanner, ddl_effects, model_headers, placeholders, sql_references, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -11,5 +11,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { code_scanner::register(module)?; sql_references::register(module)?; placeholders::register(module)?; + ddl_effects::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index 5a1046c80b..c74782f322 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -18,10 +18,10 @@ use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; const NATIVE_RECORDS_MODULE: &str = "sqlbuild._native"; -const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild._native"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild._native"; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs index 7bdc577590..2d5507f85e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -1,11 +1,9 @@ //! Python modules, names and messages of migration state rows. -pub(crate) const MIGRATION_MODELS: &str = "sqlbuild.compiler.migrations.models"; -pub(crate) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; +pub(crate) const MIGRATION_MODELS: &str = "sqlbuild._native"; +pub(crate) const MIGRATION_TYPES: &str = "sqlbuild._native"; pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild._native"; -pub(crate) const STATE_TABLE_DDL: &str = - "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; -pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild.sql_values.types"; +pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild._native"; pub(crate) const OLD_NAME_VIEW_CONFIG_KEY: &str = "old_name_view"; pub(crate) const MODEL_KIND: &str = "model"; pub(crate) const COLUMN_KIND: &str = "column"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs index 732a2326b3..4116afb30d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_io.rs @@ -6,7 +6,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyString, PyTuple, PyTupleMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_state::events::constants::OLD_NAME_VIEW_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::main::recorded_renames::recorded_renames; use sqlbuild_state::events::models::RecordedRenameKey; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs index 43cec94da4..5487870783 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_objects.rs @@ -3,9 +3,11 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyString; +use sqlbuild_core::constants::COLUMN_MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::MIGRATION_TABLE_NAME; +use sqlbuild_core::constants::OLD_NAME_VIEW_TABLE_NAME; use sqlbuild_state::events::constants::{ - COLUMN_MIGRATION_COLUMNS, COLUMN_MIGRATION_TABLE_NAME, MIGRATION_COLUMNS, MIGRATION_TABLE_NAME, - OLD_NAME_VIEW_COLUMNS, OLD_NAME_VIEW_TABLE_NAME, + COLUMN_MIGRATION_COLUMNS, MIGRATION_COLUMNS, OLD_NAME_VIEW_COLUMNS, }; use sqlbuild_state::events::models::EventOrder; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs index 9cbe53d4e2..737dc4d356 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/migration_rows.rs @@ -20,13 +20,14 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::state_rows::constants::{ - MIGRATION_MODELS, MIGRATION_TYPES, SQL_VALUE_TYPES, STATE_TABLE_DDL, + MIGRATION_MODELS, MIGRATION_TYPES, SQL_VALUE_TYPES, }; use crate::bindings::_helpers::state_rows::events::typed_insert_sql; use crate::bindings::_helpers::state_rows::migration_objects::{ layout, qualified_table, state_error, }; use crate::bindings::_helpers::state_rows::models::{EventTable, EventTableLayout}; +use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; /// DDL that creates one migration state table when it is missing. @@ -51,19 +52,17 @@ fn create_sql<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult = PyTuple::new(py, layout.columns)?.into_any(); + let sql: String = state_table_create_sql( + &table, + ( + &columns, + column_types.as_any(), + PyFrozenSet::empty(py)?.as_any(), + ), + &request.get("render_framework_type")?, + )?; + Ok(PyString::new(py, &sql).into_any()) } /// The SQL reading every stored event of one schema, and the columns it selects. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs index a130979144..59acf36d54 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/mod.rs @@ -7,4 +7,5 @@ pub(crate) mod migration_projections; pub(crate) mod migration_rows; pub(crate) mod models; pub(crate) mod registration; +pub(crate) mod state_tables; pub(crate) mod timestamps; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs index df9f69c0f7..cb78b58903 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::state_rows::{ - events, freshness, migration_io, migration_projections, migration_rows, + events, freshness, migration_io, migration_projections, migration_rows, state_tables, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -12,5 +12,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { migration_rows::register(module)?; migration_io::register(module)?; migration_projections::register(module)?; + state_tables::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs new file mode 100644 index 0000000000..3cc4994e3a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/state_tables.rs @@ -0,0 +1,148 @@ +//! Append-only state table DDL and the source freshness insert, rendered through adapter callbacks. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::constants::SOURCE_FRESHNESS_TABLE_NAME; +use sqlbuild_state::freshness::constants::{ + SOURCE_FRESHNESS_COLUMN_TYPES, STRING_STATE_VALUE_TYPE, TEXT_TIMESTAMP_STATE_VALUE_TYPE, +}; +use sqlbuild_state::state_tables::main::create_table_sql::create_table_sql; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use sqlbuild_state::state_tables::models::TableColumn; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal; +use crate::bindings::_helpers::freshness::support::ADAPTER_EXCEPTIONS; + +const ADAPTER_TYPES: &str = "sqlbuild.adapter.contract.types"; +const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; +/// Each state value type a state table may declare and the framework type that renders it. +const DECLARED_TYPES: [(&str, &str); 3] = [ + (STRING_STATE_VALUE_TYPE, "STRING"), + ("integer", "INTEGER"), + (TEXT_TIMESTAMP_STATE_VALUE_TYPE, "TIMESTAMP"), +]; + +/// Create-if-missing DDL whose column types the adapter renders. +pub(crate) fn state_table_create_sql<'py>( + qualified_name: &str, + (columns, column_types, required_columns): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), + render_framework_type: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = render_framework_type.py(); + let framework_types: Bound<'py, PyAny> = imported(py, ADAPTER_TYPES, "FrameworkType")?; + let mut rendered: Vec<(&str, String)> = Vec::with_capacity(DECLARED_TYPES.len()); + for (declared, framework) in DECLARED_TYPES { + let text: String = render_framework_type + .call1((framework_types.getattr(framework)?,))? + .extract()?; + rendered.push((declared, text)); + } + let mut definitions: Vec = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let declared: Bound<'py, PyAny> = column_types.get_item(&column)?; + let declared_text: String = declared.str()?.to_str()?.to_owned(); + let Some((_, rendered_type)) = rendered.iter().find(|(name, _)| *name == declared_text) + else { + return Err(PyKeyError::new_err(declared.unbind())); + }; + definitions.push(TableColumn { + name: column.extract()?, + rendered_type: rendered_type.clone(), + required: required_columns.contains(&column)?, + }); + } + Ok(create_table_sql(qualified_name, &definitions)) +} + +/// Render create-if-missing DDL for string, integer, and text-timestamp state columns. +#[pyfunction(name = "render_state_table_create_sql")] +#[pyo3(signature = (**request))] +fn render_create_sql(request: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("render_state_table_create_sql", request)?; + state_table_create_sql( + &request.get("qualified_name")?.extract::()?, + ( + &request.get("columns")?, + &request.get("column_types")?, + &request.get("required_columns")?, + ), + &request.get("render_framework_type")?, + ) +} + +fn adapter_user_error(py: Python<'_>, message: &str) -> PyErr { + match imported(py, ADAPTER_EXCEPTIONS, "AdapterUserError").and_then(|class| { + call_with( + &class, + &[("message", PyString::new(py, message).into_any())], + ) + }) { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Render DML that appends source freshness records. +#[pyfunction] +#[pyo3(signature = (*, database, schema, records, render_qualified_name))] +fn render_insert_source_freshness_records_sql<'py>( + database: &Bound<'py, PyAny>, + schema: &Bound<'py, PyAny>, + records: &Bound<'py, PyAny>, + render_qualified_name: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = database.py(); + let table: Bound<'py, PyAny> = call_with( + render_qualified_name, + &[ + ("database", database.clone()), + ("schema", schema.clone()), + ( + "name", + PyString::new(py, SOURCE_FRESHNESS_TABLE_NAME).into_any(), + ), + ], + )?; + if table.is_none() { + return Err(adapter_user_error(py, MISSING_SCHEMA_MESSAGE)); + } + let mut rows: Vec> = Vec::new(); + for record in records.try_iter()? { + let record: Bound<'py, PyAny> = record?; + let row: Vec = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, declared)| { + render_state_sql_literal(py, &record.getattr(*column)?, declared) + }) + .collect::>()?; + rows.push(row); + } + let names: Vec<&str> = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect(); + Ok(values_insert_sql( + &table.extract::()?, + &names, + &rows, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(render_create_sql, module)?)?; + module.add_function(wrap_pyfunction!( + render_insert_source_freshness_records_sql, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index 30a05c9b4f..7070281c01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod equality; pub(crate) mod normalization; +pub(crate) mod nullability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs new file mode 100644 index 0000000000..5ca8f2ccff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/nullability.rs @@ -0,0 +1,75 @@ +//! Adapter nullability rules over `InferredNullability` arguments, decided by the native rules. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::nullability_rules::main::conditional_result_nullability::conditional_result_nullability; +use sqlbuild_analysis::nullability_rules::main::safe_cast_nullability::safe_cast_nullability; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const NULLABILITY_CLASS: &str = "InferredNullability"; +const UNKNOWN_MEMBER: &str = "UNKNOWN"; + +/// The `InferredNullability` member whose value is `value`. +fn member<'py>(args: &Bound<'py, PyAny>, value: &str) -> PyResult> { + imported(args.py(), LINEAGE_TYPES_MODULE, NULLABILITY_CLASS)?.call1((value,)) +} + +/// The text of one argument; an argument that is not a string matches no state. +fn argument_text(value: &Bound<'_, PyAny>) -> String { + match value.downcast::() { + Ok(text) => text.to_string(), + Err(_) => String::new(), + } +} + +/// The text of the first `count` arguments, read by index. +fn argument_values(args: &Bound<'_, PyAny>, count: usize) -> PyResult> { + (0..count) + .map(|index| Ok(argument_text(&args.get_item(index)?))) + .collect() +} + +/// The first argument itself, or `InferredNullability.UNKNOWN` without arguments. +#[pyfunction] +fn infer_first_arg_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult> { + if !args.is_truthy()? { + return imported(args.py(), LINEAGE_TYPES_MODULE, NULLABILITY_CLASS)? + .getattr(UNKNOWN_MEMBER); + } + args.get_item(0) +} + +/// IF/IFF-style nullability from the true and false result arguments. +#[pyfunction] +fn infer_conditional_result_nullability<'py>( + args: &Bound<'py, PyAny>, +) -> PyResult> { + let values: Vec = argument_values(args, args.len()?)?; + let values: Vec<&str> = values.iter().map(String::as_str).collect(); + member(args, conditional_result_nullability(&values)) +} + +/// SAFE_CAST/TRY_CAST nullability without assuming the conversion succeeds. +#[pyfunction] +fn infer_safe_cast_nullability<'py>(args: &Bound<'py, PyAny>) -> PyResult> { + let values: Vec = if args.is_truthy()? { + argument_values(args, 1)? + } else { + Vec::new() + }; + let values: Vec<&str> = values.iter().map(String::as_str).collect(); + member(args, safe_cast_nullability(&values)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_first_arg_nullability, module)?)?; + module.add_function(wrap_pyfunction!( + infer_conditional_result_nullability, + module + )?)?; + module.add_function(wrap_pyfunction!(infer_safe_cast_nullability, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs index b1afbad36f..1077f61865 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::type_system::normalization; +use crate::bindings::_helpers::type_system::{normalization, nullability}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { normalization::register(module)?; + nullability::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 141ff1ece5..d47d397c40 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -6,10 +6,11 @@ use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, - contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, planner_registration, - project_assembly, records, refactor, references, render, rules, scopes, semantic_checks, - semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + contracts, cursor, dag_artifact, diagnostics, discovery, exported_constants, fingerprints, + frontier, graph, lifecycle_events, lineage, macro_bridge, manifest, microbatches, model_config, + plan_output, planner_registration, project_assembly, records, refactor, references, render, + rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, + type_system, }; use crate::bindings::models; @@ -52,9 +53,11 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { authored_values::registration::register(module)?; auditing::registration::register(module)?; cursor::registration::register(module)?; + exported_constants::registration::register(module)?; state_rows::registration::register(module)?; - diagnostics::redaction::register(module)?; - diagnostics::trackers::register(module)?; + microbatches::registration::register(module)?; + lifecycle_events::registration::register(module)?; + diagnostics::registration::register(module)?; analysis::sql_test_comparisons::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs b/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs index 6237dd184e..7c7455d18d 100644 --- a/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs +++ b/crates/sqlbuild-refactor/src/refactor_command/_helpers/quoting.rs @@ -1,18 +1,6 @@ //! Command lines quoted for the shell of the platform they are printed on. -/// Python's `shlex.quote`. -fn shell_quote(text: &str) -> String { - if text.is_empty() { - return "''".to_owned(); - } - let safe: bool = text - .chars() - .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); - if safe { - return text.to_owned(); - } - format!("'{}'", text.replace('\'', "'\"'\"'")) -} +use sqlbuild_core::shell_text::main::shell_quote::shell_quote; /// Python's `subprocess.list2cmdline` for one argument. fn windows_quote(argument: &str) -> String { diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index a7d43d7096..ed6d6fa9af 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,9 +1,11 @@ +use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use std::path::Path; pub(crate) fn load(project_dir: &Path) -> Result { - let path = project_dir.join("sqlbuild_project.toml"); + let path = project_dir.join(PROJECT_CONFIG_FILENAME); let source = std::fs::read_to_string(&path).map_err(|error| { format!( "could not load rules config from {}: {error}", @@ -100,7 +102,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { "max_ranking_order_by", "min_audits_per_model", "min_tests_per_model", - "min_custom_rule_test_cases", + MIN_CUSTOM_RULE_TEST_CASES_KEY, "max_subdomain_depth", "min_shared_owner_prefix_directories", "max_role_container_depth", diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs index 7108ee1055..690d165255 100644 --- a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs +++ b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs @@ -5,8 +5,8 @@ use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; use crate::configuration::errors::ConfigPolicyError; use crate::configuration::main::setting_help::setting_help; -use crate::constants::PROJECT_CONFIG_FILE; use crate::models::RulesConfig; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; /// The policy error when `allow_exceptions = false` and exception or ignore entries exist. #[must_use] diff --git a/crates/sqlbuild-rules/src/configuration/main/setting_help.rs b/crates/sqlbuild-rules/src/configuration/main/setting_help.rs index 4cf8eabbf2..567bf96acd 100644 --- a/crates/sqlbuild-rules/src/configuration/main/setting_help.rs +++ b/crates/sqlbuild-rules/src/configuration/main/setting_help.rs @@ -1,6 +1,7 @@ //! Remediation text that shows the exact `sqlbuild_project.toml` setting to write. -use crate::constants::{PROJECT_CONFIG_FILE, SETTING_SNIPPET_INDENT}; +use crate::constants::SETTING_SNIPPET_INDENT; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; /// `, set this in sqlbuild_project.toml:` followed by the section and `key = value`. pub(crate) fn setting_help(purpose: &str, section: &str, key: &str, value: &str) -> String { diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index f2b6b08109..c8e55bb51f 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -29,7 +29,8 @@ pub(crate) const DECLARATION_DOMAIN_COMPONENTS: usize = 3; pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; -pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; +/// Threshold key setting the fewest test cases a custom rule must ship. +pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 7c4bd3a4cb..685595fba4 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -12,6 +12,7 @@ pub mod errors; pub mod fix_verification; pub mod lint_text; pub mod models; +pub mod project_tree; pub mod rule_cache; pub mod rule_catalogue; pub mod rule_results; diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index b713e61363..fbc6ff9d4b 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -121,7 +121,6 @@ pub(crate) const NON_CONTIGUOUS_FIX: &str = "fix overlaps generated SQL or a non-contiguous authored region"; pub(crate) const GENERATED_FIX: &str = "fix overlaps generated SQL"; pub(crate) const GENERATED_SQL_SUFFIX: &str = "(in generated SQL)"; -pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; pub(crate) const DEFAULT_LINE_WIDTH: i64 = 100; pub(crate) const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; diff --git a/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs b/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs index f017d05a27..d9cf308333 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs @@ -5,9 +5,10 @@ use std::path::Path; use crate::lint_text::_helpers::settings::{adapter_dialect, positive_int, read_table}; use crate::lint_text::constants::{ DEFAULT_DIALECT, DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES, DEFAULT_MAX_LITERAL_LENGTH, - DEFAULT_MAX_RANKING_ORDER_BY, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, + DEFAULT_MAX_RANKING_ORDER_BY, LOCAL_CONFIG_FILENAME, }; use crate::lint_text::models::FormatSettings; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; /// `[format]` widths, rule thresholds and the adapter dialect, the local adapter winning. pub fn resolve_format_settings(project_dir: &Path) -> Result { diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs new file mode 100644 index 0000000000..0419da5111 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/entries.rs @@ -0,0 +1,41 @@ +//! How the project walk classifies, filters and names directory entries. + +use std::ffi::{OsStr, OsString}; +use std::fs::DirEntry; +use std::path::{Path, PathBuf}; + +use crate::project_tree::constants::{EXCLUDED_ROOTS, HIDDEN_PREFIX, POSIX_SEPARATOR}; +use crate::project_tree::models::EntryKind; + +/// How the walk treats one directory entry, without following links. +pub(crate) fn kind(entry: &DirEntry) -> EntryKind { + match entry.file_type() { + Ok(file_type) if file_type.is_symlink() => EntryKind::Link, + Ok(file_type) if file_type.is_dir() => EntryKind::Directory, + _ => EntryKind::Other, + } +} + +/// Whether the walk lists and enters a directory entry. +pub(crate) fn listed_directory(entry: &DirEntry, at_root: bool) -> bool { + let name: OsString = entry.file_name(); + let name: &[u8] = name.as_encoded_bytes(); + if name.first() == Some(&HIDDEN_PREFIX) { + return false; + } + !(at_root + && EXCLUDED_ROOTS + .iter() + .any(|excluded| name == excluded.as_bytes())) +} + +/// The project-relative `as_posix()` spelling of `name` inside `relative`. +pub(crate) fn posix_child(relative: &Path, name: &OsStr) -> PathBuf { + if relative.as_os_str().is_empty() { + return PathBuf::from(name); + } + let mut child: OsString = relative.as_os_str().to_os_string(); + child.push(POSIX_SEPARATOR); + child.push(name); + PathBuf::from(child) +} diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs new file mode 100644 index 0000000000..a37fa5f9d5 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/files.rs @@ -0,0 +1,130 @@ +//! Resolution of one project-relative path to a real, readable project file. + +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::relative_paths::{pure_parts, pure_relative_to}; +use crate::project_tree::constants::MISSING_FILE_ERRORS; +use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::main::tracked_text_path::tracked_text_path; +use crate::project_tree::models::PureParts; + +/// `root / path` for a relative POSIX path: the root joined with each component. +#[cfg(unix)] +pub(crate) fn joined(root: &Path, parts: &PureParts) -> PathBuf { + let mut path: PathBuf = root.to_path_buf(); + path.extend(&parts.tail); + path +} + +/// `root / path` as `ntpath.join` builds it, keeping or replacing the root's drive. +#[cfg(not(unix))] +pub(crate) fn joined(root: &Path, parts: &PureParts) -> PathBuf { + use std::ffi::OsString; + use std::path::MAIN_SEPARATOR_STR; + + use crate::project_tree::_helpers::relative_paths::same_part; + + let anchor: &str = &parts.anchor.to_string_lossy(); + let drive: &str = anchor.trim_end_matches(MAIN_SEPARATOR_STR); + let root_anchor: OsString = pure_parts(root).anchor; + let root_anchor: &str = &root_anchor.to_string_lossy(); + let root_drive: &str = root_anchor.trim_end_matches(MAIN_SEPARATOR_STR); + let mut path: PathBuf = if drive.len() < anchor.len() { + let drive: &str = if drive.is_empty() { root_drive } else { drive }; + PathBuf::from(format!("{drive}{MAIN_SEPARATOR_STR}")) + } else if drive.is_empty() || same_part(drive.as_ref(), root_drive.as_ref()) { + root.components().collect() + } else { + PathBuf::from(drive) + }; + path.extend(&parts.tail); + path +} + +/// `path.resolve()` of an existing path, spelled without Windows' verbatim `\\?\` prefix. +#[cfg(unix)] +pub(crate) fn real_path(path: &Path) -> std::io::Result { + std::fs::canonicalize(path) +} + +/// `path.resolve()` of an existing path, spelled without Windows' verbatim `\\?\` prefix. +#[cfg(not(unix))] +pub(crate) fn real_path(path: &Path) -> std::io::Result { + use std::ffi::OsString; + use std::path::{Component, Prefix}; + + let real: PathBuf = std::fs::canonicalize(path)?; + let mut components = real.components(); + let Some(Component::Prefix(prefix)) = components.next() else { + return Ok(real); + }; + let anchor: OsString = match prefix.kind() { + Prefix::VerbatimDisk(drive) => OsString::from(format!("{}:", char::from(drive))), + Prefix::VerbatimUNC(server, share) => { + let mut anchor: OsString = OsString::from(r"\\"); + anchor.push(server); + anchor.push(r"\"); + anchor.push(share); + anchor + } + _ => return Ok(real), + }; + let mut plain: PathBuf = PathBuf::from(anchor); + plain.extend(components); + Ok(plain) +} + +/// `path.is_relative_to(root)`, case-folded on Windows as `pathlib` compares there. +pub(crate) fn inside(path: &Path, root: &Path) -> bool { + if cfg!(unix) { + path.starts_with(root) + } else { + pure_relative_to(&pure_parts(path), &pure_parts(root)).is_some() + } +} + +/// `Path.is_file()`: false for the failures `pathlib` ignores, the failure otherwise. +pub(crate) fn points_to_file(path: &Path) -> Result { + match std::fs::metadata(path) { + Ok(metadata) => Ok(metadata.is_file()), + Err(error) => match error.raw_os_error() { + Some(errno) if !MISSING_FILE_ERRORS.contains(&errno) => { + Err(ProjectTreeFileError::Unreadable { + path: path.to_path_buf(), + errno, + }) + } + _ => Ok(false), + }, + } +} + +/// A regular, non-link file under the project's resolved parent directory of the path. +pub(crate) fn regular_file(root: &Path, parts: &PureParts) -> Option { + let (name, parent) = parts.tail.split_last()?; + let parent: PureParts = PureParts { + anchor: parts.anchor.clone(), + tail: parent.to_vec(), + }; + let directory: PathBuf = match real_path(&joined(root, &parent)) { + Ok(directory) => directory, + Err(_unresolved) => return None, + }; + if !inside(&directory, root) { + return None; + } + let candidate: PathBuf = directory.join(name); + match std::fs::symlink_metadata(&candidate) { + Ok(metadata) => metadata.file_type().is_file().then_some(candidate), + Err(_unreadable) => None, + } +} + +/// `candidate` when its lowercased suffix is a tracked text type. +pub(crate) fn tracked(candidate: PathBuf) -> Result { + if tracked_text_path(&candidate) { + Ok(candidate) + } else { + Err(ProjectTreeFileError::Untracked) + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs new file mode 100644 index 0000000000..ad40bd0d59 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod entries; +pub(crate) mod files; +pub(crate) mod relative_paths; diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs new file mode 100644 index 0000000000..4cd03fe999 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/relative_paths.rs @@ -0,0 +1,179 @@ +//! `pathlib` reads of project paths: POSIX over raw bytes, Windows through the platform path rules. + +use std::ffi::{OsStr, OsString}; +use std::path::{MAIN_SEPARATOR, MAIN_SEPARATOR_STR, Path}; + +use crate::project_tree::constants::{CURRENT, PARENT, POSIX_SEPARATOR_BYTE}; +use crate::project_tree::models::PureParts; + +/// `PurePosixPath(path)`: its root (`/` or exactly `//`) and the components after it. +#[cfg(unix)] +pub(crate) fn pure_parts(path: &Path) -> PureParts { + use std::os::unix::ffi::OsStrExt; + + let separator: u8 = POSIX_SEPARATOR_BYTE; + let bytes: &[u8] = path.as_os_str().as_bytes(); + let anchor: &[u8] = match bytes { + [first, second, rest @ ..] + if *first == separator && *second == separator && rest.first() != Some(&separator) => + { + &bytes[..2] + } + [first, ..] if *first == separator => &bytes[..1], + _ => &[], + }; + PureParts { + anchor: OsStr::from_bytes(anchor).to_os_string(), + tail: bytes + .split(|byte| *byte == separator) + .filter(|part| !part.is_empty() && *part != CURRENT.as_bytes()) + .map(|part| OsStr::from_bytes(part).to_os_string()) + .collect(), + } +} + +/// `PureWindowsPath(path)`: its drive and root with `\` separators, then each component. +#[cfg(not(unix))] +pub(crate) fn pure_parts(path: &Path) -> PureParts { + use std::path::Component; + + use crate::project_tree::constants::POSIX_SEPARATOR; + + let mut parts: PureParts = PureParts { + anchor: OsString::new(), + tail: Vec::new(), + }; + for component in path.components() { + match component { + Component::Prefix(prefix) => parts.anchor.push( + prefix + .as_os_str() + .to_string_lossy() + .replace(POSIX_SEPARATOR, MAIN_SEPARATOR_STR), + ), + Component::RootDir => parts.anchor.push(MAIN_SEPARATOR_STR), + Component::CurDir => {} + Component::ParentDir => parts.tail.push(OsString::from(PARENT)), + Component::Normal(part) => parts.tail.push(part.to_os_string()), + } + } + parts +} + +/// Whether two parts are equal as `pathlib` compares them: exactly, or case-folded on Windows. +pub(crate) fn same_part(left: &OsStr, right: &OsStr) -> bool { + if cfg!(unix) { + left == right + } else { + left.to_string_lossy().to_lowercase() == right.to_string_lossy().to_lowercase() + } +} + +/// Whether `pathlib` considers two pure paths equal. +pub(crate) fn same_path(left: &PureParts, right: &PureParts) -> bool { + same_part(&left.anchor, &right.anchor) + && left.tail.len() == right.tail.len() + && left + .tail + .iter() + .zip(&right.tail) + .all(|(left, right)| same_part(left, right)) +} + +/// `str(path)`: the anchor, then the components joined by the platform separator, or `.`. +pub(crate) fn pure_text(parts: &PureParts) -> OsString { + if parts.anchor.is_empty() && parts.tail.is_empty() { + return OsString::from(CURRENT); + } + let mut text: OsString = parts.anchor.clone(); + for (index, part) in parts.tail.iter().enumerate() { + if index > 0 { + text.push(MAIN_SEPARATOR_STR); + } + text.push(part); + } + text +} + +/// `path.as_posix()` as encoded bytes. +pub(crate) fn posix_bytes(parts: &PureParts) -> Vec { + pure_text(parts) + .as_encoded_bytes() + .iter() + .map(|byte| { + if char::from(*byte) == MAIN_SEPARATOR { + POSIX_SEPARATOR_BYTE + } else { + *byte + } + }) + .collect() +} + +/// `path.parent`: a bare anchor or `.` is its own parent. +pub(crate) fn pure_parent(parts: &PureParts) -> PureParts { + let mut parent: PureParts = parts.clone(); + parent.tail.pop(); + parent +} + +/// `path.relative_to(base)`, or `None` when `path` is not beneath `base`. +pub(crate) fn pure_relative_to(parts: &PureParts, base: &PureParts) -> Option { + let beneath: bool = same_part(&parts.anchor, &base.anchor) + && parts.tail.len() >= base.tail.len() + && parts + .tail + .iter() + .zip(&base.tail) + .all(|(part, prefix)| same_part(part, prefix)); + beneath.then(|| PureParts { + anchor: OsString::new(), + tail: parts.tail[base.tail.len()..].to_vec(), + }) +} + +/// `path.parts`: the anchor, when there is one, then each component. +pub(crate) fn part_list(parts: PureParts) -> Vec { + let anchor: Option = (!parts.anchor.is_empty()).then_some(parts.anchor); + anchor.into_iter().chain(parts.tail).collect() +} + +/// `path.is_absolute() or ".." in path.parts`. +pub(crate) fn escapes(path: &Path) -> bool { + path.is_absolute() || pure_parts(path).tail.iter().any(|part| part == PARENT) +} + +/// Path text with one private-use character per undecodable byte, as `surrogateescape` reads. +#[cfg(unix)] +pub(crate) fn escaped_text(path: &Path) -> String { + use std::os::unix::ffi::OsStrExt; + + use crate::project_tree::constants::ESCAPED_BASE; + + let mut text: String = String::new(); + for chunk in path.as_os_str().as_bytes().utf8_chunks() { + text.push_str(chunk.valid()); + text.extend( + chunk + .invalid() + .iter() + .filter_map(|byte| char::from_u32(ESCAPED_BASE + u32::from(*byte))), + ); + } + text +} + +/// Path text with one private-use character per unpaired surrogate, as Python keeps them. +#[cfg(windows)] +pub(crate) fn escaped_text(path: &Path) -> String { + use std::os::windows::ffi::OsStrExt; + + use crate::project_tree::constants::ESCAPED_BASE; + + char::decode_utf16(path.as_os_str().encode_wide()) + .filter_map(|decoded| match decoded { + Ok(character) => Some(character), + Err(error) => char::from_u32(ESCAPED_BASE + u32::from(error.unpaired_surrogate())), + }) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/constants.rs b/crates/sqlbuild-rules/src/project_tree/constants.rs new file mode 100644 index 0000000000..7759efbbb0 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/constants.rs @@ -0,0 +1,27 @@ +//! Directories the project tree skips and the file types rules may read. + +/// Top-level directories that are tooling output or environments, never project sources. +pub(crate) const EXCLUDED_ROOTS: [&str; 5] = [".git", ".venv", "logs", "target", "venv"]; +/// The lowercase suffixes of the text files rules may read, sorted. +pub(crate) const TRACKED_TEXT_SUFFIXES: [&str; 5] = [".py", ".sql", ".toml", ".yaml", ".yml"]; +/// The leading byte of a hidden directory name. +pub(crate) const HIDDEN_PREFIX: u8 = b'.'; +/// `errno` values `pathlib` treats as a missing file when it tests `is_file()`. +#[cfg(target_os = "macos")] +pub(crate) const MISSING_FILE_ERRORS: [i32; 4] = [2, 20, 9, 62]; +/// `errno` values `pathlib` treats as a missing file when it tests `is_file()`. +#[cfg(all(unix, not(target_os = "macos")))] +pub(crate) const MISSING_FILE_ERRORS: [i32; 4] = [2, 20, 9, 40]; +/// Windows error codes `pathlib` treats as a missing file: those mapping to its ignored `errno`s. +#[cfg(not(unix))] +pub(crate) const MISSING_FILE_ERRORS: [i32; 15] = [ + 2, 3, 15, 18, 53, 67, 161, 206, 6, 114, 130, 267, 21, 123, 1921, +]; +/// The separator `as_posix()` spells and the project walk joins listed paths with. +pub(crate) const POSIX_SEPARATOR: &str = "/"; +pub(crate) const POSIX_SEPARATOR_BYTE: u8 = b'/'; +/// `str()` of a path with no anchor and no components. +pub(crate) const CURRENT: &str = "."; +pub(crate) const PARENT: &str = ".."; +/// Private-use characters standing for undecodable path bytes or unpaired surrogates. +pub(crate) const ESCAPED_BASE: u32 = 0xF_0000; diff --git a/crates/sqlbuild-rules/src/project_tree/errors.rs b/crates/sqlbuild-rules/src/project_tree/errors.rs new file mode 100644 index 0000000000..94b63825bd --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/errors.rs @@ -0,0 +1,31 @@ +//! Why a project-relative path is not a file rules may read. + +use std::path::PathBuf; + +/// The reason a project path cannot be read through the project tree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProjectTreeFileError { + /// The path is absolute or climbs out with `..`. + Escapes, + /// The path holds a NUL byte, which no file name can. + EmbeddedNul, + /// No regular file exists at the path inside the project. + Missing, + /// The file is not one of the tracked text types. + Untracked, + /// The path resolves through a symbolic link. + Symlink, + /// Testing the resolved path failed with this `errno`, or Windows error code. + Unreadable { path: PathBuf, errno: i32 }, +} + +/// Why a path has no parts relative to the project or the requested directory. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RelativePartsError { + /// Resolving the absolute path failed. + Resolve(E), + /// The resolved path is outside the project. + Escapes, + /// The path is not beneath the requested directory. + NotBeneath, +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs new file mode 100644 index 0000000000..4ef4911b9a --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -0,0 +1,9 @@ +pub mod project_path_escapes; +pub mod project_relative_parts; +pub mod project_tree_children; +pub mod project_tree_descendants; +pub mod project_tree_entries; +pub mod project_tree_file; +pub mod project_tree_glob; +pub mod tracked_text_path; +pub mod tracked_text_suffixes; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs b/crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs new file mode 100644 index 0000000000..a4d285536e --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_path_escapes.rs @@ -0,0 +1,11 @@ +//! Whether a project path leaves the project. + +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::escapes; + +/// `Path(path).is_absolute() or ".." in Path(path).parts` with this platform's path rules. +#[must_use] +pub fn project_path_escapes(path: &Path) -> bool { + escapes(path) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs b/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs new file mode 100644 index 0000000000..90009bdd69 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_relative_parts.rs @@ -0,0 +1,28 @@ +//! The parts of a path relative to the project and an optional directory inside it. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::relative_paths::{part_list, pure_parts, pure_relative_to}; +use crate::project_tree::errors::RelativePartsError; +use crate::project_tree::models::PureParts; + +/// `path`'s parts, made project-relative through `resolve` when absolute, then relative to `under`. +pub fn project_relative_parts( + path: &Path, + root: &Path, + under: Option<&Path>, + resolve: impl FnOnce(&Path) -> Result, +) -> Result, RelativePartsError> { + let mut parts: PureParts = pure_parts(path); + if path.is_absolute() { + let resolved: PathBuf = resolve(path).map_err(RelativePartsError::Resolve)?; + parts = pure_relative_to(&pure_parts(&resolved), &pure_parts(root)) + .ok_or(RelativePartsError::Escapes)?; + } + if let Some(under) = under { + parts = + pure_relative_to(&parts, &pure_parts(under)).ok_or(RelativePartsError::NotBeneath)?; + } + Ok(part_list(parts)) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs new file mode 100644 index 0000000000..6494af41f8 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_children.rs @@ -0,0 +1,18 @@ +//! The listed paths directly inside one project directory. + +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::{pure_parent, pure_parts, same_path}; +use crate::project_tree::models::PureParts; + +/// Indexes of the `values` whose `Path(value).parent == Path(directory)`. +#[must_use] +pub fn project_tree_children(values: &[&Path], directory: &Path) -> Vec { + let directory: PureParts = pure_parts(directory); + values + .iter() + .enumerate() + .filter(|(_, value)| same_path(&pure_parent(&pure_parts(value)), &directory)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs new file mode 100644 index 0000000000..83283fedf8 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_descendants.rs @@ -0,0 +1,28 @@ +//! The listed paths below one project directory. + +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::{posix_bytes, pure_parts}; +use crate::project_tree::constants::POSIX_SEPARATOR_BYTE; + +/// Indexes of the `values` below `directory` as `as_posix().rstrip("/") + "/"` prefixes them. +#[must_use] +pub fn project_tree_descendants(values: &[&Path], directory: Option<&Path>) -> Vec { + let prefix: Vec = match directory { + Some(directory) => { + let mut text: Vec = posix_bytes(&pure_parts(directory)); + while text.last() == Some(&POSIX_SEPARATOR_BYTE) { + text.pop(); + } + text.push(POSIX_SEPARATOR_BYTE); + text + } + None => Vec::new(), + }; + values + .iter() + .enumerate() + .filter(|(_, value)| value.as_os_str().as_encoded_bytes().starts_with(&prefix)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs new file mode 100644 index 0000000000..a925fc6b62 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_entries.rs @@ -0,0 +1,38 @@ +//! Every file and directory under a project, as `os.walk` finds them without following links. + +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::entries::{kind, listed_directory, posix_child}; +use crate::project_tree::models::{EntryKind, ProjectTreeEntry}; + +/// Files and directories under `root`, without links, hidden or top-level tooling directories. +#[must_use] +pub fn project_tree_entries(root: &Path) -> Vec { + let mut entries: Vec = Vec::new(); + let mut pending: Vec = vec![PathBuf::new()]; + while let Some(relative) = pending.pop() { + let Ok(listing) = std::fs::read_dir(root.join(&relative)) else { + continue; + }; + for entry in listing.flatten() { + let child: PathBuf = posix_child(&relative, &entry.file_name()); + match kind(&entry) { + EntryKind::Link => {} + EntryKind::Directory => { + if listed_directory(&entry, relative.as_os_str().is_empty()) { + entries.push(ProjectTreeEntry { + relative: child.clone(), + is_file: false, + }); + pending.push(child); + } + } + EntryKind::Other => entries.push(ProjectTreeEntry { + relative: child, + is_file: true, + }), + } + } + } + entries +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs new file mode 100644 index 0000000000..3f76fe8cc9 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs @@ -0,0 +1,34 @@ +//! The real file a project-relative path names, when rules may read it. + +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::files::{ + inside, joined, points_to_file, real_path, regular_file, tracked, +}; +use crate::project_tree::_helpers::relative_paths::{escapes, pure_parts}; +use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::models::PureParts; + +/// The tracked file inside the resolved `root` that the project-relative `path` names. +pub fn project_tree_file(root: &Path, path: &Path) -> Result { + if escapes(path) { + return Err(ProjectTreeFileError::Escapes); + } + if path.as_os_str().as_encoded_bytes().contains(&0) { + return Err(ProjectTreeFileError::EmbeddedNul); + } + let parts: PureParts = pure_parts(path); + if let Some(regular) = regular_file(root, &parts) { + return tracked(regular); + } + let lexical: PathBuf = joined(root, &parts); + let candidate: PathBuf = real_path(&lexical).unwrap_or(lexical); + if !inside(&candidate, root) || !points_to_file(&candidate)? { + return Err(ProjectTreeFileError::Missing); + } + let candidate: PathBuf = tracked(candidate)?; + if candidate.is_symlink() { + return Err(ProjectTreeFileError::Symlink); + } + Ok(candidate) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs b/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs new file mode 100644 index 0000000000..adec478a24 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_tree_glob.rs @@ -0,0 +1,19 @@ +//! The listed paths a shell-style pattern matches. + +use std::path::Path; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::project_tree::_helpers::relative_paths::escaped_text; + +/// Indexes of the `values` that `fnmatch.fnmatchcase(value, pattern)` accepts. +#[must_use] +pub fn project_tree_glob(values: &[&Path], pattern: &Path) -> Vec { + let pattern: String = escaped_text(pattern); + values + .iter() + .enumerate() + .filter(|(_, value)| glob_match(&escaped_text(value), &pattern)) + .map(|(index, _)| index) + .collect() +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs new file mode 100644 index 0000000000..91be1ef8be --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_path.rs @@ -0,0 +1,17 @@ +//! Whether a project path names a text file type rules may read. + +use std::path::Path; + +use sqlbuild_core::posix_path::main::posix_suffix::posix_suffix; + +use crate::project_tree::constants::TRACKED_TEXT_SUFFIXES; + +/// Whether the lowercased suffix of the path's name is a tracked text type. +#[must_use] +pub fn tracked_text_path(path: &Path) -> bool { + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + TRACKED_TEXT_SUFFIXES.contains(&posix_suffix(&name).to_lowercase().as_str()) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs new file mode 100644 index 0000000000..1219bdb01f --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/tracked_text_suffixes.rs @@ -0,0 +1,9 @@ +//! The file types rules may read, as listed in errors. + +use crate::project_tree::constants::TRACKED_TEXT_SUFFIXES; + +/// The tracked suffixes in sorted order. +#[must_use] +pub fn tracked_text_suffixes() -> &'static [&'static str] { + &TRACKED_TEXT_SUFFIXES +} diff --git a/crates/sqlbuild-rules/src/project_tree/mod.rs b/crates/sqlbuild-rules/src/project_tree/mod.rs new file mode 100644 index 0000000000..67a8c068b8 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/mod.rs @@ -0,0 +1,10 @@ +//! The deterministic project-tree view custom rules read: discovered paths and readable files. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/project_tree/models.rs b/crates/sqlbuild-rules/src/project_tree/models.rs new file mode 100644 index 0000000000..65a53b938f --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/models.rs @@ -0,0 +1,25 @@ +//! One discovered project path. + +use std::ffi::OsString; +use std::path::PathBuf; + +/// A project-relative file or directory the tree lists. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectTreeEntry { + pub relative: PathBuf, + pub is_file: bool, +} + +/// How the walk treats one directory entry. +pub(crate) enum EntryKind { + Link, + Directory, + Other, +} + +/// A pure path as `pathlib` splits it: its anchor (drive and root), then each component. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PureParts { + pub(crate) anchor: OsString, + pub(crate) tail: Vec, +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests.rs b/crates/sqlbuild-rules/src/project_tree/tests.rs new file mode 100644 index 0000000000..30903ec9e3 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests.rs @@ -0,0 +1,16 @@ +#[cfg(unix)] +#[path = "tests/test_escapes.rs"] +mod escapes; +#[cfg(unix)] +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_path_queries.rs"] +mod path_queries; +#[cfg(unix)] +#[path = "tests/test_project_tree.rs"] +mod project_tree; +#[cfg(unix)] +#[path = "tests/test_relative_parts.rs"] +mod relative_parts; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/project_tree/tests/helpers.rs b/crates/sqlbuild-rules/src/project_tree/tests/helpers.rs new file mode 100644 index 0000000000..75da32afaa --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/helpers.rs @@ -0,0 +1,24 @@ +use std::os::unix::fs::symlink; +use std::path::PathBuf; + +/// A project with nested, hidden, tooling and linked paths, and its resolved root. +pub(crate) fn linked_project() -> (tempfile::TempDir, PathBuf) { + let directory = tempfile::tempdir().expect("temporary project"); + let root: PathBuf = std::fs::canonicalize(directory.path()).expect("resolved root"); + for folder in ["models/marts", "models/.hidden", "logs", "sub/logs"] { + std::fs::create_dir_all(root.join(folder)).expect("folder"); + } + for file in [ + "models/marts/orders.sql", + "models/.dot.sql", + "models/.hidden/x.sql", + ] { + std::fs::write(root.join(file), "select 1\n").expect("file"); + } + std::fs::write(root.join("readme.md"), "# r\n").expect("readme"); + std::fs::write(root.join("sub/logs/kept.sql"), "k").expect("kept"); + symlink("marts/orders.sql", root.join("models/link.sql")).expect("link"); + symlink("marts", root.join("models/linkdir")).expect("dir link"); + symlink("../readme.md", root.join("models/readme.sql")).expect("readme link"); + (directory, root) +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs new file mode 100644 index 0000000000..7240d19d0f --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_escapes.rs @@ -0,0 +1,68 @@ +use std::path::Path; + +use crate::project_tree::main::project_path_escapes::project_path_escapes; +use crate::project_tree::tests::test_types::EscapesTestCase; + +#[test] +fn given_paths_when_checking_for_escapes_then_absolute_and_parent_paths_escape() { + let test_cases = [ + EscapesTestCase { + description: "a relative file", + path: "models/a.sql", + expected_escapes: false, + }, + EscapesTestCase { + description: "current and empty parts", + path: "./a//b/", + expected_escapes: false, + }, + EscapesTestCase { + description: "an empty path", + path: "", + expected_escapes: false, + }, + EscapesTestCase { + description: "the current directory", + path: ".", + expected_escapes: false, + }, + EscapesTestCase { + description: "an absolute path", + path: "/abs/x.yml", + expected_escapes: true, + }, + EscapesTestCase { + description: "a double-slash root", + path: "//x", + expected_escapes: true, + }, + EscapesTestCase { + description: "a leading parent", + path: "../x.sql", + expected_escapes: true, + }, + EscapesTestCase { + description: "an inner parent", + path: "a/../x.sql", + expected_escapes: true, + }, + EscapesTestCase { + description: "only a parent", + path: "..", + expected_escapes: true, + }, + EscapesTestCase { + description: "dots inside names", + path: "a..b/..c", + expected_escapes: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + project_path_escapes(Path::new(test_case.path)), + test_case.expected_escapes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs new file mode 100644 index 0000000000..964c6ebe4f --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_path_queries.rs @@ -0,0 +1,108 @@ +use std::path::Path; + +use crate::project_tree::main::project_tree_children::project_tree_children; +use crate::project_tree::main::project_tree_descendants::project_tree_descendants; +use crate::project_tree::main::project_tree_glob::project_tree_glob; +use crate::project_tree::main::tracked_text_path::tracked_text_path; +use crate::project_tree::tests::test_types::{PathSelectionTestCase, TrackedTextTestCase}; + +const VALUES: [&str; 6] = [ + "models", + "models/marts", + "models/marts/orders.sql", + "models/a.sql", + "rules", + "readme.md", +]; + +#[test] +fn given_listed_paths_when_selecting_then_children_descendants_and_globs_match_pathlib() { + let test_cases = [ + PathSelectionTestCase { + description: "the project root", + argument: None, + expected_children: vec!["models", "rules", "readme.md"], + expected_descendants: VALUES.to_vec(), + expected_glob: vec![], + }, + PathSelectionTestCase { + description: "a folder spelled with dots and slashes", + argument: Some("./models//"), + expected_children: vec!["models/marts", "models/a.sql"], + expected_descendants: vec!["models/marts", "models/marts/orders.sql", "models/a.sql"], + expected_glob: vec![], + }, + PathSelectionTestCase { + description: "a star pattern crossing folders", + argument: Some("*.sql"), + expected_children: vec![], + expected_descendants: vec![], + expected_glob: vec!["models/marts/orders.sql", "models/a.sql"], + }, + PathSelectionTestCase { + description: "a negated set", + argument: Some("[!m]*"), + expected_children: vec![], + expected_descendants: vec![], + expected_glob: vec!["rules", "readme.md"], + }, + ]; + let values: Vec<&Path> = VALUES.iter().map(Path::new).collect(); + let pick = |indexes: Vec| -> Vec<&str> { + indexes.into_iter().map(|index| VALUES[index]).collect() + }; + for test_case in test_cases { + let argument: &Path = Path::new(test_case.argument.unwrap_or("")); + assert_eq!( + ( + pick(project_tree_children(&values, argument)), + pick(project_tree_descendants( + &values, + test_case.argument.map(Path::new) + )), + pick(project_tree_glob(&values, argument)), + ), + ( + test_case.expected_children, + test_case.expected_descendants, + test_case.expected_glob + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_file_names_when_checking_tracked_text_then_only_known_suffixes_count() { + let test_cases = [ + TrackedTextTestCase { + description: "a SQL model", + path: "models/a.sql", + expected_tracked: true, + }, + TrackedTextTestCase { + description: "an uppercase suffix", + path: "models/A.YML", + expected_tracked: true, + }, + TrackedTextTestCase { + description: "a hidden name without a suffix", + path: "models/.sql", + expected_tracked: false, + }, + TrackedTextTestCase { + description: "a markdown file", + path: "readme.md", + expected_tracked: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + tracked_text_path(Path::new(test_case.path)), + test_case.expected_tracked, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs new file mode 100644 index 0000000000..7ec13537db --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs @@ -0,0 +1,113 @@ +use std::path::Path; + +use crate::project_tree::errors::ProjectTreeFileError; +use crate::project_tree::main::project_tree_entries::project_tree_entries; +use crate::project_tree::main::project_tree_file::project_tree_file; +use crate::project_tree::tests::helpers::linked_project; +use crate::project_tree::tests::test_types::{ProjectTreeEntriesTestCase, ProjectTreeFileTestCase}; + +#[test] +fn given_project_when_walked_then_links_hidden_and_root_tooling_directories_are_skipped() { + let test_cases = [ProjectTreeEntriesTestCase { + description: "links, hidden directories and top-level logs are left out", + expected_entries: vec![ + ("models", false), + ("models/.dot.sql", true), + ("models/marts", false), + ("models/marts/orders.sql", true), + ("readme.md", true), + ("sub", false), + ("sub/logs", false), + ("sub/logs/kept.sql", true), + ], + }]; + for test_case in test_cases { + let (_directory, root) = linked_project(); + let mut entries: Vec<(String, bool)> = project_tree_entries(&root) + .into_iter() + .map(|entry| (entry.relative.to_string_lossy().into_owned(), entry.is_file)) + .collect(); + entries.sort(); + let entries: Vec<(&str, bool)> = entries + .iter() + .map(|(path, is_file)| (path.as_str(), *is_file)) + .collect(); + assert_eq!( + entries, test_case.expected_entries, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_project_paths_when_resolved_then_only_tracked_files_inside_the_project_are_readable() { + let test_cases = [ + ProjectTreeFileTestCase { + description: "a regular file", + path: "models/marts/orders.sql", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "current, empty and trailing components", + path: "./models//marts/orders.sql/", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "a linked file resolves to its target", + path: "models/link.sql", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "a file under a linked directory", + path: "models/linkdir/orders.sql", + expected_file: Ok("models/marts/orders.sql"), + }, + ProjectTreeFileTestCase { + description: "a link to an untracked file", + path: "models/readme.sql", + expected_file: Err(ProjectTreeFileError::Untracked), + }, + ProjectTreeFileTestCase { + description: "an untracked file type", + path: "readme.md", + expected_file: Err(ProjectTreeFileError::Untracked), + }, + ProjectTreeFileTestCase { + description: "a missing file", + path: "missing.sql", + expected_file: Err(ProjectTreeFileError::Missing), + }, + ProjectTreeFileTestCase { + description: "a directory", + path: "models", + expected_file: Err(ProjectTreeFileError::Missing), + }, + ProjectTreeFileTestCase { + description: "a path above the project", + path: "../outside.sql", + expected_file: Err(ProjectTreeFileError::Escapes), + }, + ProjectTreeFileTestCase { + description: "a NUL byte", + path: "a\0b.sql", + expected_file: Err(ProjectTreeFileError::EmbeddedNul), + }, + ]; + for test_case in test_cases { + let (_directory, root) = linked_project(); + let resolved: Result = + project_tree_file(&root, Path::new(test_case.path)).map(|file| { + file.strip_prefix(&root) + .unwrap_or(&file) + .to_string_lossy() + .into_owned() + }); + assert_eq!( + resolved.as_ref().map(String::as_str).map_err(Clone::clone), + test_case.expected_file, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs new file mode 100644 index 0000000000..03fddedadd --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_relative_parts.rs @@ -0,0 +1,69 @@ +use std::path::{Path, PathBuf}; + +use crate::project_tree::errors::RelativePartsError; +use crate::project_tree::main::project_relative_parts::project_relative_parts; +use crate::project_tree::tests::test_types::RelativePartsTestCase; + +#[test] +fn given_paths_when_made_project_relative_then_parts_follow_relative_to() { + let test_cases = [ + RelativePartsTestCase { + description: "a relative path keeps its parts", + path: "models/../a.sql", + resolved: "", + under: None, + expected_parts: Ok(vec!["models", "..", "a.sql"]), + }, + RelativePartsTestCase { + description: "an absolute path inside the project", + path: "/project/link/a.sql", + resolved: "/project/models/a.sql", + under: Some("models"), + expected_parts: Ok(vec!["a.sql"]), + }, + RelativePartsTestCase { + description: "an absolute path outside the project", + path: "/elsewhere/a.sql", + resolved: "/elsewhere/a.sql", + under: None, + expected_parts: Err(RelativePartsError::Escapes), + }, + RelativePartsTestCase { + description: "a path outside the requested folder", + path: "models/a.sql", + resolved: "", + under: Some("rules"), + expected_parts: Err(RelativePartsError::NotBeneath), + }, + RelativePartsTestCase { + description: "a relative path is beneath the current folder", + path: "a.sql", + resolved: "", + under: Some("."), + expected_parts: Ok(vec!["a.sql"]), + }, + ]; + for test_case in test_cases { + let parts = project_relative_parts( + Path::new(test_case.path), + Path::new("/project"), + test_case.under.map(Path::new), + |_| Ok::(PathBuf::from(test_case.resolved)), + ) + .map(|parts| { + parts + .into_iter() + .map(|part| part.to_string_lossy().into_owned()) + .collect::>() + }); + assert_eq!( + parts + .as_ref() + .map(|parts| parts.iter().map(String::as_str).collect::>()) + .map_err(Clone::clone), + test_case.expected_parts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs new file mode 100644 index 0000000000..7918aa2a67 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_types.rs @@ -0,0 +1,46 @@ +#[cfg(unix)] +use crate::project_tree::errors::{ProjectTreeFileError, RelativePartsError}; + +#[cfg(unix)] +pub(crate) struct ProjectTreeEntriesTestCase { + pub(crate) description: &'static str, + pub(crate) expected_entries: Vec<(&'static str, bool)>, +} + +#[cfg(unix)] +pub(crate) struct ProjectTreeFileTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + /// The project-relative file read, or why none may be. + pub(crate) expected_file: Result<&'static str, ProjectTreeFileError>, +} + +pub(crate) struct PathSelectionTestCase { + pub(crate) description: &'static str, + pub(crate) argument: Option<&'static str>, + pub(crate) expected_children: Vec<&'static str>, + pub(crate) expected_descendants: Vec<&'static str>, + pub(crate) expected_glob: Vec<&'static str>, +} + +#[cfg(unix)] +pub(crate) struct RelativePartsTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) resolved: &'static str, + pub(crate) under: Option<&'static str>, + pub(crate) expected_parts: Result, RelativePartsError<()>>, +} + +pub(crate) struct TrackedTextTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) expected_tracked: bool, +} + +#[cfg(unix)] +pub(crate) struct EscapesTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + pub(crate) expected_escapes: bool, +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 927e472f7c..6f6a815944 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -1,5 +1,7 @@ //! Threshold defaults, guidance wording and the rule codes guidance sections depend on. +use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; + /// Effective threshold defaults, sorted by name as guidance renders them. pub(crate) const THRESHOLD_DEFAULTS: [(&str, u32); 10] = [ ("max_constant_container_files", 10), @@ -8,7 +10,7 @@ pub(crate) const THRESHOLD_DEFAULTS: [(&str, u32); 10] = [ ("max_role_container_depth", 1), ("max_subdomain_depth", 1), ("min_audits_per_model", 1), - ("min_custom_rule_test_cases", 1), + (MIN_CUSTOM_RULE_TEST_CASES_KEY, 1), ("min_shared_container_prefix_files", 2), ("min_shared_owner_prefix_directories", 2), ("min_tests_per_model", 1), diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index e27f63926b..339a58906f 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -1,5 +1,6 @@ use crate::configuration::main::setting_help::setting_help; use crate::configuration::main::toml_string_array::toml_string_array; +use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use crate::constants::{ DECLARATION_DOMAIN_COMPONENTS, ENFORCED_CONTRACT, NEGATION_OPERATOR, REFERENCE_KIND, SOURCE_REFERENCE_KIND, VIEW_MATERIALIZATION, @@ -14,6 +15,7 @@ use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, }; use globset::{Glob, GlobSetBuilder}; +use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_identifier; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy as rules_quote_policy; @@ -971,12 +973,12 @@ pub(super) fn parse_name(name: &str) -> Option { "mart", "mart_v", ]; - if !valid_lower_name_token(parts[0]) + if !is_lowercase_identifier(parts[0]) || !layers.contains(&parts[1]) - || !valid_lower_name_token(parts[2]) + || !is_lowercase_identifier(parts[2]) || parts .get(3) - .is_some_and(|value| !valid_lower_name_token(value)) + .is_some_and(|value| !is_lowercase_identifier(value)) { return None; } @@ -989,12 +991,6 @@ pub(super) fn parse_name(name: &str) -> Option { }) } -fn valid_lower_name_token(value: &str) -> bool { - let mut bytes = value.bytes(); - bytes.next().is_some_and(|byte| byte.is_ascii_lowercase()) - && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') -} - fn view_marker(parsed: &ParsedModel<'_>, rule: &RuleMetadata, faults: &FaultCollector) { let Some(parts) = parse_name(&parsed.model.name) else { return; @@ -1492,7 +1488,7 @@ fn custom_rule_test_coverage( let (selected, rule) = policy; let minimum = config .thresholds - .get("min_custom_rule_test_cases") + .get(MIN_CUSTOM_RULE_TEST_CASES_KEY) .copied() .unwrap_or(1); if minimum == 0 { diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index 8a9bf4039f..ce66e8fcf6 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -7,6 +7,7 @@ use crate::constants::{ use crate::models::{Fault, Model, RuleMetadata}; use crate::rules::_helpers::evaluation::parse_name; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const NO_CANONICAL_LAYER_FOLDER: &str = "no canonical layer folder"; @@ -85,7 +86,7 @@ pub(crate) fn internal_forward_refs( faults.push(Fault { unevaluated: false, code: rule.code.clone(), - path: "sqlbuild_project.toml".into(), + path: PROJECT_CONFIG_FILENAME.into(), line: 1, column: 1, message: format!( diff --git a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs index 4c9fe1d5ed..6b5419602d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/sql_test_rules.rs @@ -20,7 +20,7 @@ const UNIT_ROOT: &str = "tests/unit"; const SCENARIO_ROOT: &str = "tests/scenarios"; const EMPTY_INPUT_RULE_CODE: &str = "SQBRTEST203"; const ALLOWED_TESTS_OPTION: &str = "allowed_tests"; -const PROJECT_CONFIG_PATH: &str = "sqlbuild_project.toml"; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_PATH; pub(crate) fn evaluate_project( evaluation: ProjectEvaluationRequest<'_>, diff --git a/crates/sqlbuild-scopes/src/scope_cache/constants.rs b/crates/sqlbuild-scopes/src/scope_cache/constants.rs index f30b5208ef..5f261c2754 100644 --- a/crates/sqlbuild-scopes/src/scope_cache/constants.rs +++ b/crates/sqlbuild-scopes/src/scope_cache/constants.rs @@ -1,5 +1,7 @@ //! Cache schema and fingerprint versions, keys and the scope-relevant source facts. +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + pub(crate) const CACHE_SCHEMA_VERSION: i64 = 3; pub(crate) const FINGERPRINT_ALGORITHM_VERSION: i64 = 2; /// The largest cache file read; bigger files are misses. @@ -38,7 +40,7 @@ pub(crate) const COMPLETENESS_KEYS: [&str; 6] = [ ]; /// Config files the project fingerprint hashes when present. pub(crate) const FINGERPRINT_CONFIG_FILES: [&str; 4] = [ - "sqlbuild_project.toml", + PROJECT_CONFIG_FILENAME, "sqlbuild_local.toml", ".sqlbuildignore", ".gitignore", diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs index 4541c5dc2c..6d2af0501d 100644 --- a/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/layout.rs @@ -1,6 +1,7 @@ //! Fields, headers, connectors, paths and follow-up commands of scope text output. use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::shell_text::main::shell_quote::shell_quote; use crate::scope_report::models::{DiagnosticFacts, ReportDiagnostic, ScopeSection}; use crate::scope_text::constants::{ @@ -166,20 +167,6 @@ pub(crate) fn diagnostic_lines( lines } -/// Python's `shlex.quote`. -fn shell_quote(text: &str) -> String { - if text.is_empty() { - return "''".to_owned(); - } - let safe: bool = text - .chars() - .all(|character| character.is_ascii_alphanumeric() || "_@%+=:,./-".contains(character)); - if safe { - return text.to_owned(); - } - format!("'{}'", text.replace('\'', "'\"'\"'")) -} - /// What a follow-up command adds to the request's own options. #[derive(Clone, Copy, Default)] pub(crate) struct FollowUp<'a> { diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs new file mode 100644 index 0000000000..a9cfd1dbe1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/code_text.rs @@ -0,0 +1,114 @@ +//! SQL code with comments blanked and literals emptied, under one escape and bracket reading. + +/// How ambiguous lexemes are read. +#[derive(Clone, Copy)] +pub(crate) struct LexicalMode { + /// A backslash escapes the next character inside single-quoted literals. + pub(crate) backslash_escapes: bool, + /// `[...]` delimits a quoted identifier rather than plain code. + pub(crate) bracket_identifiers: bool, +} + +/// The next lexeme start at or after `position`: `--`, `/*`, a quote, `$` or `[`. +fn lexeme_start(bytes: &[u8], position: usize) -> Option { + (position..bytes.len()).find(|index| match bytes[*index] { + b'\'' | b'"' | b'`' | b'$' | b'[' => true, + b'-' => bytes.get(index + 1) == Some(&b'-'), + b'/' => bytes.get(index + 1) == Some(&b'*'), + _ => false, + }) +} + +/// Python's `text.find(needle, position)` as a byte offset. +fn find(text: &str, needle: &str, position: usize) -> Option { + text.get(position..)? + .find(needle) + .map(|offset| position + offset) +} + +/// The closing quote of a single-quoted literal opened at `begin`. +fn literal_end(text: &str, begin: usize, backslash_escapes: bool) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut position: usize = begin + 1; + loop { + let rest: &[u8] = bytes.get(position..)?; + let offset: usize = rest + .iter() + .position(|byte| *byte == b'\'' || (backslash_escapes && *byte == b'\\'))?; + let stop: usize = position + offset; + if bytes[stop] == b'\\' { + let escaped: char = text[stop + 1..].chars().next()?; + position = stop + 1 + escaped.len_utf8(); + continue; + } + if bytes.get(stop + 1) == Some(&b'\'') { + position = stop + 2; + continue; + } + return Some(stop); + } +} + +/// The closing `quote` of quoted text opened at `begin`; doubled quotes escape except `]`. +fn quoted_end(text: &str, begin: usize, quote: u8) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut position: usize = begin + 1; + loop { + let offset: usize = bytes + .get(position..)? + .iter() + .position(|byte| *byte == quote)?; + let end: usize = position + offset; + if quote == b']' || bytes.get(end + 1) != Some("e) { + return Some(end); + } + position = end + 2; + } +} + +/// The end of a `$tag$` opener at `begin`, or `None` when `$` starts no tag. +fn dollar_tag_end(bytes: &[u8], begin: usize) -> Option { + let length: usize = bytes[begin + 1..] + .iter() + .take_while(|byte| byte.is_ascii_alphabetic() || **byte == b'_') + .count(); + let close: usize = begin + 1 + length; + (bytes.get(close) == Some(&b'$')).then_some(close + 1) +} + +/// One lexeme's code replacement and the position after it; `None` when it never closes. +fn read_lexeme(text: &str, begin: usize, mode: LexicalMode) -> Option<(&str, usize)> { + let bytes: &[u8] = text.as_bytes(); + match bytes[begin] { + b'-' => Some((" ", find(text, "\n", begin).unwrap_or(text.len()))), + b'/' => find(text, "*/", begin + 2).map(|end| (" ", end + 2)), + b'\'' => literal_end(text, begin, mode.backslash_escapes).map(|end| ("''", end + 1)), + b'$' => match dollar_tag_end(bytes, begin) { + None => Some(("$", begin + 1)), + Some(tag_end) => { + let tag: &str = &text[begin..tag_end]; + find(text, tag, tag_end).map(|end| ("''", end + tag.len())) + } + }, + b'[' if !mode.bracket_identifiers => Some(("[", begin + 1)), + quote => { + let closing: u8 = if quote == b'[' { b']' } else { quote }; + quoted_end(text, begin, closing).map(|end| (&text[begin..=end], end + 1)) + } + } +} + +/// `sql` with comments as spaces and literals as `''`, or `None` when a lexeme never closes. +pub(crate) fn code_text(sql: &str, mode: LexicalMode) -> Option { + let bytes: &[u8] = sql.as_bytes(); + let mut code: String = String::with_capacity(sql.len()); + let mut position: usize = 0; + while let Some(start) = lexeme_start(bytes, position) { + code.push_str(&sql[position..start]); + let (replacement, next) = read_lexeme(sql, start, mode)?; + code.push_str(replacement); + position = next; + } + code.push_str(&sql[position..]); + Some(code) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs new file mode 100644 index 0000000000..216e517e53 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/matching.rs @@ -0,0 +1,86 @@ +//! Anchored matching primitives with Python `re` whitespace and word-boundary semantics. + +use sqlbuild_core::text::main::python_isalnum::python_isalnum; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +/// Python's `\w`: alphanumeric or `_`. +fn is_word(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +/// The position after any whitespace at `position` (`\s*+`). +pub(crate) fn skip_whitespace(text: &str, position: usize) -> usize { + let rest: &str = &text[position..]; + position + (rest.len() - rest.trim_start_matches(python_isspace).len()) +} + +/// The position after at least one whitespace character at `position` (`\s++`). +pub(crate) fn whitespace(text: &str, position: usize) -> Option { + let end: usize = skip_whitespace(text, position); + (end > position).then_some(end) +} + +/// Whether only whitespace remains from `position` (`\s*+$`). +pub(crate) fn only_whitespace(text: &str, position: usize) -> bool { + skip_whitespace(text, position) == text.len() +} + +/// Whether `position` is a word boundary (`\b`). +pub(crate) fn word_boundary(text: &str, position: usize) -> bool { + let before: bool = text[..position].chars().next_back().is_some_and(is_word); + let after: bool = text[position..].chars().next().is_some_and(is_word); + before != after +} + +/// The position after an ASCII case-insensitive `word` at `position`. +pub(crate) fn word(text: &str, position: usize, word: &str) -> Option { + let end: usize = position + word.len(); + text.as_bytes() + .get(position..end) + .filter(|candidate| candidate.eq_ignore_ascii_case(word.as_bytes())) + .map(|_| end) +} + +/// The position after `words` separated by whitespace (`A\s++B`). +pub(crate) fn phrase(text: &str, position: usize, words: &[&str]) -> Option { + let mut end: usize = position; + for (index, expected) in words.iter().enumerate() { + if index > 0 { + end = whitespace(text, end)?; + } + end = word(text, end, expected)?; + } + Some(end) +} + +/// The position after the first of `phrases` that matches and is followed by whitespace. +pub(crate) fn phrase_then_whitespace( + text: &str, + position: usize, + phrases: &[&[&str]], +) -> Option { + phrases + .iter() + .find_map(|words| whitespace(text, phrase(text, position, words)?)) +} + +/// The position after `words` when it ends at a word boundary (`A\b`). +fn bounded_words(text: &str, position: usize, words: &[&str]) -> Option { + phrase(text, position, words).filter(|end| word_boundary(text, *end)) +} + +/// The position after the first of `phrases` that ends at a word boundary (`(?:A|B)\b`). +pub(crate) fn bounded_phrase(text: &str, position: usize, phrases: &[&[&str]]) -> Option { + phrases + .iter() + .find_map(|words| bounded_words(text, position, words)) +} + +/// The position after any repetitions of `phrases`, each followed by whitespace (`(?:..\s++)*+`). +pub(crate) fn repeated_modifiers(text: &str, position: usize, phrases: &[&[&str]]) -> usize { + let mut end: usize = position; + while let Some(next) = phrase_then_whitespace(text, end, phrases) { + end = next; + } + end +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs new file mode 100644 index 0000000000..a979b2d312 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod code_text; +pub(crate) mod matching; +pub(crate) mod names; +pub(crate) mod relations; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs new file mode 100644 index 0000000000..d911ba183e --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/names.rs @@ -0,0 +1,128 @@ +//! Quoted and unquoted relation name parts and their invalidation keys. + +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::ddl_effects::_helpers::matching::skip_whitespace; +use crate::ddl_effects::constants::QUOTED_IDENTIFIER_MIN_LENGTH; + +/// Which characters an unquoted name part may contain. +#[derive(Clone, Copy)] +pub(crate) enum UnquotedPart { + /// SQL text: starts with a letter, digit or `_`, then may also contain `$`. + Statement, + /// Qualified names from callers: letters, digits, `_`, `$` and `-`. + Loose, +} + +/// A qualified name: where it ends and its last part. +pub(crate) struct QualifiedName<'a> { + pub(crate) end: usize, + pub(crate) last_part: &'a str, +} + +/// The end of the text after `open` up to and including the next `close`. +fn closed_by(text: &str, open: usize, close: u8) -> Option { + let rest: &[u8] = text.as_bytes().get(open + 1..)?; + rest.iter() + .position(|byte| *byte == close) + .map(|offset| open + offset + 2) +} + +/// The end of a double-quoted part whose doubled quotes are escapes. +fn double_quoted(text: &str, open: usize) -> Option { + let bytes: &[u8] = text.as_bytes(); + let mut position: usize = open + 1; + loop { + match bytes.get(position) { + None => return None, + Some(b'"') if bytes.get(position + 1) == Some(&b'"') => position += 2, + Some(b'"') => return Some(position + 1), + Some(_) => position += 1, + } + } +} + +fn unquoted(text: &str, start: usize, kind: UnquotedPart) -> Option { + let bytes: &[u8] = text.as_bytes(); + let first_allowed = |byte: u8| match kind { + UnquotedPart::Statement => byte.is_ascii_alphanumeric() || byte == b'_', + UnquotedPart::Loose => byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$' | b'-'), + }; + let next_allowed = |byte: u8| match kind { + UnquotedPart::Statement => byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$'), + UnquotedPart::Loose => first_allowed(byte), + }; + if !first_allowed(*bytes.get(start)?) { + return None; + } + let length: usize = bytes[start + 1..] + .iter() + .take_while(|byte| next_allowed(**byte)) + .count(); + Some(start + 1 + length) +} + +/// The end of one name part starting at `start`. +pub(crate) fn name_part(text: &str, start: usize, kind: UnquotedPart) -> Option { + match text.as_bytes().get(start)? { + b'"' => double_quoted(text, start), + b'`' => closed_by(text, start, b'`'), + b'[' => closed_by(text, start, b']'), + _ => unquoted(text, start, kind), + } +} + +/// One or more parts separated by dots with optional whitespace around them. +pub(crate) fn qualified_name( + text: &str, + start: usize, + kind: UnquotedPart, +) -> Option> { + let mut part_start: usize = start; + let mut end: usize = name_part(text, start, kind)?; + loop { + let dot: usize = skip_whitespace(text, end); + if text.as_bytes().get(dot) != Some(&b'.') { + break; + } + let next_start: usize = skip_whitespace(text, dot + 1); + let Some(next_end) = name_part(text, next_start, kind) else { + break; + }; + part_start = next_start; + end = next_end; + } + Some(QualifiedName { + end, + last_part: &text[part_start..end], + }) +} + +/// A part without its surrounding identifier quotes, unescaping doubled double quotes. +fn strip_identifier_quotes(part: &str) -> String { + let bytes: &[u8] = part.as_bytes(); + if bytes.len() < QUOTED_IDENTIFIER_MIN_LENGTH { + return part.to_owned(); + } + let inner = || &part[1..part.len() - 1]; + match (bytes[0], bytes[bytes.len() - 1]) { + (b'"', b'"') => inner().replace("\"\"", "\""), + (b'`', b'`') | (b'[', b']') => inner().to_owned(), + _ => part.to_owned(), + } +} + +/// The invalidation key of one unquoted or quoted unqualified relation name. +pub(crate) fn name_key(name: &str) -> String { + python_casefold(&strip_identifier_quotes(name)) +} + +/// The key of a qualified name's last part; a backticked part keys its last dotted segment. +pub(crate) fn part_name_key(part: &str) -> String { + if !part.starts_with('`') { + return name_key(part); + } + let stripped: String = strip_identifier_quotes(part); + let segment: &str = stripped.rsplit('.').next().unwrap_or_default(); + python_casefold(segment) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs new file mode 100644 index 0000000000..c33a15d955 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/_helpers/relations.rs @@ -0,0 +1,227 @@ +//! Relation names a statement's code text creates, drops, alters or loads. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::ddl_effects::_helpers::matching::{ + bounded_phrase, only_whitespace, phrase, phrase_then_whitespace, repeated_modifiers, + skip_whitespace, whitespace, word, word_boundary, +}; +use crate::ddl_effects::_helpers::names::{UnquotedPart, part_name_key, qualified_name}; +use crate::ddl_effects::constants::{ + ALTER_FOLLOWING_WORDS, COPY_INTO, CREATE_FOLLOWING_WORDS, CREATE_SCHEMA_IF_NOT_EXISTS, + DROP_FOLLOWING_WORDS, FROM_WORD, IF_EXISTS, IF_NOT_EXISTS, INTO, NEUTRAL_OBJECT_KINDS, + NEUTRAL_OBJECT_LEADS, NEUTRAL_OBJECT_MODIFIERS, OR_REPLACE, QUERY_STATEMENTS, + READ_OR_DATA_STATEMENTS, RELATION_DDL_STATEMENTS, RELATION_KIND_MODIFIERS, RELATION_KINDS, + RENAMING_ALTERS, STATEMENT_SEPARATOR, TRANSACTION_END_STATEMENTS, +}; +use crate::ddl_effects::models::StatementMetadataEffect; + +/// Which statement form named the relation, deciding what may follow its name. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Form { + Create, + Drop, + Alter, + CopyInto, + SelectInto, + SecondRelation, +} + +/// A matched relation name's key and the text after the name. +struct NamedRelation<'a> { + key: String, + rest: &'a str, +} + +/// The effect of every relation in the statement possibly changing. +pub(crate) fn all_relations() -> StatementMetadataEffect { + StatementMetadataEffect { + invalidates_all: true, + ..StatementMetadataEffect::default() + } +} + +/// A qualified name at `start` followed by whitespace, `(` or the end. +fn named_relation(text: &str, start: usize) -> Option> { + let name = qualified_name(text, start, UnquotedPart::Statement)?; + let rest: &str = &text[name.end..]; + let delimited: bool = rest + .chars() + .next() + .is_none_or(|next| next == '(' || python_isspace(next)); + delimited.then(|| NamedRelation { + key: part_name_key(name.last_part), + rest, + }) +} + +/// `LEAD\s++` modifiers `(TABLE|VIEW)\s++` and an optional `guard\s++` before the name. +fn guarded_relation<'a>(text: &'a str, lead: &str, guard: &[&str]) -> Option> { + let after_lead: usize = whitespace(text, word(text, 0, lead)?)?; + let after_modifiers: usize = repeated_modifiers(text, after_lead, RELATION_KIND_MODIFIERS); + let name_start: usize = phrase_then_whitespace(text, after_modifiers, RELATION_KINDS)?; + phrase(text, name_start, guard) + .and_then(|end| whitespace(text, end)) + .and_then(|guarded| named_relation(text, guarded)) + .or_else(|| named_relation(text, name_start)) +} + +fn copy_into_relation(text: &str) -> Option> { + named_relation(text, whitespace(text, phrase(text, 0, COPY_INTO)?)?) +} + +fn second_relation(rest: &str) -> Option> { + let clause: usize = whitespace(rest, 0)?; + named_relation(rest, phrase_then_whitespace(rest, clause, RENAMING_ALTERS)?) +} + +/// The first `\bINTO\s++name` in the statement. +fn select_into_relation(text: &str) -> Option> { + text.char_indices() + .filter(|(index, _)| word(text, *index, INTO).is_some() && word_boundary(text, *index)) + .find_map(|(index, _)| named_relation(text, whitespace(text, index + INTO.len())?)) +} + +fn has_into_keyword(text: &str) -> bool { + text.char_indices().any(|(index, _)| { + word(text, index, INTO) + .is_some_and(|end| word_boundary(text, index) && word_boundary(text, end)) + }) +} + +/// Whether `rest` is text the form allows after the relation name. +fn allowed_following(form: Form, rest: &str) -> bool { + match form { + Form::Create => { + let start: usize = skip_whitespace(rest, 0); + start == rest.len() + || rest[start..].starts_with('(') + || bounded_phrase(rest, start, CREATE_FOLLOWING_WORDS).is_some() + } + Form::Drop => { + let start: usize = skip_whitespace(rest, 0); + let end: usize = DROP_FOLLOWING_WORDS + .iter() + .find_map(|words| phrase(rest, start, words)) + .unwrap_or(start); + only_whitespace(rest, end) + } + Form::Alter => whitespace(rest, 0) + .is_some_and(|start| bounded_phrase(rest, start, ALTER_FOLLOWING_WORDS).is_some()), + Form::CopyInto | Form::SelectInto => whitespace(rest, 0) + .is_some_and(|start| bounded_phrase(rest, start, &[FROM_WORD]).is_some()), + Form::SecondRelation => only_whitespace(rest, 0), + } +} + +/// The relation keys a matched name names, or `None` when what follows is ambiguous. +fn unambiguous_names(form: Form, relation: Option>) -> Option> { + let relation: NamedRelation<'_> = relation?; + if !allowed_following(form, relation.rest) { + return None; + } + let renaming: bool = form == Form::Alter + && whitespace(relation.rest, 0) + .is_some_and(|start| bounded_phrase(relation.rest, start, RENAMING_ALTERS).is_some()); + let mut names: BTreeSet = BTreeSet::from([relation.key]); + if renaming { + names.extend(unambiguous_names( + Form::SecondRelation, + second_relation(relation.rest), + )?); + } + Some(names) +} + +fn named_effect(names: BTreeSet) -> StatementMetadataEffect { + StatementMetadataEffect { + relation_names: names, + ..StatementMetadataEffect::default() + } +} + +fn select_into_effect(statement: &str) -> StatementMetadataEffect { + if !has_into_keyword(statement) { + return StatementMetadataEffect::default(); + } + unambiguous_names(Form::SelectInto, select_into_relation(statement)) + .map_or_else(all_relations, named_effect) +} + +fn relation_ddl_effect(statement: &str) -> StatementMetadataEffect { + [ + ( + Form::Create, + guarded_relation(statement, "CREATE", IF_NOT_EXISTS), + ), + (Form::Drop, guarded_relation(statement, "DROP", IF_EXISTS)), + (Form::Alter, guarded_relation(statement, "ALTER", IF_EXISTS)), + (Form::CopyInto, copy_into_relation(statement)), + ] + .into_iter() + .find_map(|(form, relation)| unambiguous_names(form, relation)) + .map_or_else(all_relations, named_effect) +} + +fn creates_schema_if_missing(statement: &str) -> bool { + let Some(name_start) = phrase(statement, 0, CREATE_SCHEMA_IF_NOT_EXISTS) + .and_then(|end| whitespace(statement, end)) + else { + return false; + }; + let name_length: usize = statement[name_start..] + .chars() + .take_while(|character| *character != STATEMENT_SEPARATOR && !python_isspace(*character)) + .map(char::len_utf8) + .sum(); + name_length > 0 && only_whitespace(statement, name_start + name_length) +} + +fn is_neutral_object(statement: &str) -> bool { + let Some(after_lead) = phrase_then_whitespace(statement, 0, NEUTRAL_OBJECT_LEADS) else { + return false; + }; + let after_replace: usize = + phrase_then_whitespace(statement, after_lead, &[OR_REPLACE]).unwrap_or(after_lead); + let kind_start: usize = repeated_modifiers(statement, after_replace, NEUTRAL_OBJECT_MODIFIERS); + bounded_phrase(statement, kind_start, NEUTRAL_OBJECT_KINDS).is_some() +} + +/// The effect of one reading of the statement's code text; `None` code changes everything. +pub(crate) fn code_effect(code: Option<&str>) -> StatementMetadataEffect { + let Some(code) = code else { + return all_relations(); + }; + let statement: &str = python_strip(python_strip(code).trim_end_matches(STATEMENT_SEPARATOR)) + .trim_start_matches('('); + let first_length: usize = statement + .bytes() + .take_while(u8::is_ascii_alphabetic) + .count(); + if first_length == 0 || statement.contains(STATEMENT_SEPARATOR) { + return all_relations(); + } + let keyword: String = statement[..first_length].to_ascii_uppercase(); + if QUERY_STATEMENTS.contains(&keyword.as_str()) { + return select_into_effect(statement); + } + if READ_OR_DATA_STATEMENTS.contains(&keyword.as_str()) { + return StatementMetadataEffect::default(); + } + if TRANSACTION_END_STATEMENTS.contains(&keyword.as_str()) { + return StatementMetadataEffect { + ends_transaction: true, + ..StatementMetadataEffect::default() + }; + } + if !RELATION_DDL_STATEMENTS.contains(&keyword.as_str()) { + return all_relations(); + } + if creates_schema_if_missing(statement) || is_neutral_object(statement) { + return StatementMetadataEffect::default(); + } + relation_ddl_effect(statement) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs new file mode 100644 index 0000000000..5c5c822933 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/constants.rs @@ -0,0 +1,135 @@ +//! Statement keywords that decide how executed SQL affects cached relation metadata. + +/// Statements that only read, unless they select into a new table. +pub(crate) const QUERY_STATEMENTS: &[&str] = &["SELECT", "WITH"]; +/// Statements that read, change data or session state, but never relation existence or columns. +pub(crate) const READ_OR_DATA_STATEMENTS: &[&str] = &[ + "VALUES", + "SHOW", + "DESCRIBE", + "DESC", + "EXPLAIN", + "INSERT", + "UPDATE", + "DELETE", + "MERGE", + "TRUNCATE", + "BEGIN", + "START", + "SAVEPOINT", + "RELEASE", + "SET", + "UNSET", + "GRANT", + "REVOKE", + "COMMENT", + "PUT", + "GET", + "LIST", + "LS", + "REMOVE", + "RM", + "ANALYZE", +]; +/// Statements that end the current transaction. +pub(crate) const TRANSACTION_END_STATEMENTS: &[&str] = &["COMMIT", "ROLLBACK", "END", "ABORT"]; +/// Statements whose relation name is read from the statement. +pub(crate) const RELATION_DDL_STATEMENTS: &[&str] = &["CREATE", "DROP", "ALTER", "COPY"]; +/// Modifier phrases allowed between `CREATE`/`DROP`/`ALTER` and `TABLE`/`VIEW`. +pub(crate) const RELATION_KIND_MODIFIERS: &[&[&str]] = &[ + &["OR", "REPLACE"], + &["LOCAL"], + &["GLOBAL"], + &["TEMP"], + &["TEMPORARY"], + &["TRANSIENT"], + &["VOLATILE"], + &["SECURE"], + &["RECURSIVE"], + &["MATERIALIZED"], + &["EXTERNAL"], + &["DYNAMIC"], + &["ICEBERG"], + &["HYBRID"], + &["UNLOGGED"], + &["FOREIGN"], + &["STREAMING"], +]; +/// Relation kinds whose name follows the modifiers. +pub(crate) const RELATION_KINDS: &[&[&str]] = &[&["TABLE"], &["VIEW"]]; +/// The optional existence guard of `CREATE`. +pub(crate) const IF_NOT_EXISTS: &[&str] = &["IF", "NOT", "EXISTS"]; +/// The optional existence guard of `DROP` and `ALTER`. +pub(crate) const IF_EXISTS: &[&str] = &["IF", "EXISTS"]; +/// Words that may follow the created relation's name. +pub(crate) const CREATE_FOLLOWING_WORDS: &[&[&str]] = &[ + &["AS"], + &["CLONE"], + &["SHALLOW"], + &["DEEP"], + &["LIKE"], + &["COPY"], + &["USING"], + &["COMMENT"], + &["CLUSTER"], + &["PARTITION"], + &["OPTIONS"], + &["WITH"], + &["TBLPROPERTIES"], + &["LOCATION"], + &["DATA_RETENTION_TIME_IN_DAYS"], + &["CHANGE_TRACKING"], +]; +/// The optional trailing word of `DROP`. +pub(crate) const DROP_FOLLOWING_WORDS: &[&[&str]] = &[&["CASCADE"], &["RESTRICT"], &["PURGE"]]; +/// Words that may follow the altered relation's name. +pub(crate) const ALTER_FOLLOWING_WORDS: &[&[&str]] = &[ + &["ADD"], + &["DROP"], + &["ALTER"], + &["MODIFY"], + &["RENAME"], + &["SWAP"], + &["SET"], + &["UNSET"], + &["CLUSTER"], + &["RECLUSTER"], + &["OWNER"], + &["SUSPEND"], + &["RESUME"], + &["REFRESH"], +]; +/// Alter clauses that name a second relation. +pub(crate) const RENAMING_ALTERS: &[&[&str]] = &[&["RENAME", "TO"], &["SWAP", "WITH"]]; +/// The word that must follow the target of `COPY INTO` and `SELECT ... INTO`. +pub(crate) const FROM_WORD: &[&str] = &["FROM"]; +/// `COPY INTO` before the loaded relation. +pub(crate) const COPY_INTO: &[&str] = &["COPY", "INTO"]; +/// The `INTO` keyword of `SELECT ... INTO`. +pub(crate) const INTO: &str = "INTO"; +/// Statement leads whose object is never a relation. +pub(crate) const NEUTRAL_OBJECT_LEADS: &[&[&str]] = &[&["CREATE"], &["DROP"], &["ALTER"]]; +/// The optional replacement phrase of a neutral object statement. +pub(crate) const OR_REPLACE: &[&str] = &["OR", "REPLACE"]; +/// Modifiers allowed before a neutral object kind. +pub(crate) const NEUTRAL_OBJECT_MODIFIERS: &[&[&str]] = + &[&["SECURE"], &["TEMP"], &["TEMPORARY"], &["AGGREGATE"]]; +/// Object kinds whose statements never change relation existence or columns. +pub(crate) const NEUTRAL_OBJECT_KINDS: &[&[&str]] = &[ + &["FUNCTION"], + &["PROCEDURE"], + &["SEQUENCE"], + &["STAGE"], + &["FILE", "FORMAT"], + &["INDEX"], + &["UNIQUE", "INDEX"], + &["WAREHOUSE"], + &["MACRO"], +]; +/// `CREATE SCHEMA IF NOT EXISTS` before the schema name. +pub(crate) const CREATE_SCHEMA_IF_NOT_EXISTS: &[&str] = + &["CREATE", "SCHEMA", "IF", "NOT", "EXISTS"]; +/// The statement separator. +pub(crate) const STATEMENT_SEPARATOR: char = ';'; +/// The shortest name that can be wrapped in identifier quotes. +pub(crate) const QUOTED_IDENTIFIER_MIN_LENGTH: usize = 2; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs new file mode 100644 index 0000000000..afc03a0b13 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/mod.rs @@ -0,0 +1,3 @@ +pub mod qualified_relation_name_key; +pub mod relation_name_key; +pub mod statement_metadata_effect; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs new file mode 100644 index 0000000000..bd094f0910 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/qualified_relation_name_key.rs @@ -0,0 +1,15 @@ +//! The cache invalidation key of a qualified relation name's last part. + +use crate::ddl_effects::_helpers::matching::{only_whitespace, skip_whitespace}; +use crate::ddl_effects::_helpers::names::{UnquotedPart, part_name_key, qualified_name}; + +/// The key for the last part of a qualified name, or `None` when it cannot be parsed. +#[must_use] +pub fn qualified_relation_name_key(qualified: &str) -> Option { + let name = qualified_name( + qualified, + skip_whitespace(qualified, 0), + UnquotedPart::Loose, + )?; + only_whitespace(qualified, name.end).then(|| part_name_key(name.last_part)) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs new file mode 100644 index 0000000000..621d8fbb05 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/relation_name_key.rs @@ -0,0 +1,9 @@ +//! The cache invalidation key of one relation name. + +use crate::ddl_effects::_helpers::names::name_key; + +/// The invalidation key for one unquoted or quoted unqualified relation name. +#[must_use] +pub fn relation_name_key(name: &str) -> String { + name_key(name) +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs b/crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs new file mode 100644 index 0000000000..0022a087ac --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/main/statement_metadata_effect.rs @@ -0,0 +1,37 @@ +//! Classify executed SQL by its effect on cached relation existence and columns. + +use crate::ddl_effects::_helpers::code_text::{LexicalMode, code_text}; +use crate::ddl_effects::_helpers::relations::{all_relations, code_effect}; +use crate::ddl_effects::models::StatementMetadataEffect; + +/// Which cached relation names `sql` may have changed; readings that disagree change all. +#[must_use] +pub fn statement_metadata_effect(sql: &str) -> StatementMetadataEffect { + let backslash_modes: &[bool] = if sql.contains('\\') { + &[false, true] + } else { + &[false] + }; + let bracket_modes: &[bool] = if sql.contains('[') { + &[false, true] + } else { + &[false] + }; + let mut effects: Vec = Vec::new(); + for backslash_escapes in backslash_modes { + for bracket_identifiers in bracket_modes { + let mode: LexicalMode = LexicalMode { + backslash_escapes: *backslash_escapes, + bracket_identifiers: *bracket_identifiers, + }; + let effect: StatementMetadataEffect = code_effect(code_text(sql, mode).as_deref()); + if !effects.contains(&effect) { + effects.push(effect); + } + } + } + match <[StatementMetadataEffect; 1]>::try_from(effects) { + Ok([effect]) => effect, + Err(_) => all_relations(), + } +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/mod.rs b/crates/sqlbuild-sqltext/src/ddl_effects/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/models.rs b/crates/sqlbuild-sqltext/src/ddl_effects/models.rs new file mode 100644 index 0000000000..4952635ab0 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/models.rs @@ -0,0 +1,14 @@ +//! What one executed statement may have changed in cached relation metadata. + +use std::collections::BTreeSet; + +/// Cached relation names a statement may have changed; ambiguity changes all of them. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StatementMetadataEffect { + /// Invalidation keys of the relations the statement names. + pub relation_names: BTreeSet, + /// Every cached relation may have changed. + pub invalidates_all: bool, + /// The statement ends the current transaction. + pub ends_transaction: bool, +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests.rs new file mode 100644 index 0000000000..b66f762b66 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_ddl_effects.rs"] +mod ddl_effects; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs new file mode 100644 index 0000000000..761ce6a14a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_ddl_effects.rs @@ -0,0 +1,223 @@ +//! Executed SQL is classified conservatively by its effect on cached relation metadata. + +use crate::ddl_effects::main::qualified_relation_name_key::qualified_relation_name_key; +use crate::ddl_effects::main::relation_name_key::relation_name_key; +use crate::ddl_effects::main::statement_metadata_effect::statement_metadata_effect; +use crate::ddl_effects::models::StatementMetadataEffect; +use crate::ddl_effects::tests::test_types::{NameKeyTestCase, StatementEffectTestCase}; + +#[test] +fn given_executed_statement_when_classifying_then_reports_possible_metadata_changes() { + let test_cases = [ + StatementEffectTestCase { + description: "create or replace transient table", + sql: "CREATE OR REPLACE TRANSIENT TABLE analytics.marts.orders__delta AS SELECT 1 AS id", + expected_relation_names: &["orders__delta"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "quoted three-part view", + sql: r#"CREATE OR REPLACE VIEW "ANALYTICS"."MARTS"."ORDERS_V" AS SELECT 1 AS id"#, + expected_relation_names: &["orders_v"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "rename names both relations", + sql: "ALTER TABLE marts.orders__stage RENAME TO marts.orders", + expected_relation_names: &["orders", "orders__stage"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "backtick qualified name keys its last segment", + sql: "CREATE OR REPLACE TABLE `project.dataset.Orders` AS SELECT 1 AS id", + expected_relation_names: &["orders"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "bracket qualified name", + sql: "DROP TABLE IF EXISTS [dbo].[orders__delta]", + expected_relation_names: &["orders__delta"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "select into after a common table expression", + sql: "WITH v AS (SELECT 1 AS id) SELECT id INTO dbo.orders_history FROM v", + expected_relation_names: &["orders_history"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "copy into a table", + sql: "COPY INTO raw.orders FROM @landing/orders", + expected_relation_names: &["orders"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "separator inside a literal is data", + sql: "CREATE OR REPLACE TABLE marts.notes AS SELECT 'a;b -- c' AS note", + expected_relation_names: &["notes"], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "function body in dollar quotes", + sql: "CREATE OR REPLACE FUNCTION f() RETURNS INT AS $$ SELECT 1; SELECT 2 $$", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "create schema if missing", + sql: "CREATE SCHEMA IF NOT EXISTS analytics.marts", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "procedure call", + sql: "CALL marts.rebuild_orders()", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "multi statement", + sql: "INSERT INTO t VALUES (1); DROP TABLE t", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "unterminated quoted name", + sql: r#"CREATE TABLE "open AS SELECT 1"#, + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "unquoted non-ascii name", + sql: "CREATE TABLE s.café (id INT)", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "backslash-escaped quote is ambiguous", + sql: r"SELECT 'a\' AS x FROM t WHERE y = '; DROP TABLE u'", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "rename target followed by more text", + sql: "ALTER TABLE a RENAME TO b c", + expected_relation_names: &[], + expected_invalidates_all: true, + expected_ends_transaction: false, + }, + StatementEffectTestCase { + description: "commit ends the transaction", + sql: "COMMIT;", + expected_relation_names: &[], + expected_invalidates_all: false, + expected_ends_transaction: true, + }, + ]; + for test_case in test_cases { + let effect: StatementMetadataEffect = statement_metadata_effect(test_case.sql); + assert_eq!( + effect + .relation_names + .iter() + .map(String::as_str) + .collect::>(), + test_case.expected_relation_names, + "{}", + test_case.description + ); + assert_eq!( + effect.invalidates_all, test_case.expected_invalidates_all, + "{}", + test_case.description + ); + assert_eq!( + effect.ends_transaction, test_case.expected_ends_transaction, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_qualified_name_when_keying_then_uses_last_part_or_reports_unparseable() { + let test_cases = [ + NameKeyTestCase { + description: "unquoted part starting with a digit", + name: "my-proj.ds.2024_orders", + expected_key: Some("2024_orders"), + }, + NameKeyTestCase { + description: "backticked path keys its last segment", + name: "`my-proj.ds.Orders`", + expected_key: Some("orders"), + }, + NameKeyTestCase { + description: "quoted dotted part keeps its dot", + name: r#""ANALYTICS"."MARTS"."my.table""#, + expected_key: Some("my.table"), + }, + NameKeyTestCase { + description: "trailing text is unparseable", + name: "ds.orders extra", + expected_key: None, + }, + NameKeyTestCase { + description: "unquoted non-ascii part is unparseable", + name: "ds.café", + expected_key: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + qualified_relation_name_key(test_case.name).as_deref(), + test_case.expected_key, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relation_name_when_keying_then_strips_quotes_and_casefolds() { + let test_cases = [ + NameKeyTestCase { + description: "doubled quotes unescape", + name: r#""Or""ders""#, + expected_key: Some(r#"or"ders"#), + }, + NameKeyTestCase { + description: "python casefold expands sharp s", + name: "STRAßE", + expected_key: Some("strasse"), + }, + NameKeyTestCase { + description: "bracketed name", + name: "[Orders]", + expected_key: Some("orders"), + }, + ]; + for test_case in test_cases { + assert_eq!( + Some(relation_name_key(test_case.name).as_str()), + test_case.expected_key, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs new file mode 100644 index 0000000000..c027cdfd78 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/ddl_effects/tests/test_types.rs @@ -0,0 +1,13 @@ +pub(super) struct StatementEffectTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_relation_names: &'static [&'static str], + pub(super) expected_invalidates_all: bool, + pub(super) expected_ends_transaction: bool, +} + +pub(super) struct NameKeyTestCase { + pub(super) description: &'static str, + pub(super) name: &'static str, + pub(super) expected_key: Option<&'static str>, +} diff --git a/crates/sqlbuild-sqltext/src/lib.rs b/crates/sqlbuild-sqltext/src/lib.rs index 59a15f977b..a1e0563af4 100644 --- a/crates/sqlbuild-sqltext/src/lib.rs +++ b/crates/sqlbuild-sqltext/src/lib.rs @@ -5,6 +5,7 @@ pub mod code_scanner; pub mod compiler; mod constants; +pub mod ddl_effects; pub mod resolution; pub mod sql_references; pub mod sql_scan; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs index b9bec2ab1a..f7d11818a4 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/constants.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/constants.rs @@ -6,3 +6,11 @@ pub(crate) const STRING_PREFIX_CHARACTERS: &[u8] = b"bBeErR"; pub(crate) const STRING_PREFIX_MAX_LENGTH: usize = 2; /// Length of a triple-quote string delimiter. pub(crate) const TRIPLE_QUOTE_LENGTH: usize = 3; +/// Line comment prefixes SQLBuild's scanners understand. +pub const SUPPORTED_LINE_COMMENT_PREFIXES: [&str; 3] = ["#", "--", "//"]; +/// The generic SQL line comment prefix. +pub const GENERIC_LINE_COMMENT_PREFIX: &str = "--"; +/// Quote characters a triple-quoted string may repeat. +pub const TRIPLE_QUOTES: [&str; 2] = ["'''", "\"\"\""]; +pub const ESCAPE_CHARACTER: char = '\\'; +pub const BLOCK_COMMENT_OPEN: &str = "/*"; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs new file mode 100644 index 0000000000..ff2cd882d3 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/lexical_syntax_key.rs @@ -0,0 +1,30 @@ +//! The process-stable text key of one dialect's lexical rules. + +use crate::sql_scan::models::LexicalSyntax; + +/// `SqlLexicalSyntax.cache_key`: sorted quotes, the four flags as Python booleans, sorted prefixes. +#[must_use] +pub fn lexical_syntax_key(syntax: &LexicalSyntax) -> String { + let flag = |value: bool| if value { "True" } else { "False" }; + let mut quotes: Vec<&str> = syntax + .backslash_escape_quotes + .iter() + .map(String::as_str) + .collect(); + quotes.sort_unstable(); + let mut prefixes: Vec<&str> = syntax + .line_comment_prefixes + .iter() + .map(String::as_str) + .collect(); + prefixes.sort_unstable(); + [ + quotes.join(","), + flag(syntax.escape_string_prefix).to_owned(), + flag(syntax.raw_string_prefix).to_owned(), + flag(syntax.triple_quoted_strings).to_owned(), + flag(syntax.nested_block_comments).to_owned(), + prefixes.join(","), + ] + .join("|") +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs index 2b5b0542f4..69cddeb649 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/mod.rs @@ -2,9 +2,12 @@ pub mod comment_end; pub mod dialect_matching_paren; pub mod dialect_non_code_end; pub mod dialect_non_code_ranges; +pub mod lexical_syntax_key; pub mod matching_paren; pub mod non_code_end; pub mod quote_end; pub mod quote_policy; +pub mod reads_differently_from_generic; pub mod rules_dialect; pub mod skip_whitespace; +pub mod unsupported_line_comment_prefixes; diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs new file mode 100644 index 0000000000..05479da834 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/reads_differently_from_generic.rs @@ -0,0 +1,26 @@ +//! Whether one dialect's lexical rules can read a SQL text differently from generic SQL. + +use crate::sql_scan::constants::{ + BLOCK_COMMENT_OPEN, ESCAPE_CHARACTER, GENERIC_LINE_COMMENT_PREFIX, TRIPLE_QUOTES, +}; +use crate::sql_scan::models::LexicalSyntax; + +/// `SqlLexicalSyntax.reads_differently_from_generic`. +#[must_use] +pub fn reads_differently_from_generic(syntax: &LexicalSyntax, sql: &str) -> bool { + if sql.contains(ESCAPE_CHARACTER) + && (!syntax.backslash_escape_quotes.is_empty() || syntax.escape_string_prefix) + { + return true; + } + if syntax.triple_quoted_strings && TRIPLE_QUOTES.iter().any(|quote| sql.contains(quote)) { + return true; + } + if syntax.nested_block_comments && sql.matches(BLOCK_COMMENT_OPEN).count() > 1 { + return true; + } + syntax + .line_comment_prefixes + .iter() + .any(|prefix| prefix != GENERIC_LINE_COMMENT_PREFIX && sql.contains(prefix.as_str())) +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs new file mode 100644 index 0000000000..02f3662da1 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/unsupported_line_comment_prefixes.rs @@ -0,0 +1,16 @@ +//! The line comment prefixes a dialect declares that SQLBuild cannot scan. + +use crate::sql_scan::constants::SUPPORTED_LINE_COMMENT_PREFIXES; + +/// The declared prefixes outside the supported set, sorted and without repeats. +#[must_use] +pub fn unsupported_line_comment_prefixes(prefixes: &[String]) -> Vec { + let mut unsupported: Vec = prefixes + .iter() + .filter(|prefix| !SUPPORTED_LINE_COMMENT_PREFIXES.contains(&prefix.as_str())) + .cloned() + .collect(); + unsupported.sort_unstable(); + unsupported.dedup(); + unsupported +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests.rs index ab9f7b24dc..3da57ecf98 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests.rs @@ -2,6 +2,8 @@ mod dialect_scan; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_lexical_rules.rs"] +mod lexical_rules; #[path = "tests/test_sql_scan.rs"] mod sql_scan; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs index 3a9c34da1a..4adf633977 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs @@ -41,3 +41,15 @@ pub(crate) fn raw_strings() -> LexicalSyntax { ..generic() } } + +/// Lexical rules with the given quotes, `[escape, raw, triple, nested]` flags and prefixes. +pub(crate) fn syntax(quotes: &[&str], flags: [bool; 4], prefixes: &[&str]) -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: quotes.iter().map(|quote| (*quote).to_owned()).collect(), + escape_string_prefix: flags[0], + raw_string_prefix: flags[1], + triple_quoted_strings: flags[2], + nested_block_comments: flags[3], + line_comment_prefixes: prefixes.iter().map(|prefix| (*prefix).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs new file mode 100644 index 0000000000..8d862507f7 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_lexical_rules.rs @@ -0,0 +1,82 @@ +use crate::sql_scan::main::lexical_syntax_key::lexical_syntax_key; +use crate::sql_scan::main::reads_differently_from_generic::reads_differently_from_generic; +use crate::sql_scan::main::unsupported_line_comment_prefixes::unsupported_line_comment_prefixes; +use crate::sql_scan::tests::helpers::syntax; +use crate::sql_scan::tests::test_types::{LexicalRulesTestCase, PrefixCheckTestCase}; + +#[test] +fn given_lexical_rules_when_keyed_and_compared_then_match_python_properties() { + let test_cases = [ + LexicalRulesTestCase { + description: "generic rules read SQL as generic SQL", + syntax: syntax(&[], [false; 4], &["--"]), + sql: "select '\\' -- note", + expected_key: "|False|False|False|False|--", + expected_reads_differently: false, + }, + LexicalRulesTestCase { + description: "backslash quotes change escaped text", + syntax: syntax(&["\"", "'"], [false, true, false, false], &["--", "#"]), + sql: "select 'a\\'", + expected_key: "\",'|False|True|False|False|#,--", + expected_reads_differently: true, + }, + LexicalRulesTestCase { + description: "nested comments need two openers", + syntax: syntax(&[], [false, false, false, true], &["--"]), + sql: "/* a /* b */ */ select 1", + expected_key: "|False|False|False|True|--", + expected_reads_differently: true, + }, + LexicalRulesTestCase { + description: "a hash comment prefix reads hash text", + syntax: syntax(&[], [false; 4], &["--", "#"]), + sql: "select 1 # trailing", + expected_key: "|False|False|False|False|#,--", + expected_reads_differently: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + lexical_syntax_key(&test_case.syntax), + reads_differently_from_generic(&test_case.syntax, test_case.sql), + ), + ( + test_case.expected_key.to_owned(), + test_case.expected_reads_differently + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_declared_prefixes_when_checking_then_only_unsupported_ones_are_returned() { + let test_cases = [ + PrefixCheckTestCase { + description: "unsupported prefixes are sorted without repeats", + declared: &["%", "--", "%", ";"], + expected_unsupported: &["%", ";"], + }, + PrefixCheckTestCase { + description: "supported prefixes pass", + declared: &["#", "//", "--"], + expected_unsupported: &[], + }, + ]; + for test_case in test_cases { + let declared: Vec = test_case + .declared + .iter() + .map(|prefix| (*prefix).to_owned()) + .collect(); + assert_eq!( + unsupported_line_comment_prefixes(&declared), + test_case.expected_unsupported, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs index ba72c5cbfb..86883ba217 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs @@ -1,5 +1,5 @@ -use crate::sql_scan::models::QuotePolicy; use crate::sql_scan::models::Unclosed; +use crate::sql_scan::models::{LexicalSyntax, QuotePolicy}; pub(crate) struct MatchingParenPolicyTestCase { pub(crate) description: &'static str, @@ -28,3 +28,17 @@ pub(crate) struct DialectNonCodeRangesTestCase { pub(crate) syntax: crate::sql_scan::models::LexicalSyntax, pub(crate) expected_marker_protected: bool, } + +pub(crate) struct LexicalRulesTestCase { + pub(crate) description: &'static str, + pub(crate) syntax: LexicalSyntax, + pub(crate) sql: &'static str, + pub(crate) expected_key: &'static str, + pub(crate) expected_reads_differently: bool, +} + +pub(crate) struct PrefixCheckTestCase { + pub(crate) description: &'static str, + pub(crate) declared: &'static [&'static str], + pub(crate) expected_unsupported: &'static [&'static str], +} diff --git a/crates/sqlbuild-state/src/events/constants.rs b/crates/sqlbuild-state/src/events/constants.rs index f0cf958db1..c89814977d 100644 --- a/crates/sqlbuild-state/src/events/constants.rs +++ b/crates/sqlbuild-state/src/events/constants.rs @@ -15,9 +15,6 @@ pub(crate) const ALIAS_KEY_SEPARATOR: &str = ":"; pub(crate) const PYTHON_TRUE: &str = "True"; pub(crate) const PYTHON_FALSE: &str = "False"; pub(crate) const PYTHON_NONE: &str = "None"; -pub const MIGRATION_TABLE_NAME: &str = "_sqlbuild_migrations"; -pub const COLUMN_MIGRATION_TABLE_NAME: &str = "_sqlbuild_column_migrations"; -pub const OLD_NAME_VIEW_TABLE_NAME: &str = "_sqlbuild_old_name_views"; /// Event columns stored as text timestamps; every other event column is a string. pub const TIMESTAMP_COLUMNS: [&str; 2] = ["expires_at", "created_at"]; pub const MIGRATION_COLUMNS: [&str; 15] = [ diff --git a/crates/sqlbuild-state/src/events/main/microbatch_event_id.rs b/crates/sqlbuild-state/src/events/main/microbatch_event_id.rs new file mode 100644 index 0000000000..bd71669b91 --- /dev/null +++ b/crates/sqlbuild-state/src/events/main/microbatch_event_id.rs @@ -0,0 +1,25 @@ +//! `microbatch_event_id`: the content-addressed ID of one logical microbatch event. + +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::models::JsonValue; + +use crate::events::_helpers::identity::{json_digest, optional_text, text}; +use crate::microbatches::models::MicrobatchEventIdentity; + +/// SHA-256 of the compact, ASCII-escaped JSON array of the identity fields. +pub fn microbatch_event_id(identity: &MicrobatchEventIdentity) -> Result { + let value = JsonValue::Array(vec![ + text(&identity.scope_kind), + text(&identity.scope_key), + text(&identity.model_name), + optional_text(identity.target_database.as_deref()), + optional_text(identity.target_schema.as_deref()), + text(&identity.target_name), + text(&identity.physical_generation_id), + text(&identity.record_type), + optional_text(identity.partition_start.as_deref()), + optional_text(identity.partition_end.as_deref()), + text(&identity.completion_reason), + ]); + json_digest(&value, false) +} diff --git a/crates/sqlbuild-state/src/events/main/mod.rs b/crates/sqlbuild-state/src/events/main/mod.rs index 9b543bf5fb..459f576db9 100644 --- a/crates/sqlbuild-state/src/events/main/mod.rs +++ b/crates/sqlbuild-state/src/events/main/mod.rs @@ -7,6 +7,7 @@ pub mod event_exists_sql; pub mod event_insert_sql; pub mod event_read_sql; pub mod grants_json; +pub mod microbatch_event_id; pub mod migration_event_id; pub mod newest_column_event; pub mod newest_fingerprint; diff --git a/crates/sqlbuild-state/src/freshness/constants.rs b/crates/sqlbuild-state/src/freshness/constants.rs index 19e375d039..8083997c83 100644 --- a/crates/sqlbuild-state/src/freshness/constants.rs +++ b/crates/sqlbuild-state/src/freshness/constants.rs @@ -28,7 +28,6 @@ pub(crate) const MARCH_BASED_MONTH_DAYS: i128 = 153; pub(crate) const MONTH_LENGTH_DIVISOR: i128 = 5; pub(crate) const MONTHS_FROM_MARCH: i128 = 9; pub(crate) const FEBRUARY: u32 = 2; -pub const SOURCE_FRESHNESS_TABLE_NAME: &str = "_sqlbuild_source_freshness"; /// Columns a source freshness state table must have, in Python's declaration order. pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ "source_name", @@ -38,7 +37,22 @@ pub const REQUIRED_STATE_COLUMNS: [&str; 6] = [ "data_version_hash", "observed_at", ]; -pub(crate) const STATE_SELECT_COLUMNS: &str = "source_name, target_database, target_schema, \ -target_name, run_id, strategy, value_kind, data_version, data_version_hash, observed_at"; +/// State value type of a text column. +pub const STRING_STATE_VALUE_TYPE: &str = "string"; +/// State value type of a timestamp stored as ISO text. +pub const TEXT_TIMESTAMP_STATE_VALUE_TYPE: &str = "text_timestamp"; +/// Source freshness state table columns in table order, with the state value type each declares. +pub const SOURCE_FRESHNESS_COLUMN_TYPES: [(&str, &str); 10] = [ + ("source_name", STRING_STATE_VALUE_TYPE), + ("target_database", STRING_STATE_VALUE_TYPE), + ("target_schema", STRING_STATE_VALUE_TYPE), + ("target_name", STRING_STATE_VALUE_TYPE), + ("run_id", STRING_STATE_VALUE_TYPE), + ("strategy", STRING_STATE_VALUE_TYPE), + ("value_kind", STRING_STATE_VALUE_TYPE), + ("data_version", STRING_STATE_VALUE_TYPE), + ("data_version_hash", STRING_STATE_VALUE_TYPE), + ("observed_at", TEXT_TIMESTAMP_STATE_VALUE_TYPE), +]; pub(crate) const PHYSICAL_TABLE_SOURCE_FRAGMENT: &str = "requires a physical table source"; pub(crate) const INCOMPLETE_CONFIGURATION_FRAGMENT: &str = "incomplete"; diff --git a/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs index 9dc677915d..443585bfdc 100644 --- a/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs +++ b/crates/sqlbuild-state/src/freshness/main/read_latest_sql.rs @@ -1,11 +1,16 @@ //! `read_latest_sql`: the windowed SELECT of the latest state row per source identity. -use crate::freshness::constants::STATE_SELECT_COLUMNS; +use crate::freshness::constants::SOURCE_FRESHNESS_COLUMN_TYPES; /// Latest row per source, target database, schema and name, newest observation and run first. pub fn read_latest_sql(qualified_name: &str) -> String { + let columns: String = SOURCE_FRESHNESS_COLUMN_TYPES + .iter() + .map(|(column, _)| *column) + .collect::>() + .join(", "); format!( - "SELECT {STATE_SELECT_COLUMNS} FROM (SELECT {STATE_SELECT_COLUMNS}, ROW_NUMBER() OVER (\ + "SELECT {columns} FROM (SELECT {columns}, ROW_NUMBER() OVER (\ PARTITION BY source_name, target_database, target_schema, target_name \ ORDER BY observed_at DESC, run_id DESC) AS __sqlbuild_latest_rank FROM {qualified_name}) \ AS __sqlbuild_latest_source_freshness WHERE __sqlbuild_latest_rank = 1" diff --git a/crates/sqlbuild-state/src/lib.rs b/crates/sqlbuild-state/src/lib.rs index 3e6c8463ae..571e27e44c 100644 --- a/crates/sqlbuild-state/src/lib.rs +++ b/crates/sqlbuild-state/src/lib.rs @@ -5,3 +5,5 @@ pub mod events; pub mod execution_history; pub mod freshness; +pub mod microbatches; +pub mod state_tables; diff --git a/crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs b/crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs new file mode 100644 index 0000000000..82e057fa81 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/_helpers/coverage.rs @@ -0,0 +1,146 @@ +//! The boundary sweep and gap classification behind microbatch coverage projection. + +use std::cmp::Ordering; +use std::collections::{BinaryHeap, HashSet}; + +use crate::microbatches::_helpers::ordering::{key_cmp, lt, lte, sorted_by_bound}; +use crate::microbatches::models::{BoundInterval, CompletionFact, CursorBound, TextInterval}; + +/// One swept segment between adjacent boundaries and its latest active completion. +pub(crate) struct Segment { + pub(crate) start: CursorBound, + pub(crate) end: CursorBound, + pub(crate) completion: usize, +} + +fn text_interval(start: &CursorBound, end: &CursorBound) -> TextInterval { + TextInterval { + start: start.text.clone(), + end: end.text.clone(), + } +} + +/// Every distinct bound text, sorted by key. +pub(crate) fn ordered_boundaries( + completions: &[CompletionFact], + expected: &[BoundInterval], +) -> Vec { + let mut boundaries: Vec = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + let intervals = completions + .iter() + .map(|completion| &completion.interval) + .chain(expected); + for interval in intervals { + for bound in [&interval.start, &interval.end] { + if seen.insert(bound.text.as_str()) { + boundaries.push(bound.clone()); + } + } + } + boundaries.sort_by(|left, right| { + key_cmp(&left.key, &right.key).then_with(|| left.text.cmp(&right.text)) + }); + boundaries +} + +/// Sweep adjacent boundaries, keeping the most recently created active completion. +pub(crate) fn latest_completion_segments( + completions: &[CompletionFact], + boundaries: &[CursorBound], +) -> Vec { + let mut latest_order: Vec = (0..completions.len()).collect(); + latest_order.sort_by(|left, right| { + let (left, right) = (&completions[*left], &completions[*right]); + left.created + .partial_cmp(&right.created) + .unwrap_or(Ordering::Equal) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + let mut rank: Vec = vec![0; completions.len()]; + for (position, index) in latest_order.into_iter().enumerate() { + rank[index] = position; + } + let mut by_start: Vec = (0..completions.len()).collect(); + by_start.sort_by(|left, right| { + key_cmp( + &completions[*left].interval.start.key, + &completions[*right].interval.start.key, + ) + }); + let mut active: BinaryHeap<(usize, usize)> = BinaryHeap::new(); + let mut next: usize = 0; + let mut segments: Vec = Vec::new(); + for pair in boundaries.windows(2) { + let (start, end) = (&pair[0], &pair[1]); + while next < by_start.len() && lte(&completions[by_start[next]].interval.start, start) { + active.push((rank[by_start[next]], by_start[next])); + next += 1; + } + while let Some((_, top)) = active.peek() { + if lte(end, &completions[*top].interval.end) { + break; + } + active.pop(); + } + if let Some((_, top)) = active.peek() { + segments.push(Segment { + start: start.clone(), + end: end.clone(), + completion: *top, + }); + } + } + segments +} + +/// The uncovered parts of each expected interval, split into known-missing and unaccounted. +pub(crate) fn classify_gaps( + expected: &[BoundInterval], + accounted: &[BoundInterval], + latest_ordinary_end: Option<&CursorBound>, +) -> (Vec, Vec) { + let ordered_expected: Vec = sorted_by_bound(expected, |item| &item.start); + let ordered_accounted: Vec = sorted_by_bound(accounted, |item| &item.start); + let mut known_missing: Vec = Vec::new(); + let mut unaccounted: Vec = Vec::new(); + let mut accounted_index: usize = 0; + for expected in &ordered_expected { + while accounted_index < ordered_accounted.len() + && lte(&ordered_accounted[accounted_index].end, &expected.start) + { + accounted_index += 1; + } + let mut frontier: &CursorBound = &expected.start; + let mut gaps: Vec<(&CursorBound, &CursorBound)> = Vec::new(); + for interval in &ordered_accounted[accounted_index..] { + if !lt(&interval.start, &expected.end) { + break; + } + if lt(frontier, &interval.start) { + gaps.push((frontier, &interval.start)); + } + if lt(frontier, &interval.end) { + frontier = &interval.end; + } + if lte(&expected.end, frontier) { + break; + } + } + if lt(frontier, &expected.end) { + gaps.push((frontier, &expected.end)); + } + for (start, end) in gaps { + let accounted_before: bool = accounted_index > 0 || lt(&expected.start, start); + let known: bool = + accounted_before && latest_ordinary_end.is_some_and(|latest| lte(end, latest)); + let gap: TextInterval = text_interval(start, end); + if known { + known_missing.push(gap); + } else { + unaccounted.push(gap); + } + } + } + (known_missing, unaccounted) +} diff --git a/crates/sqlbuild-state/src/microbatches/_helpers/mod.rs b/crates/sqlbuild-state/src/microbatches/_helpers/mod.rs new file mode 100644 index 0000000000..262e03e5b4 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod coverage; +pub(crate) mod ordering; diff --git a/crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs b/crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs new file mode 100644 index 0000000000..1bc1a17c6e --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/_helpers/ordering.rs @@ -0,0 +1,47 @@ +//! Cursor bound comparisons and stable sorts by bound key. + +use std::cmp::Ordering; + +use sqlbuild_cursor::algebra::models::SortKey; + +use crate::microbatches::models::CursorBound; + +/// How two keys of one cursor type order; integers and moments never meet in one history. +pub(crate) fn key_cmp(left: &SortKey, right: &SortKey) -> Ordering { + match (left, right) { + (SortKey::Integer(left), SortKey::Integer(right)) => left.cmp(right), + (SortKey::Moment(left), SortKey::Moment(right)) => left.cmp(right), + (SortKey::Integer(_), SortKey::Moment(_)) => Ordering::Less, + (SortKey::Moment(_), SortKey::Integer(_)) => Ordering::Greater, + } +} + +pub(crate) fn lt(left: &CursorBound, right: &CursorBound) -> bool { + key_cmp(&left.key, &right.key) == Ordering::Less +} + +pub(crate) fn lte(left: &CursorBound, right: &CursorBound) -> bool { + key_cmp(&left.key, &right.key) != Ordering::Greater +} + +/// `items` stably sorted by the key `bound` selects. +pub(crate) fn sorted_by_bound(items: &[T], bound: impl Fn(&T) -> &CursorBound) -> Vec { + let mut sorted: Vec = items.to_vec(); + sorted.sort_by(|left, right| key_cmp(&bound(left).key, &bound(right).key)); + sorted +} + +/// The first non-empty bound holding the smallest (`wanted = Less`) or largest key. +pub(crate) fn extreme<'a>( + bounds: impl Iterator, + wanted: Ordering, +) -> Option<&'a CursorBound> { + let mut best: Option<&CursorBound> = None; + for bound in bounds.filter(|bound| !bound.text.is_empty()) { + match best { + Some(current) if key_cmp(&bound.key, ¤t.key) != wanted => {} + _ => best = Some(bound), + } + } + best +} diff --git a/crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs b/crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs new file mode 100644 index 0000000000..dc96267040 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/create_index_sqls.rs @@ -0,0 +1,15 @@ +//! `create_index_sqls`: the non-unique scope and provenance indexes of the state table. + +/// One `CREATE INDEX IF NOT EXISTS` per rendered index name and its rendered columns. +#[must_use] +pub fn create_index_sqls(table: &str, indexes: &[(String, Vec)]) -> Vec { + indexes + .iter() + .map(|(name, columns)| { + format!( + "CREATE INDEX IF NOT EXISTS {name} ON {table} ({})", + columns.join(", ") + ) + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs b/crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs new file mode 100644 index 0000000000..3306463c69 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/existing_event_ids_sql.rs @@ -0,0 +1,10 @@ +//! `existing_event_ids_sql`: the lookup of event IDs already stored before a bulk insert. + +/// `SELECT event_id FROM WHERE event_id IN ()`. +#[must_use] +pub fn existing_event_ids_sql(table: &str, event_id_literals: &[String]) -> String { + format!( + "SELECT event_id FROM {table} WHERE event_id IN ({})", + event_id_literals.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs b/crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs new file mode 100644 index 0000000000..044afc896c --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/insert_many_sql.rs @@ -0,0 +1,24 @@ +//! `insert_many_sql`: one guarded set-based insert of events into one state table. + +/// Rows selected under the first row's column aliases, inserted where the event ID is new. +#[must_use] +pub fn insert_many_sql(table: &str, columns: &[String], rows: &[Vec]) -> String { + let mut selections: Vec = Vec::with_capacity(rows.len()); + for (index, row) in rows.iter().enumerate() { + if index > 0 { + selections.push(format!("SELECT {}", row.join(", "))); + continue; + } + let mut aliased: Vec = Vec::with_capacity(row.len()); + for (literal, column) in row.iter().zip(columns) { + aliased.push(format!("{literal} AS {column}")); + } + selections.push(format!("SELECT {}", aliased.join(", "))); + } + let columns: String = columns.join(", "); + format!( + "INSERT INTO {table} ({columns}) SELECT {columns} FROM ({}) AS incoming \ +WHERE NOT EXISTS (SELECT 1 FROM {table} AS existing WHERE existing.event_id = incoming.event_id)", + selections.join(" UNION ALL ") + ) +} diff --git a/crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs b/crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs new file mode 100644 index 0000000000..f64d75c6a2 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/latest_requirement.rs @@ -0,0 +1,29 @@ +//! `latest_requirement`: the newest replay requirement created for the compiled version. + +use std::cmp::Ordering; + +use crate::microbatches::models::RequirementFact; + +/// The index of the first event ordering last by creation time then event ID, if any. +#[must_use] +pub fn latest_requirement(events: &[RequirementFact], current: &str) -> Option { + let mut best: Option = None; + for (index, event) in events.iter().enumerate() { + if !event.is_requirement || event.required_model_version_hash.as_deref() != Some(current) { + continue; + } + let newer: bool = best.is_none_or(|best| { + let held: &RequirementFact = &events[best]; + event + .created + .partial_cmp(&held.created) + .unwrap_or(Ordering::Equal) + .then_with(|| event.event_id.cmp(&held.event_id)) + == Ordering::Greater + }); + if newer { + best = Some(index); + } + } + best +} diff --git a/crates/sqlbuild-state/src/microbatches/main/mod.rs b/crates/sqlbuild-state/src/microbatches/main/mod.rs new file mode 100644 index 0000000000..bfe1dfee3e --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/mod.rs @@ -0,0 +1,7 @@ +pub mod create_index_sqls; +pub mod existing_event_ids_sql; +pub mod insert_many_sql; +pub mod latest_requirement; +pub mod project_coverage; +pub mod project_replay; +pub mod read_scope_sql; diff --git a/crates/sqlbuild-state/src/microbatches/main/project_coverage.rs b/crates/sqlbuild-state/src/microbatches/main/project_coverage.rs new file mode 100644 index 0000000000..5c33f44b8b --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/project_coverage.rs @@ -0,0 +1,73 @@ +//! `project_coverage`: fold completion facts and classify uncovered expected intervals. + +use std::cmp::Ordering; + +use crate::microbatches::_helpers::coverage::{ + Segment, classify_gaps, latest_completion_segments, ordered_boundaries, +}; +use crate::microbatches::_helpers::ordering::{extreme, lt, lte, sorted_by_bound}; +use crate::microbatches::models::{ + BoundInterval, CompletionFact, CoverageProjection, CoverageSegment, CursorBound, TextInterval, +}; + +/// Latest-completion segments, the contiguous frontier and the known-missing and unaccounted gaps. +#[must_use] +pub fn project_coverage( + completions: &[CompletionFact], + expected: &[BoundInterval], +) -> CoverageProjection { + let boundaries: Vec = ordered_boundaries(completions, expected); + let segments: Vec = latest_completion_segments(completions, &boundaries); + let accounted: Vec = segments + .iter() + .map(|segment| BoundInterval { + start: segment.start.clone(), + end: segment.end.clone(), + }) + .collect(); + let mut frontier: Option<&CursorBound> = extreme( + expected.iter().map(|interval| &interval.start), + Ordering::Less, + ); + let ordered_accounted: Vec = sorted_by_bound(&accounted, |item| &item.start); + if let Some(mut current) = frontier.filter(|_| !accounted.is_empty()) { + for interval in &ordered_accounted { + if !lte(&interval.start, current) { + break; + } + if lt(current, &interval.end) { + current = &interval.end; + } + } + frontier = Some(current); + } + let latest_ordinary_end: Option<&CursorBound> = extreme( + completions + .iter() + .filter(|completion| completion.ordinary) + .map(|completion| &completion.interval.end), + Ordering::Greater, + ); + let (known_missing, unaccounted) = classify_gaps(expected, &accounted, latest_ordinary_end); + let text = |segment: &Segment| TextInterval { + start: segment.start.text.clone(), + end: segment.end.text.clone(), + }; + CoverageProjection { + unknown_fingerprints: segments + .iter() + .filter(|segment| completions[segment.completion].unknown_fingerprint) + .map(text) + .collect(), + contiguous_frontier: frontier.map(|bound| bound.text.clone()), + known_missing, + unaccounted, + segments: segments + .iter() + .map(|segment| CoverageSegment { + interval: text(segment), + completion: segment.completion, + }) + .collect(), + } +} diff --git a/crates/sqlbuild-state/src/microbatches/main/project_replay.rs b/crates/sqlbuild-state/src/microbatches/main/project_replay.rs new file mode 100644 index 0000000000..8f8dc559ca --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/project_replay.rs @@ -0,0 +1,71 @@ +//! `project_replay`: derive a replay requirement's completion without mutable status. + +use crate::microbatches::_helpers::ordering::{lt, lte, sorted_by_bound}; +use crate::microbatches::models::{ + BoundInterval, CoveragePiece, CursorBound, ReplayProjection, ReplayState, ReplayVersions, +}; + +fn fully_covered(expected: &BoundInterval, pieces: &[&CoveragePiece]) -> bool { + let mut frontier: &CursorBound = &expected.start; + for piece in sorted_by_bound(pieces, |piece| &piece.interval.start) { + if lte(&piece.interval.end, frontier) { + continue; + } + if lt(frontier, &piece.interval.start) { + return false; + } + frontier = &piece.interval.end; + if lte(&expected.end, frontier) { + return true; + } + } + false +} + +/// Superseded for another version; otherwise which expected intervals are missing or unverified. +#[must_use] +pub fn project_replay( + versions: ReplayVersions<'_>, + expected: &[BoundInterval], + coverage: &[CoveragePiece], +) -> ReplayProjection { + if versions.required != Some(versions.current) { + return ReplayProjection { + state: ReplayState::Superseded, + missing: Vec::new(), + unknown_fingerprints: Vec::new(), + }; + } + let mut missing: Vec = Vec::new(); + let mut unknown: Vec = Vec::new(); + for (index, interval) in expected.iter().enumerate() { + let pieces: Vec<&CoveragePiece> = coverage + .iter() + .filter(|piece| { + lt(&piece.interval.start, &interval.end) && lt(&interval.start, &piece.interval.end) + }) + .collect(); + if !fully_covered(interval, &pieces) { + missing.push(index); + } else if pieces.iter().any(|piece| piece.unknown_fingerprint) { + unknown.push(index); + } else if pieces + .iter() + .any(|piece| piece.model_version_hash.as_deref() != Some(versions.current)) + { + missing.push(index); + } + } + let state: ReplayState = if !missing.is_empty() { + ReplayState::Incomplete + } else if !unknown.is_empty() { + ReplayState::CompleteWithUnknownFingerprints + } else { + ReplayState::VerifiedComplete + }; + ReplayProjection { + state, + missing, + unknown_fingerprints: unknown, + } +} diff --git a/crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs b/crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs new file mode 100644 index 0000000000..2b27e75d96 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/main/read_scope_sql.rs @@ -0,0 +1,19 @@ +//! `read_scope_sql`: the ordered read of one scope's stored events. + +use crate::microbatches::models::ScopeRead; + +/// Every column of one scope's events, optionally one generation, by creation then event ID. +#[must_use] +pub fn read_scope_sql(table: &str, read: &ScopeRead<'_>) -> String { + let generation: String = read + .generation + .map(|literal| format!("AND physical_generation_id = {literal} ")) + .unwrap_or_default(); + format!( + "SELECT {} FROM {table} WHERE scope_kind = {} AND scope_key = {} {generation}\ +ORDER BY created_at, event_id", + read.columns.join(", "), + read.scope_kind, + read.scope_key + ) +} diff --git a/crates/sqlbuild-state/src/microbatches/mod.rs b/crates/sqlbuild-state/src/microbatches/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/microbatches/models.rs b/crates/sqlbuild-state/src/microbatches/models.rs new file mode 100644 index 0000000000..dad4315763 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/models.rs @@ -0,0 +1,121 @@ +//! Append-only microbatch facts, cursor bounds and the coverage derived from them. + +use sqlbuild_cursor::algebra::models::SortKey; + +/// One raw cursor bound and the key it sorts and compares by. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CursorBound { + pub text: String, + pub key: SortKey, +} + +/// A half-open interval between two cursor bounds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BoundInterval { + pub start: CursorBound, + pub end: CursorBound, +} + +/// A half-open interval as its raw bound texts. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TextInterval { + pub start: String, + pub end: String, +} + +/// One partition or synthetic completion that names both partition bounds. +#[derive(Clone, Debug, PartialEq)] +pub struct CompletionFact { + pub interval: BoundInterval, + /// Whether this is an ordinary partition completion rather than a synthetic one. + pub ordinary: bool, + pub unknown_fingerprint: bool, + /// `created_at.timestamp()`. + pub created: f64, + pub event_id: String, +} + +/// One interval and the index of the latest completion accounting for it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CoverageSegment { + pub interval: TextInterval, + pub completion: usize, +} + +/// Physical and fingerprint coverage derived from completion facts. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CoverageProjection { + pub segments: Vec, + pub contiguous_frontier: Option, + pub known_missing: Vec, + pub unaccounted: Vec, + pub unknown_fingerprints: Vec, +} + +/// One projected coverage interval as a replay requirement reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CoveragePiece { + pub interval: BoundInterval, + pub unknown_fingerprint: bool, + pub model_version_hash: Option, +} + +/// The versions one replay requirement is checked against. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReplayVersions<'a> { + pub required: Option<&'a str>, + pub current: &'a str, +} + +/// The derived state of one replay requirement. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReplayState { + Incomplete, + VerifiedComplete, + CompleteWithUnknownFingerprints, + Superseded, +} + +/// A replay requirement's state and the expected intervals still missing or unverified. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayProjection { + pub state: ReplayState, + /// Indexes into the expected intervals. + pub missing: Vec, + pub unknown_fingerprints: Vec, +} + +/// One replay requirement event as the latest-requirement selection reads it. +#[derive(Clone, Debug, PartialEq)] +pub struct RequirementFact { + pub is_requirement: bool, + pub required_model_version_hash: Option, + pub created: f64, + pub event_id: String, +} + +/// The fields of a microbatch event's content-addressed identity, in identity order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MicrobatchEventIdentity { + pub scope_kind: String, + pub scope_key: String, + pub model_name: String, + pub target_database: Option, + pub target_schema: Option, + pub target_name: String, + pub physical_generation_id: String, + pub record_type: String, + pub partition_start: Option, + pub partition_end: Option, + pub completion_reason: String, +} + +/// A microbatch state table read: its selected columns and scope literals. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeRead<'a> { + pub columns: &'a [String], + pub scope_kind: &'a str, + pub scope_key: &'a str, + /// The generation literal, or `None` for every generation. + pub generation: Option<&'a str>, +} diff --git a/crates/sqlbuild-state/src/microbatches/tests.rs b/crates/sqlbuild-state/src/microbatches/tests.rs new file mode 100644 index 0000000000..e9d7907f38 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_projection.rs"] +mod projection; +#[path = "tests/test_statements.rs"] +mod statements; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/microbatches/tests/helpers.rs b/crates/sqlbuild-state/src/microbatches/tests/helpers.rs new file mode 100644 index 0000000000..0f4d610246 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/helpers.rs @@ -0,0 +1,69 @@ +use sqlbuild_cursor::algebra::models::SortKey; + +use crate::microbatches::models::{ + BoundInterval, CompletionFact, CursorBound, MicrobatchEventIdentity, TextInterval, +}; + +/// An integer cursor bound. +pub(crate) fn bound(value: i128) -> CursorBound { + CursorBound { + text: value.to_string(), + key: SortKey::Integer(value), + } +} + +/// An integer interval `[start, end)`. +pub(crate) fn interval(start: i128, end: i128) -> BoundInterval { + BoundInterval { + start: bound(start), + end: bound(end), + } +} + +/// A known-fingerprint ordinary completion created `created` seconds in. +pub(crate) fn completion( + event_id: &str, + (start, end): (i128, i128), + created: f64, +) -> CompletionFact { + CompletionFact { + interval: interval(start, end), + ordinary: true, + unknown_fingerprint: false, + created, + event_id: event_id.to_owned(), + } +} + +/// The identity of an ordinary completion of `orders` over `[0, 1)`. +pub(crate) fn completion_identity() -> MicrobatchEventIdentity { + MicrobatchEventIdentity { + scope_kind: "direct_logical".to_owned(), + scope_key: "duckdb:main.orders".to_owned(), + model_name: "orders".to_owned(), + target_database: None, + target_schema: Some("main".to_owned()), + target_name: "orders".to_owned(), + physical_generation_id: "gen-1".to_owned(), + record_type: "partition_completion".to_owned(), + partition_start: Some("0".to_owned()), + partition_end: Some("1".to_owned()), + completion_reason: "initial".to_owned(), + } +} + +/// Interval texts as owned pairs. +pub(crate) fn texts(intervals: &[TextInterval]) -> Vec<(String, String)> { + intervals + .iter() + .map(|interval| (interval.start.clone(), interval.end.clone())) + .collect() +} + +/// Borrowed pairs as owned pairs. +pub(crate) fn owned(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(start, end)| ((*start).to_owned(), (*end).to_owned())) + .collect() +} diff --git a/crates/sqlbuild-state/src/microbatches/tests/test_projection.rs b/crates/sqlbuild-state/src/microbatches/tests/test_projection.rs new file mode 100644 index 0000000000..dcd9767232 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/test_projection.rs @@ -0,0 +1,133 @@ +use crate::microbatches::main::project_coverage::project_coverage; +use crate::microbatches::main::project_replay::project_replay; +use crate::microbatches::models::{CoveragePiece, ReplayState, ReplayVersions}; +use crate::microbatches::tests::helpers::{completion, interval, owned, texts}; +use crate::microbatches::tests::test_types::{CoverageTestCase, ReplayTestCase}; + +#[test] +fn given_completion_history_when_projecting_then_latest_completion_and_gaps_are_derived() { + let test_cases = [ + CoverageTestCase { + description: "newer overlapping completion wins and a middle gap is known missing", + completions: vec![ + completion("a", (0, 2), 1.0), + completion("b", (1, 2), 2.0), + completion("c", (3, 4), 3.0), + ], + expected: vec![ + interval(0, 1), + interval(1, 2), + interval(2, 3), + interval(3, 4), + ], + expected_segments: vec![("0", "1", 0), ("1", "2", 1), ("3", "4", 2)], + expected_frontier: Some("2"), + expected_known_missing: vec![("2", "3")], + expected_unaccounted: Vec::new(), + }, + CoverageTestCase { + description: "no history leaves every expected interval unaccounted", + completions: Vec::new(), + expected: vec![interval(5, 6)], + expected_segments: Vec::new(), + expected_frontier: Some("5"), + expected_known_missing: Vec::new(), + expected_unaccounted: vec![("5", "6")], + }, + ]; + + for test_case in test_cases { + let projection = project_coverage(&test_case.completions, &test_case.expected); + let segments: Vec<(String, String, usize)> = projection + .segments + .iter() + .map(|segment| { + ( + segment.interval.start.clone(), + segment.interval.end.clone(), + segment.completion, + ) + }) + .collect(); + let expected_segments: Vec<(String, String, usize)> = test_case + .expected_segments + .iter() + .map(|(start, end, index)| ((*start).to_owned(), (*end).to_owned(), *index)) + .collect(); + assert_eq!(segments, expected_segments, "{}", test_case.description); + assert_eq!( + projection.contiguous_frontier.as_deref(), + test_case.expected_frontier, + "{}", + test_case.description + ); + assert_eq!( + texts(&projection.known_missing), + owned(&test_case.expected_known_missing), + "{}", + test_case.description + ); + assert_eq!( + texts(&projection.unaccounted), + owned(&test_case.expected_unaccounted), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_replay_requirement_when_projecting_then_state_follows_coverage_versions() { + let piece = |start: i128, end: i128, version: &str, unknown: bool| CoveragePiece { + interval: interval(start, end), + unknown_fingerprint: unknown, + model_version_hash: Some(version.to_owned()), + }; + let test_cases = [ + ReplayTestCase { + description: "other version is superseded", + required: Some("F1"), + coverage: Vec::new(), + expected_state: ReplayState::Superseded, + expected_missing: Vec::new(), + }, + ReplayTestCase { + description: "current version everywhere is verified", + required: Some("F2"), + coverage: vec![piece(0, 2, "F2", false)], + expected_state: ReplayState::VerifiedComplete, + expected_missing: Vec::new(), + }, + ReplayTestCase { + description: "old version piece is missing", + required: Some("F2"), + coverage: vec![piece(0, 1, "F2", false), piece(1, 2, "F1", false)], + expected_state: ReplayState::Incomplete, + expected_missing: vec![1], + }, + ReplayTestCase { + description: "unknown fingerprint completes with unknown", + required: Some("F2"), + coverage: vec![piece(0, 2, "F2", true)], + expected_state: ReplayState::CompleteWithUnknownFingerprints, + expected_missing: Vec::new(), + }, + ]; + + for test_case in test_cases { + let projection = project_replay( + ReplayVersions { + required: test_case.required, + current: "F2", + }, + &[interval(0, 1), interval(1, 2)], + &test_case.coverage, + ); + assert_eq!( + (projection.state, projection.missing), + (test_case.expected_state, test_case.expected_missing), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/microbatches/tests/test_statements.rs b/crates/sqlbuild-state/src/microbatches/tests/test_statements.rs new file mode 100644 index 0000000000..af30110414 --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/test_statements.rs @@ -0,0 +1,47 @@ +use crate::events::main::microbatch_event_id::microbatch_event_id; +use crate::microbatches::main::insert_many_sql::insert_many_sql; +use crate::microbatches::tests::helpers::completion_identity; +use crate::microbatches::tests::test_types::{EventIdTestCase, InsertManyTestCase}; + +#[test] +fn given_event_identity_when_hashing_then_matches_python_json_digest() { + let test_cases = [EventIdTestCase { + description: "ordinary completion", + identity: completion_identity(), + expected_id: "86cf4c901147bf7e0bde6673b0725aed47d013211f9ccef8bb84f77fca1454ae", + }]; + + for test_case in test_cases { + assert_eq!( + microbatch_event_id(&test_case.identity).ok().as_deref(), + Some(test_case.expected_id), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_rendering_bulk_insert_then_first_row_carries_aliases() { + let test_cases = [InsertManyTestCase { + description: "two rows", + rows: vec![ + vec!["'a'".to_owned(), "1".to_owned()], + vec!["'b'".to_owned(), "2".to_owned()], + ], + expected_sql: "INSERT INTO s.t (event_id, n) SELECT event_id, n FROM (SELECT 'a' AS event_id, 1 AS n UNION ALL SELECT 'b', 2) AS incoming WHERE NOT EXISTS (SELECT 1 FROM s.t AS existing WHERE existing.event_id = incoming.event_id)", + }]; + + for test_case in test_cases { + assert_eq!( + insert_many_sql( + "s.t", + &["event_id".to_owned(), "n".to_owned()], + &test_case.rows + ), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/microbatches/tests/test_types.rs b/crates/sqlbuild-state/src/microbatches/tests/test_types.rs new file mode 100644 index 0000000000..10d82e23da --- /dev/null +++ b/crates/sqlbuild-state/src/microbatches/tests/test_types.rs @@ -0,0 +1,33 @@ +use crate::microbatches::models::{ + BoundInterval, CompletionFact, CoveragePiece, MicrobatchEventIdentity, ReplayState, +}; + +pub(crate) struct CoverageTestCase { + pub(crate) description: &'static str, + pub(crate) completions: Vec, + pub(crate) expected: Vec, + pub(crate) expected_segments: Vec<(&'static str, &'static str, usize)>, + pub(crate) expected_frontier: Option<&'static str>, + pub(crate) expected_known_missing: Vec<(&'static str, &'static str)>, + pub(crate) expected_unaccounted: Vec<(&'static str, &'static str)>, +} + +pub(crate) struct ReplayTestCase { + pub(crate) description: &'static str, + pub(crate) required: Option<&'static str>, + pub(crate) coverage: Vec, + pub(crate) expected_state: ReplayState, + pub(crate) expected_missing: Vec, +} + +pub(crate) struct EventIdTestCase { + pub(crate) description: &'static str, + pub(crate) identity: MicrobatchEventIdentity, + pub(crate) expected_id: &'static str, +} + +pub(crate) struct InsertManyTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec>, + pub(crate) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs b/crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs new file mode 100644 index 0000000000..e164cf9037 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/create_table_sql.rs @@ -0,0 +1,18 @@ +//! `create_table_sql`: create-if-missing DDL for one append-only state table. + +use crate::state_tables::models::TableColumn; + +/// `CREATE TABLE IF NOT EXISTS
( [ NOT NULL], ...)`. +pub fn create_table_sql(qualified_name: &str, columns: &[TableColumn]) -> String { + let definitions: Vec = columns + .iter() + .map(|column| { + let required: &str = if column.required { " NOT NULL" } else { "" }; + format!("{} {}{required}", column.name, column.rendered_type) + }) + .collect(); + format!( + "CREATE TABLE IF NOT EXISTS {qualified_name} ({})", + definitions.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/state_tables/main/mod.rs b/crates/sqlbuild-state/src/state_tables/main/mod.rs new file mode 100644 index 0000000000..712abe33fc --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/mod.rs @@ -0,0 +1,2 @@ +pub mod create_table_sql; +pub mod values_insert_sql; diff --git a/crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs b/crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs new file mode 100644 index 0000000000..ce9212da9f --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/main/values_insert_sql.rs @@ -0,0 +1,14 @@ +//! `values_insert_sql`: rows of rendered literals appended with one `INSERT ... VALUES`. + +/// `INSERT INTO
() VALUES (), ...`. +pub fn values_insert_sql(qualified_name: &str, columns: &[&str], rows: &[Vec]) -> String { + let values: Vec = rows + .iter() + .map(|row| format!("({})", row.join(", "))) + .collect(); + format!( + "INSERT INTO {qualified_name} ({}) VALUES {}", + columns.join(", "), + values.join(", ") + ) +} diff --git a/crates/sqlbuild-state/src/state_tables/mod.rs b/crates/sqlbuild-state/src/state_tables/mod.rs new file mode 100644 index 0000000000..1a70a48981 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/mod.rs @@ -0,0 +1,5 @@ +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-state/src/state_tables/models.rs b/crates/sqlbuild-state/src/state_tables/models.rs new file mode 100644 index 0000000000..7a343e134d --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/models.rs @@ -0,0 +1,9 @@ +//! Columns of append-only state tables as their DDL declares them. + +/// One declared column: its name, the adapter's rendered type and whether it is `NOT NULL`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TableColumn { + pub name: String, + pub rendered_type: String, + pub required: bool, +} diff --git a/crates/sqlbuild-state/src/state_tables/tests.rs b/crates/sqlbuild-state/src/state_tables/tests.rs new file mode 100644 index 0000000000..4af576b46f --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_sql.rs"] +mod sql; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-state/src/state_tables/tests/helpers.rs b/crates/sqlbuild-state/src/state_tables/tests/helpers.rs new file mode 100644 index 0000000000..e69081ecf1 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::state_tables::models::TableColumn; + +/// One declared column of a test table. +pub(crate) fn column(name: &str, rendered_type: &str, required: bool) -> TableColumn { + TableColumn { + name: name.to_owned(), + rendered_type: rendered_type.to_owned(), + required, + } +} diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs new file mode 100644 index 0000000000..5c9ef41669 --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests/test_sql.rs @@ -0,0 +1,50 @@ +use crate::state_tables::main::create_table_sql::create_table_sql; +use crate::state_tables::main::values_insert_sql::values_insert_sql; +use crate::state_tables::tests::helpers::column; +use crate::state_tables::tests::test_types::{CreateTableTestCase, ValuesInsertTestCase}; + +#[test] +fn given_columns_when_rendering_ddl_then_required_columns_are_not_null() { + let test_cases = [ + CreateTableTestCase { + description: "mixed required columns", + columns: vec![column("a", "VARCHAR", true), column("b", "BIGINT", false)], + expected_sql: "CREATE TABLE IF NOT EXISTS s.t (a VARCHAR NOT NULL, b BIGINT)", + }, + CreateTableTestCase { + description: "no columns", + columns: Vec::new(), + expected_sql: "CREATE TABLE IF NOT EXISTS s.t ()", + }, + ]; + + for test_case in test_cases { + assert_eq!( + create_table_sql("s.t", &test_case.columns), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_rendering_insert_then_each_row_is_one_values_tuple() { + let test_cases = [ValuesInsertTestCase { + description: "two rows", + rows: vec![ + vec!["'a'".to_owned(), "NULL".to_owned()], + vec!["'b'".to_owned(), "'c'".to_owned()], + ], + expected_sql: "INSERT INTO s.t (x, y) VALUES ('a', NULL), ('b', 'c')", + }]; + + for test_case in test_cases { + assert_eq!( + values_insert_sql("s.t", &["x", "y"], &test_case.rows), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/state_tables/tests/test_types.rs b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs new file mode 100644 index 0000000000..836ccf548d --- /dev/null +++ b/crates/sqlbuild-state/src/state_tables/tests/test_types.rs @@ -0,0 +1,13 @@ +use crate::state_tables::models::TableColumn; + +pub(crate) struct CreateTableTestCase { + pub(crate) description: &'static str, + pub(crate) columns: Vec, + pub(crate) expected_sql: &'static str, +} + +pub(crate) struct ValuesInsertTestCase { + pub(crate) description: &'static str, + pub(crate) rows: Vec>, + pub(crate) expected_sql: &'static str, +} diff --git a/fensu.toml b/fensu.toml index 141966664d..db6188ab05 100644 --- a/fensu.toml +++ b/fensu.toml @@ -23,6 +23,11 @@ exclude = ["tests/e2e/fixtures/**"] [targets.python.thresholds] max_positional_args = 1 +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cost/models.py" +reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models and types they use." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/lint/__init__.py" @@ -31,7 +36,12 @@ reason = "Native bindings own lint entry points, expansion preparation and compi [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" -reason = "Native bindings own the compile reuse check, replay and launch records; this package declares the reuse models and constants and keeps the provider settings, runtime identity and startup inputs the native code reads from the interpreter." +reason = "Native bindings own the compile reuse check, runtime identity, replay and launch records; this package declares the reuse models and keeps the provider settings and startup inputs the native code reads from the interpreter." + +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/microbatches/__init__.py" +reason = "Native bindings own microbatch coverage and replay projection, event IDs and state SQL; this package keeps the executor's direct event store, the event row codec and the event, scope and projection models it consumes." [[targets.python.rule_exceptions]] rule = "FFR309" @@ -63,21 +73,6 @@ rule = "FFR309" path = "src/sqlbuild/compiler/migrations/__init__.py" reason = "Native bindings own migration planning and state rows; this package only declares the migration models, decisions and table names the executor imports." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/source_freshness/__init__.py" -reason = "Native bindings own source freshness observation, planning, propagation and state reads and writes; this package only declares the models, types and constants they build." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/cost/constants.py" -reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/contract_adoption/__init__.py" -reason = "Native bindings own contract comparison, edits and repository writes; this package only declares the adoption models, types and errors the native command builds and raises." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/pipeline/__init__.py" @@ -143,11 +138,6 @@ rule = "FFR309" path = "src/sqlbuild/adapters/sqlserver/__init__.py" reason = "The SQL Server backend intentionally owns behavior through its adapter class rather than function entrypoints." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/contracts/__init__.py" -reason = "Model contracts are evaluated natively; the package keeps only the validation result model its callers read." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/fingerprints/__init__.py" @@ -163,51 +153,16 @@ rule = "FFR309" path = "src/sqlbuild/compiler/references/__init__.py" reason = "SQL references are scanned natively; the package keeps the reference models, kinds and the reference-call patterns the Python planner still matches." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/resource_names/__init__.py" -reason = "Resource identities are validated natively; the package keeps the error class the native check raises." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/refactoring/__init__.py" -reason = "Refactorings run natively; the package keeps the compiled-project input, manual-location model and error classes the native command reads and raises." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/scopes/__init__.py" -reason = "Declaration scopes are built, queried and validated natively; the package keeps the index, lookup and report models and the errors the native bindings raise." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/sql_test_glue/__init__.py" -reason = "SQL test assembly and planning run natively; the package keeps the request, result and row shapes the planner and compile stage read." - -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/graph/__init__.py" -reason = "Graph algorithms and views run natively; the package keeps only the edge and view models their callers read." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/sql_values/exceptions.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types and the rendering error adapters raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/cli/output/__init__.py" -reason = "Execution-result output is rendered natively; the package keeps only the integration-result constants its callers and readers share." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/lineage/__init__.py" reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/analysis_session/__init__.py" -reason = "Model analysis runs natively; the package keeps only the request and result models, row types and constants the native session and its callers share." - [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" @@ -351,6 +306,16 @@ rule = "FFR403" path = "src/sqlbuild/rules/testing.py" reason = "This deliberate public shim keeps test-harness internals outside the stable authoring API." +[[targets.python.rule_exceptions]] +rule = "FFR403" +path = "src/sqlbuild/compiler/compile/models.py" +reason = "Project macro modules import MacroContext from this path for their ctx annotation; the class is hosted natively and this module keeps that import path working." + +[[targets.python.rule_exceptions]] +rule = "FFR001" +path = "src/sqlbuild/compiler/compile/models.py" +reason = "The only model here is the natively hosted MacroContext, re-exported for project macro imports." + [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/integrations/dagster/classes/sqlbuild_cli_resource.py" diff --git a/python-boundary.toml b/python-boundary.toml index 8667730cda..4b1df2ee41 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -1,809 +1,778 @@ # Python that stays once Rust hosts SQLBuild. Every other Python file under src/sqlbuild is # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. +# Each `[[stays]]` entry names the one rule that keeps it Python and a note with the evidence: +# 1 = executor or adapter runtime and the shapes it consumes; 2 = thin shim that invokes user +# Python; 3 = public facade (facade only); 4 = glue around a Python-only third-party library. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 5847 +remaining_lines = 0 [[stays]] path = "executor" -reason = "executor" +rule = 1 note = "Runs plans against warehouses; a separate later decision." [[stays]] path = "adapters" -reason = "adapter" +rule = 1 note = "Built-in warehouse adapters that project adapters subclass and override." [[stays]] path = "adapter/contract" -reason = "adapter" +rule = 1 note = "BaseAdapter and StrictAdapter, the documented classes project adapters subclass." [[stays]] path = "adapter/discovery" -reason = "user-code" +rule = 2 note = "Imports project adapter modules." [[stays]] path = "adapter/relations" -reason = "adapter" -note = "Relation inspection wrappers adapters call, with per-connection caches." - -[[stays]] -path = "adapter/state_sql" -reason = "adapter" -note = "State-table SQL rendered through adapter callbacks." +rule = 1 +note = "Relation inspection wrappers adapters call, with per-connection caches; executed-statement classification and invalidation keys are native." [[stays]] path = "adapter/type_system" -reason = "adapter" -note = "Type normalisation adapters import; already a thin native wrapper." +rule = 1 +note = "Type normalisation and the public nullability rule callables adapters import and register (profile_inputs.rs recognises them by path); normalisation and the first_arg, conditional_result and safe_cast rules are native." [[stays]] path = "provider" -reason = "user-code" +rule = 2 note = "Provider base class users subclass, and injection into user functions." [[stays]] path = "python_nodes" -reason = "user-code" -note = "Decorators that run when user modules are imported." +rule = 3 +note = "Public decorators and authoring shapes re-exported by assets.py, tasks.py, loaders.py, hooks.py and retries.py, run when user modules are imported; RetryPolicy and AuditCase validation, retry delays and hook-read normalization are native (validate_retry_policy, retry_delay_seconds, validate_audit_case, normalize_hook_reads)." [[stays]] path = "microbatches" -reason = "executor" -note = "Microbatch state store the executor and adapters use." +rule = 1 +note = "Rule 1: the executor's direct-mode event store (warehouse I/O and retries), the event row codec and the event, scope and projection shapes it consumes. Coverage and replay projection, event IDs and state SQL are native (sqlbuild-state microbatches)." [[stays]] path = "rules" -reason = "public-api" +rule = 3 note = "Public custom-rule API and test helper." [[stays]] path = "diagnostics/_helpers/logging.py" -reason = "public-api" -note = "Installs SQLBuild's handlers on Python's logging, which users and integrations configure." - -[[stays]] -path = "diagnostics/_helpers/logging_formatters.py" -reason = "public-api" -note = "Python logging formatters for console and file diagnostics." - -[[stays]] -path = "diagnostics/_helpers/redaction.py" -reason = "public-api" -note = "Collects redacted extra fields from a Python LogRecord; the redaction itself is native." +rule = 4 +note = "Installs SQLBuild's handlers on Python's logging and owns the diagnostics context variables; SQL digests and record field sets are native (sql_log_record, debug_event_record_fields)." [[stays]] path = "diagnostics/classes/diagnostic_record_redactor.py" -reason = "public-api" -note = "Redaction entry points that Python logging handlers call per record." +rule = 4 +note = "Redaction entry points that Python logging handlers call per record; text, value and extra-field redaction are native." [[stays]] path = "diagnostics/classes/diagnostics_console_formatter.py" -reason = "public-api" -note = "A logging.Formatter subclass for the console stream." +rule = 4 +note = "Rule 4 glue: the logging.Formatter subclass for the console stream; the line text is native (format_diagnostics_console_record) and only stdlib exception formatting stays here." [[stays]] path = "diagnostics/classes/dynamic_stderr_handler.py" -reason = "public-api" +rule = 4 note = "A logging.Handler that follows the current sys.stderr." [[stays]] path = "diagnostics/classes/invocation_diagnostic_routing.py" -reason = "public-api" +rule = 4 note = "Attaches and removes per-invocation logging handlers." [[stays]] path = "diagnostics/classes/log_record_route_filter.py" -reason = "public-api" -note = "A logging.Filter that routes records to destinations." +rule = 4 +note = "A logging.Filter subclass; routing and SQL redaction of the record copy are native (route_diagnostic_log_record)." [[stays]] path = "diagnostics/main" -reason = "public-api" +rule = 1 note = "Diagnostics entry points built on Python logging: configuration, context, debug and SQL logging." [[stays]] path = "diagnostics/_helpers/process_resources.py" -reason = "public-api" +rule = 4 note = "Samples process resources for debug logging through Python's resource module." [[stays]] path = "rule_engine/_helpers/host" -reason = "user-code" +rule = 2 note = "Rule 2: the subprocess host that loads and runs user rule modules (custom_host, custom_evaluation, tracked_facts, module_state, custom_host_launch); host fan-out, partitioning and CPU capacity run natively." [[stays]] path = "rule_engine/_helpers/engine/custom_payloads.py" -reason = "user-code" +rule = 2 note = "Reads user rule functions (source lines, qualified names, import closures) to describe custom rules for native selection." [[stays]] path = "rule_engine/_helpers/engine/_custom_catalogue.py" -reason = "user-code" +rule = 2 note = "Imports the project's custom rule modules and collects their @rule functions for the native catalogue." [[stays]] path = "rule_engine/classes/rule_context.py" -reason = "user-code" -note = "The context object user rules receive." +rule = 3 +note = "The context object user rules receive; finding path and position validation is native (finding_position)." [[stays]] path = "rule_engine/_helpers/engine/definition.py" -reason = "user-code" +rule = 2 note = "The @rule decorator implementation; inspects user rule functions and their signatures." -[[stays]] -path = "rule_engine/_helpers/engine/hermeticity.py" -reason = "user-code" -note = "Statically checks the imports of user rule source files." - [[stays]] path = "rule_engine/_helpers/engine/rule_harness.py" -reason = "user-code" -note = "Public custom-rule test harness that runs user rules over a real compiled project." +rule = 2 +note = "Public custom-rule test harness that writes a temporary project and runs user rules over it; rule-option TOML literals are native (rule_option_toml_value)." [[stays]] path = "rule_engine/_helpers/engine/fact_replay.py" -reason = "user-code" -note = "Rule 2: the fact digest scheme shared with the rule host (tracked_facts.py, custom_host.py); recomputes values through the user-facing fact views and encodes live Python objects with orjson." +rule = 2 +note = "Rule 2: the fact digest scheme shared with the rule host (tracked_facts.py, custom_host.py); recomputes values through the user-facing fact views and encodes live Python objects with orjson. The readable-file selection is native (tracked_text_file_values)." [[stays]] path = "rule_engine/main/define.py" -reason = "user-code" +rule = 3 note = "Public @rule decorator entry point." [[stays]] path = "rule_engine/main/evaluate_rule.py" -reason = "user-code" +rule = 3 note = "Public custom-rule test harness entry point." [[stays]] path = "rule_engine/classes/audit_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/column_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/contract_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/declaration_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/graph_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/model_test_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/project_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/sql_facts.py" -reason = "user-code" +rule = 3 note = "Fact view user rules read through RuleContext; the Python API over native data." [[stays]] path = "rule_engine/classes/project_tree.py" -reason = "user-code" -note = "Project-tree fact view user rules read through RuleContext." +rule = 3 +note = "Project-tree fact view user rules read through RuleContext; the walk, path validation, file resolution and the children, descendants, glob and relative_parts queries are native (project_tree_*), and only the file read and the model lookup stay here." [[stays]] path = "rule_engine/classes/sql_document.py" -reason = "user-code" +rule = 4 note = "Lazy Polyglot SQL document user rules read through RuleContext." [[stays]] path = "rule_engine/classes/fact_reads.py" -reason = "user-code" +rule = 2 note = "Records the fact-view reads of one user rule evaluation." [[stays]] path = "rule_engine/classes/shared_fact_encoder.py" -reason = "user-code" +rule = 2 note = "Encodes the Python fact-view values user rules read, for their digests." [[stays]] path = "runtime/observability" -reason = "public-api" -note = "Documented observability API used throughout the executor." +rule = 1 +note = "Documented observability API used throughout the executor: lifecycle emitters, dispatcher, scopes and envelope shapes. The event catalogue, envelope, diagnostic log, run bound, operation metadata, hook and producer validation and JSON freezing are native (sqlbuild-core lifecycle_events)." [[stays]] path = "runtime/execution_history/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public execution history shapes below; storage, projection and validation are native (sqlbuild-state execution_history)." [[stays]] path = "runtime/execution_history/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: StoredEvent, filter, page and RunRecord dataclasses re-exported by sqlbuild.execution_history; __post_init__ delegates validation to native validate_history_* functions." [[stays]] path = "runtime/execution_history/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: EventFamily, RunStatus, cursor aliases and the storage protocols re-exported by sqlbuild.execution_history." [[stays]] path = "runtime/execution_history/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.execution_history and raised by the native backends." -[[stays]] -path = "runtime/execution_history/constants.py" -reason = "public-api" -note = "Rule 3 shapes: page limit and schema version constants re-exported by sqlbuild.execution_history." - [[stays]] path = "runtime/execution_limits" -reason = "executor" +rule = 1 note = "Deadline the executor scheduler and warehouse cursors enforce." [[stays]] path = "runtime/compute_logs/classes" -reason = "user-code" -note = "Rule 1 capture shim: sys.stdout/sys.stderr tees and a logging handler that record Python output from user code, adapters and the executor (_dispatch_with_compute_logs.py, compile_command/reuse_attempt.rs); storage itself is native LocalFilesystemComputeLogStorage." +rule = 1 +note = "Rule 1 capture shim: sys.stdout/sys.stderr tees and a logging.Handler that record Python output from user code, adapters and the executor (_dispatch_with_compute_logs.py, compile_command/reuse_attempt.rs); record retention, SQL redaction, identity and severity are native (compute_diagnostic_log_fields) and storage is native LocalFilesystemComputeLogStorage." [[stays]] path = "runtime/compute_logs/main" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim entry: installs the stream tees around the CLI command (_dispatch_with_compute_logs.py)." [[stays]] path = "runtime/compute_logs/_helpers" -reason = "user-code" +rule = 1 note = "Rule 1 tee shim: how much of a write the wrapped Python stream accepted (text_tee.py, binary_tee.py)." [[stays]] path = "runtime/compute_logs/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public compute log shapes below." [[stays]] path = "runtime/compute_logs/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: capture metadata and result dataclasses re-exported by sqlbuild.compute_logs." [[stays]] path = "runtime/compute_logs/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: ComputeLogStream and the ComputeLogStorage protocol re-exported by sqlbuild.compute_logs." [[stays]] path = "runtime/compute_logs/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.compute_logs and raised by the native storage." -[[stays]] -path = "runtime/compute_logs/constants.py" -reason = "public-api" -note = "Rule 3 shapes: format version, retention and read-limit constants re-exported by sqlbuild.compute_logs." - [[stays]] path = "runtime/event_exporting/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public sink declarations below; the dispatcher, queue, accounting and command scope are native EventExporterDispatcher/EventExporterCommandScope." [[stays]] path = "runtime/event_exporting/main/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public sink decorators and the user-sink invocation shims below." [[stays]] path = "runtime/event_exporting/main/lifecycle_event_sink.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: the @lifecycle_event_sink decorator re-exported by sqlbuild.sinks, run when user sink modules are imported." [[stays]] path = "runtime/event_exporting/main/command_output_sink.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: the @command_output_sink decorator re-exported by sqlbuild.sinks, run when user sink modules are imported." [[stays]] path = "runtime/event_exporting/main/get_lifecycle_event_sink_definition.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: reads the decorator's definition off a user function; re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/main/get_command_output_sink_definition.py" -reason = "public-api" +rule = 3 note = "Rule 3 facade: reads the decorator's definition off a user function; re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" -reason = "user-code" +rule = 2 note = "Rule 2 shim: the daemon delivery thread's target that calls one user lifecycle sink and reports how it ended to the native dispatcher, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." [[stays]] path = "runtime/event_exporting/main/_run_command_output_sinks.py" -reason = "user-code" +rule = 2 note = "Rule 2 shim: target of each dispatcher's single command-output sink thread; waits on a Python queue and calls user sinks in order on that one thread (provider thread affinity), reporting each outcome through a Python queue, so no native frame is on that thread when finalization can end it (event_exporting/dispatcher.rs)." [[stays]] path = "runtime/event_exporting/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shape: LifecycleEventSinkDefinition re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shape: LifecycleEventKind re-exported by sqlbuild.sinks." [[stays]] path = "runtime/event_exporting/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: errors the public decorators raise and the native dispatcher raises." [[stays]] path = "runtime/event_exporting/constants.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: decorator attribute names, sink kinds and severities the public decorators and discovery read." [[stays]] path = "runtime/output_capture/classes" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim: sys.stdout/sys.stderr tees forwarding accepted writes from user code, adapters and the executor to the native OutputCaptureDispatcher." [[stays]] path = "runtime/output_capture/main" -reason = "user-code" +rule = 1 note = "Rule 1 capture shim entry: swaps sys.stdout/sys.stderr for the tees around one CLI command (Rust cli/observability.rs)." [[stays]] path = "runtime/output_capture/__init__.py" -reason = "public-api" +rule = 3 note = "Package marker of the public command-output shapes below." [[stays]] path = "runtime/output_capture/models.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: CommandOutputRecord and sink definitions re-exported by sqlbuild.sinks; validation is native validate_command_output_record." [[stays]] path = "runtime/output_capture/types.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: CommandOutputStream re-exported by sqlbuild.sinks, priorities and the exporter protocol." [[stays]] path = "runtime/output_capture/exceptions.py" -reason = "public-api" +rule = 3 note = "Rule 3 shapes: CommandOutputValidationError re-exported by sqlbuild.sinks and raised by the native records." [[stays]] path = "runtime/output_capture/constants.py" -reason = "public-api" -note = "Rule 3 shapes: the invocation-context variable and wire constants used by the models and integrations." +rule = 3 +note = "Rule 3 shapes: the command-output sink parameter name and the installed producer version, no logic." [[stays]] path = "integrations/dagster" -reason = "public-api" -note = "Documented orchestrator integration and rule 4 Dagster glue: context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing are native." +rule = 4 +note = "Documented orchestrator integration and rule 4 Dagster glue: overridable translator methods, translator invocation, context reads, subprocess streaming and Dagster event construction; CLI arguments, selector logs, payload and failure parsing, asset/check/selection projection of the DAG artifact and the default translator mappings (dagster_default_*) are native." [[stays]] path = "integrations/rivers" -reason = "public-api" -note = "Documented orchestrator integration." +rule = 4 +note = "Documented orchestrator integration and rule 4 Rivers glue: translator invocation and AssetDef construction; the asset projection of the DAG artifact is native." [[stays]] path = "integrations/dlt" -reason = "user-code" -note = "Loader runtime that runs inside user loaders; synthetic loader record building and naming are native." +rule = 4 +note = "Loader runtime that runs inside user loaders and calls the dlt destination factories; synthetic loader record building, naming and adapter-config to destination mapping are native (dlt_destination_arguments)." [[stays]] path = "integrations/ingestr" -reason = "user-code" -note = "Loader runtime that runs inside the executor load phase; synthetic loader record building and naming are native." +rule = 1 +note = "Loader runtime that runs the ingestr subprocess inside the executor load phase; synthetic loader record building, naming, destination URIs and command lines are native." [[stays]] path = "integrations/dbt/_helpers/profile/load.py" -reason = "user-code" +rule = 4 note = "Rule 4: PyYAML glue only; parses dbt_project.yml and profiles.yml with yaml.safe_load and reports YAMLError as DbtProfileError. Existence, mapping, field and target validation and output selection are native (dbt_interop/profile_load.rs)." [[stays]] path = "integrations/dbt/_helpers/profile/render.py" -reason = "user-code" -note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does." +rule = 4 +note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does; as_bool and as_number coercions are native." [[stays]] path = "cli/commands/main/entrypoint/_dispatch_with_compute_logs.py" -reason = "user-code" +rule = 1 note = "Captures the command's stdout, stderr and logging into the local compute log around user code." [[stays]] path = "cli/commands/_helpers/entry/compute_log_diagnostics.py" -reason = "user-code" +rule = 1 note = "Logs compute log capture failures through Python logging during capture." [[stays]] path = "cli/compile_reuse/_helpers/provider_settings.py" -reason = "user-code" +rule = 4 note = "Inspects user provider settings classes for the environment and files they read." -[[stays]] -path = "cli/compile_reuse/_helpers/runtime_identity.py" -reason = "user-code" -note = "Interpreter and invocation facts of the Python host that user code runs in." - [[stays]] path = "cli/compile_reuse/_helpers/startup_inputs.py" -reason = "user-code" -note = "Interpreter and site-path inputs user modules import from." +rule = 2 +note = "Reads the live interpreter's sys.path, site-packages, executable and prefixes that user modules import from; absolute paths, .pth discovery, ordering and stamps are native (startup_input_stamps)." [[stays]] path = "cli/entry/main/entry.py" -reason = "public-api" +rule = 3 note = "The installed sqlbuild-python console entry point." [[stays]] path = "cli/entry/main/launched.py" -reason = "public-api" +rule = 3 note = "The Python entry point the native sqb executable hands invocations to." [[stays]] path = "cli/entry/_helpers/streams.py" -reason = "public-api" +rule = 3 note = "Reconfigures the Python process's standard streams for the console entry points." [[stays]] path = "compiler/python_nodes/_helpers/identity.py" -reason = "user-code" +rule = 2 note = "Identity hashing over live user functions (source and globals)." [[stays]] path = "compiler/macro_bridge/_helpers/store_environment.py" -reason = "user-code" -note = "Fingerprints the Python interpreter and installed packages that project macros import from; the macro call store cache key." +rule = 2 +note = "Fingerprints the installed packages, import path and live module files that project macros import from; the macro call store cache key. The interpreter stamp it shares with compile reuse is native (interpreter_identity)." [[stays]] path = "rule_engine/classes/runtime_guard.py" -reason = "user-code" +rule = 2 note = "Guards custom-rule execution against side effects in user rule code." [[stays]] path = "rule_engine/classes/module_state_monitor.py" -reason = "user-code" +rule = 2 note = "Watches user rule modules for state changes between custom-rule runs." [[stays]] path = "compiler/compile/_helpers/render/user_macros.py" -reason = "user-code" +rule = 2 note = "Loads and executes project macro modules, and builds the declaration and enum mappings user macros read." [[stays]] path = "compiler/macro_bridge/classes/macro_bridge.py" -reason = "user-code" +rule = 2 note = "Tracks the loaded user modules macro call results depend on, and runs a render stage through the bridge." [[stays]] path = "compiler/python_nodes/_helpers/user_inventory.py" -reason = "user-code" +rule = 2 note = "Reads loaded user node functions: identities over live functions and decorated dependency names; graph construction is native." [[stays]] path = "compiler/discovery/_helpers/filesystem/project_modules.py" -reason = "user-code" +rule = 2 note = "Rule 2 shim: imports project Python node, hook, provider, sink and materialization modules and reads live members, signatures, provider parameter annotations and source lines; record building, provider naming and validation, settings errors, factory results and duplicate checks are native." [[stays]] path = "compiler/discovery/_helpers/validation/discovery.py" -reason = "user-code" +rule = 2 note = "Reads decorator definitions from loaded project Python node functions." [[stays]] path = "compiler/discovery/main/runtime_extensions.py" -reason = "user-code" +rule = 2 note = "Binds imported project providers to event exporter and sink declarations." [[stays]] path = "compiler/compile/_helpers/diagnostics/python_resources.py" -reason = "user-code" +rule = 2 note = "Reads docstrings and source lines of loaded project Python functions and provider classes." [[stays]] path = "compiler/discovery/_helpers/filesystem/command_output_sinks.py" -reason = "user-code" +rule = 2 note = "Inspects loaded user sink functions." [[stays]] path = "compiler/pipeline/_helpers/materializations.py" -reason = "user-code" +rule = 2 note = "Imports user materialization modules." [[stays]] path = "compiler/macro_bridge/main" -reason = "user-code" +rule = 2 note = "Macro bridge entry points for the running render stage that executes user macros." [[stays]] path = "compiler/compile/_helpers/macro_bridge/call_store.py" -reason = "user-code" +rule = 2 note = "Hands the running macro bridge its user macro call store." [[stays]] path = "compiler/python_nodes/main/identity.py" -reason = "user-code" +rule = 2 note = "Identity entry point over live user node functions." [[stays]] path = "compiler/python_nodes/main/_hook_identities.py" -reason = "user-code" +rule = 2 note = "Builds identities of loaded user hook functions." [[stays]] path = "compiler/auditing/main/above.py" -reason = "public-api" +rule = 3 note = "Audit bound helper project audit factories call." [[stays]] path = "compiler/auditing/main/below.py" -reason = "public-api" +rule = 3 note = "Audit bound helper project audit factories call." [[stays]] path = "compiler/auditing/main/outside.py" -reason = "public-api" +rule = 3 note = "Audit bound helper project audit factories call." [[stays]] path = "compiler/auditing/main/audit_factory.py" -reason = "public-api" +rule = 3 note = "Decorator that declares project audit factories." [[stays]] path = "compiler/auditing/main/get_audit_factory_definition.py" -reason = "public-api" +rule = 3 note = "Reads the declaration attached to a project audit factory." [[stays]] path = "compiler/sql_analysis/main/import_polyglot.py" -reason = "public-api" +rule = 4 note = "Imports the Polyglot Python package for the Python API." [[stays]] path = "compiler/sql_analysis/main/import_polyglot_sql.py" -reason = "public-api" +rule = 4 note = "Imports the Polyglot SQL Python package for the Python API." [[stays]] path = "compiler/sql_analysis/_helpers/polyglot.py" -reason = "public-api" +rule = 4 note = "Wraps the Polyglot Python package with complexity guards." [[stays]] path = "assets.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (asset, get_asset_definition)." [[stays]] path = "audits.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "checks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (check, get_check_definition)." [[stays]] path = "compute_logs.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (open_local_compute_log_storage)." [[stays]] path = "execution_history.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (append_and_project, canonical_event_id, canonical_event_content, project_runs, validate_page_limit)." [[stays]] path = "factories.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (factory, get_factory_definition)." [[stays]] path = "functions.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "hooks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (hook, get_hook_definition)." [[stays]] path = "loaders.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "observability.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "postgres_history.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "providers.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "refs.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (model, source, seed)." [[stays]] path = "retries.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "sinks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "sqlite_history.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports only." [[stays]] path = "tasks.py" -reason = "public-api" -note = "Public facade." +rule = 3 +note = "Public facade: re-exports plus one-line delegating wrappers (task, get_task_definition)." [[stays]] path = "cost/models.py" -reason = "executor" +rule = 1 note = "Shape rule: data records only (StatementExecutionTelemetry, CostResourceContext, RunCostSummary), no logic; consumed by adapters/snowflake/classes/snowflake_adapter.py, adapters/snowflake/classes/snowflake_cursor.py, executor/build/classes/build_scheduler.py and 2 more." [[stays]] path = "cli/compile_reuse/models.py" -reason = "user-code" -note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, cli/compile_reuse/_helpers/runtime_identity.py." - -[[stays]] -path = "compiler/source_freshness/constants.py" -reason = "adapter" -note = "Shape rule: data constants only (SOURCE_FRESHNESS_TABLE_NAME, COLUMN_SOURCE_NAME, COLUMN_TARGET_DATABASE, COLUMN_TARGET_SCHEMA, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapter/state_sql/_helpers/source_freshness.py, adapters/bigquery/classes/bigquery_adapter.py and 6 more." +rule = 4 +note = "Shape rule: data records only (CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult), no logic; consumed by cli/compile_reuse/_helpers/provider_settings.py, and read natively by the compile reuse check." [[stays]] path = "compiler/auditing/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (AuditSeverity, AuditRunScope, AuditAttachmentKind, AuditEvaluationMode, ...), no logic; consumed by audits.py, compiler/auditing/main/above.py, compiler/auditing/main/below.py and 17 more." [[stays]] path = "compiler/python_nodes/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (PythonNodeKind, PythonNodeStatus, SkipMode, PythonNodeFanInAction), no logic; consumed by assets.py, assets.pyi, executor/load/_helpers/dag_runtime.py and 16 more." [[stays]] path = "compiler/macro_bridge/constants.py" -reason = "user-code" +rule = 2 note = "Shape rule: data constants only (ACTIVE_MACRO_BRIDGE, COMPILES_STARTED, STORE_MODULE_PATHS, MACRO_CALL_STORE_FILE_NAME, ...), no logic; consumed by compiler/macro_bridge/_helpers/store_environment.py, compiler/macro_bridge/classes/macro_bridge.py, compiler/macro_bridge/main/_claim_first_compile.py and 2 more." [[stays]] path = "compiler/references/models.py" -reason = "executor" +rule = 1 note = "Shape rule: data records only (RelationName, ProjectRelation, ProjectRelationIndex), no logic; consumed by executor/python_nodes/classes/runtime_relation_guard.py." [[stays]] path = "runtime/contracts/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (ExecutionResourceKind, ConnectionElapsedCallback, NodeStartCallback), no logic; consumed by executor/build/_helpers/scheduler.py, executor/build/_helpers/source_node.py, executor/build/classes/build_scheduler.py and 15 more." [[stays]] path = "errors/contracts/exceptions.py" -reason = "executor" +rule = 1 note = "Shape rule: data exceptions only (SharedInputError, ExecutorInputError), no logic; consumed by executor/audit_results/_helpers/sql.py, executor/build/_helpers/external_source_loads.py, executor/build/_helpers/retention.py and 53 more." [[stays]] path = "compiler/fingerprints/constants.py" -reason = "executor" -note = "Shape rule: data constants only (FINGERPRINT_TABLE_NAME, COLUMN_NODE_TYPE, COLUMN_NODE_NAME, COLUMN_TARGET_DATABASE, ...), no logic; consumed by adapter/contract/classes/duckdb_backed_adapter.py, adapters/bigquery/classes/bigquery_adapter.py, adapters/databricks/classes/databricks_adapter.py and 13 more." +rule = 1 +note = "Shape rule: Python-only node type names (NODE_TYPE_HOOK, NODE_TYPE_QUERY_DIFF_ARTIFACT), no logic; table and column names are native (sqlbuild._native)." [[stays]] path = "diagnostics/models.py" -reason = "public-api" +rule = 4 note = "Shape rule: data records only (ProcessResourceUsage, DiagnosticRoutingOptions), no logic; consumed by cli/commands/main/entrypoint/_dispatch_with_compute_logs.py, diagnostics/classes/invocation_diagnostic_routing.py, diagnostics/main/_log_process_resources.py and 3 more." [[stays]] path = "compiler/dag/types.py" -reason = "public-api" +rule = 4 note = "Shape rule: data enums and aliases only (NodeKind), no logic; consumed by integrations/dagster/constants.py, integrations/rivers/constants.py." [[stays]] path = "runtime/contracts/models.py" -reason = "executor" +rule = 1 note = "Shape rule: data records only (ConnectionHooks), no logic; consumed by executor/pipeline/_helpers/scenario.py." [[stays]] path = "cursor_algebra/types.py" -reason = "executor" +rule = 1 note = "Shape rule: data enums and aliases only (CursorScalar, BoundSentinel, Bound), no logic; consumed by executor/load/_helpers/cursors.py, executor/load/main/_execute.py, executor/run/_helpers/materializations/microbatch.py and 2 more." [[stays]] path = "cost/types.py" -reason = "adapter" +rule = 1 note = "Shape rule: data enums and aliases only (CostCapability, CostStatus), no logic; consumed by adapters/snowflake/classes/snowflake_adapter.py." [[stays]] path = "compiler/lineage/types.py" -reason = "adapter" -note = "Shape rule: data enums and aliases only (InferredNullability), no logic; consumed by adapter/contract/types.py, adapter/type_system/_helpers/inference_rules.py, adapter/type_system/main/conditional_result_nullability.py and 2 more." +rule = 1 +note = "Shape rule: data enums and aliases only (InferredNullability), no logic; consumed by adapter/contract/types.py, adapter/type_system/main/conditional_result_nullability.py, adapter/type_system/main/first_arg_nullability.py and adapter/type_system/main/safe_cast_nullability.py, and read natively to build nullability results." [[stays]] path = "compiler/macro_bridge/models.py" -reason = "user-code" +rule = 2 note = "Shape rule: data records only (ModuleSources), no logic; consumed by compiler/macro_bridge/_helpers/store_environment.py, compiler/macro_bridge/classes/macro_bridge.py." [[stays]] path = "integrations/dbt/exceptions.py" -reason = "user-code" +rule = 4 note = "Shape rule: data exceptions only (DbtProfileError), no logic; consumed by integrations/dbt/_helpers/profile/load.py, integrations/dbt/_helpers/profile/render.py." -[[stays]] -path = "compiler/migrations/constants.py" -reason = "executor" -note = "Shape rule: data constants only (MIGRATION_TABLE_NAME, MIGRATION_WRITE_ATTEMPTS, COLUMN_MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME), no logic; consumed by adapters/sqlserver/classes/sqlserver_adapter.py, executor/janitor/constants.py, executor/migrations/_helpers/columns.py and 4 more." - [[stays]] path = "diagnostics/constants.py" -reason = "public-api" -note = "Shape rule: data constants only (LOGGER_ROOT_NAME, LOGGER_PREFIX, SQL_DIGEST_FIELD, SQL_TEXT_FIELD, ...), no logic; consumed by diagnostics/_helpers/logging.py, diagnostics/_helpers/redaction.py, diagnostics/classes/invocation_diagnostic_routing.py and 2 more." +rule = 4 +note = "Shape rule: the SQL_PRIVATE_RECORD_MARKER object() sentinel only, no logic; its identity marks trusted private SQL records for the native compute log handler; the record field names are native (sqlbuild._native); consumed by diagnostics/classes/invocation_diagnostic_routing.py." [[stays]] path = "cursor_algebra/exceptions.py" -reason = "executor" +rule = 1 note = "Shape rule: data exceptions only (CursorAlgebraError), no logic; consumed by executor/run/models.py." [[stays]] path = "spec/contracts/constants.py" -reason = "adapter" -note = "Shape rule: data constants only (DEFAULT_SEED_CSV_SETTINGS, ZERO_DAY_CURSOR_DURATION), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/classes/duckdb_backed_adapter.py, adapter/contract/classes/materialization.py and 6 more." +rule = 1 +note = "Shape rule: data constants only (DEFAULT_SEED_CSV_SETTINGS), no logic; consumed by adapter contract classes." [[stays]] path = "cli/entry/constants.py" -reason = "public-api" +rule = 3 note = "Shape rule: data constants only (NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, OUTPUT_STREAM_ENCODING, OUTPUT_STREAM_ERRORS, ...), no logic; consumed by cli/entry/_helpers/streams.py, cli/entry/main/entry.py, cli/entry/main/launched.py." -[[stays]] -path = "cli/compile_reuse/constants.py" -reason = "user-code" -note = "Shape rule: data constants only (REUSE_FORMAT_VERSION, OUTPUT_FILE_DESCRIPTORS), no logic; consumed by cli/compile_reuse/_helpers/runtime_identity.py." - [[stays]] path = "sql_values/exceptions.py" -reason = "adapter" +rule = 1 note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; consumed by adapter/contract/classes/base_adapter.py, adapter/contract/exceptions.py." [[stays]] -path = "cli/output/constants.py" -reason = "public-api" -note = "Shape rule: data constants only (INTEGRATION_RESULT_PATH_ENV), no logic; consumed by integrations/dagster/_helpers/invocation_factory.py." +path = "compiler/compile/types.py" +rule = 1 +note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." [[stays]] -path = "compiler/compile/types.py" -reason = "executor" -note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/janitor/_helpers/plan.py, executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." +path = "rule_engine/types.py" +rule = 3 +note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, RuleOptionValue, FactKey, CustomRulePlan), no logic; re-exported by rules/__init__.py and consumed by rule_engine/main/define.py, rule_engine/_helpers/host/custom_host.py, rule_engine/_helpers/engine/fact_replay.py and 4 more." -[[handled_native_errors]] -location = "compiler/planner/models.py::CursorBounds._parse_inferred" -note = "A bound that is not integer text is parsed as a timestamp instead; CursorAlgebraError is an input error, and native compiler failures are not caught." +[[stays]] +path = "rule_engine/constants.py" +rule = 2 +note = "Shape rule: custom-rule host and fact-name constants (CUSTOM_HOST_MAX_TRACKED_READS, CUSTOM_HOST_STARTUP_ENVIRONMENT, FACT_PROJECT_MODELS, ...) used by the Python custom-rule host, no logic; versions, launch module, shared keys and the import allowlist are native (sqlbuild._native)." + +[[stays]] +path = "compiler/compile/exceptions.py" +rule = 2 +note = "Shape rule: data exceptions only (MacroDeclarationLookupError, a CompileInputError that is also a KeyError, which a native error class with one parent cannot express), no logic; consumed by compiler/compile/_helpers/render/user_macros.py." + +[[stays]] +path = "compiler/sql_analysis/constants.py" +rule = 4 +note = "Rule 4: polyglot_sql glue only; the entry point, call-shape and option names (POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_COMPLEXITY_GUARD_OPTION, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, ...) that compiler/sql_analysis/_helpers/polyglot.py uses to wrap the polyglot_sql Python package." + +[[stays]] +path = "compiler/compile/models.py" +rule = 3 +note = "Shape rule: re-export only (MacroContext, now hosted natively), no logic; project macro modules import it from this path to annotate their ctx parameter, as tests/e2e/fixtures/waffle_shop/models/marts/_macros/time.py does." diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index 651959d413..78c4a05c6e 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -7,8 +7,13 @@ from scripts.cold_compile_performance._helpers.dense_project import dense_model_name from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules -from sqlbuild.rule_engine.models import Rule, RulesConfig +from sqlbuild._native import ( + Rule, + RulesConfig, + build_rule_catalogue, + load_rules_config, + select_catalogue_rules, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( _RETAINED_BENCHMARK_PAYLOAD_KEYS, diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 55b090329d..015c086250 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -39,10 +39,8 @@ FailureCase, ProjectRun, ) -from sqlbuild._native import DiscoveryError, ResourceIdentityError -from sqlbuild.compiler.compile import constants as compile_constants +from sqlbuild._native import DiscoveryError, ResourceIdentityError, ScopeDiagnosticCode from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.scopes.types import ScopeDiagnosticCode _SOURCE_DIRECTORY: str = "source" _CODE_KEYWORD: str = "code" @@ -96,18 +94,10 @@ def discovery_error_codes() -> frozenset[str]: def render_error_codes() -> frozenset[str]: """Return every code render, scope, and attachment can report, from compiler constants.""" - compile_codes: set[str] = { - value - for name, value in vars(compile_constants).items() - if name.endswith("_CODE") - and isinstance(value, str) - and DIAGNOSTIC_CODE_PATTERN.fullmatch(value) - } return frozenset( { CompileInputError.code, BUILT_IN_AUDIT_SHADOW_CODE, - *compile_codes, *(code.value for code in ScopeDiagnosticCode), *RENDER_RAISED_CODES, } diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index 0730f7136e..5116c308c5 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -26,8 +26,7 @@ ANALYSIS_UPSTREAM_CHAIN_HOPS, COMPILED_PROJECT_CAPTURE_TYPE, ) -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, CompiledProject _SET_OPERATION: re.Pattern[str] = re.compile(r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE) _LEADING_CTE: re.Pattern[str] = re.compile(r"^\s*WITH\s+\w+\s+AS\s*\(", re.IGNORECASE) diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index b4febeba93..da0e2810aa 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -35,12 +35,11 @@ STRATEGY_HEADER_KEYS, TAB, ) -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES +from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, DiscoveredProjectInputs from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs _OMITTED_DISCOVERY_FIELDS: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get( - f"{DiscoveredProjectInputs.__module__}:{DiscoveredProjectInputs.__qualname__}", frozenset() + "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", frozenset() ) _CROSS_FILE_MACRO_IMPORT: re.Pattern[str] = re.compile(r"^(?:from|import) macros\.", re.MULTILINE) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 3d97b3a040..bdfad436fa 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -661,7 +661,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: } ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( "compile/_helpers/diagnostics", - "contracts", "sql_analysis", ) ANALYSIS_NATIVE_CODE_SCAN_ROOTS: tuple[str, ...] = ( diff --git a/scripts/python_boundary/_helpers/policy.py b/scripts/python_boundary/_helpers/policy.py index a3c2538b46..8bcd3ac7c0 100644 --- a/scripts/python_boundary/_helpers/policy.py +++ b/scripts/python_boundary/_helpers/policy.py @@ -2,11 +2,18 @@ from __future__ import annotations +import re import tomllib from pathlib import Path from typing import Any -from scripts.python_boundary.constants import ESCAPING_PATH_PARTS, SOURCE_ROOT, STAY_REASONS +from scripts.python_boundary.constants import ( + ESCAPING_PATH_PARTS, + NOTE_RULE_PATTERN, + SOURCE_ROOT, + STAY_KEYS, + STAY_RULES, +) from scripts.python_boundary.models import BoundaryPolicy, ReviewedNativeHandler, StayEntry @@ -19,8 +26,9 @@ def read_boundary_policy(path: Path) -> BoundaryPolicy: stays=tuple( StayEntry( path=str(entry.get("path", "")), - reason=str(entry.get("reason", "")), + rule=entry.get("rule"), note=str(entry.get("note", "")), + unknown_keys=tuple(sorted(set(entry) - STAY_KEYS)), ) for entry in document.get("stays", ()) ), @@ -49,14 +57,33 @@ def get_policy_errors(*, root: Path, policy: BoundaryPolicy) -> tuple[str, ...]: errors.append(f"stays entry {entry.path!r} must name a module under {SOURCE_ROOT}.") elif not (source / entry.path).exists(): errors.append(f"stays entry {entry.path!r} does not exist under {SOURCE_ROOT}.") - if entry.reason not in STAY_REASONS: - errors.append( - f"stays entry {entry.path!r} has reason {entry.reason!r}; " - f"use one of {', '.join(STAY_REASONS)}." - ) - if not entry.note.strip(): - errors.append(f"stays entry {entry.path!r} needs a note explaining why it stays.") + errors.extend(_stay_rule_errors(entry)) for handler in policy.handled_native_errors: if not handler.note.strip(): errors.append(f"handled_native_errors entry {handler.location!r} needs a note.") return tuple(errors) + + +def _stay_rule_errors(entry: StayEntry) -> tuple[str, ...]: + """Errors for a stays entry without exactly one valid rule backed by a note.""" + errors: list[str] = [] + choices: str = "; ".join(f"{number} = {meaning}" for number, meaning in STAY_RULES.items()) + if entry.unknown_keys: + errors.append( + f"stays entry {entry.path!r} has unknown key(s) {', '.join(entry.unknown_keys)}; " + "an entry has only path, rule and note." + ) + rule: object = entry.rule + if rule is None: + errors.append(f"stays entry {entry.path!r} needs a rule: one of {choices}.") + elif isinstance(rule, bool) or not isinstance(rule, int) or rule not in STAY_RULES: + errors.append(f"stays entry {entry.path!r} has rule {rule!r}; use one of {choices}.") + if not entry.note.strip(): + errors.append(f"stays entry {entry.path!r} needs a note with the evidence for its rule.") + cited: set[int] = {int(number) for number in re.findall(NOTE_RULE_PATTERN, entry.note)} + if isinstance(rule, int) and not isinstance(rule, bool) and cited - {rule}: + errors.append( + f"stays entry {entry.path!r} has rule {rule} but its note cites rule " + f"{', '.join(str(number) for number in sorted(cited - {rule}))}." + ) + return tuple(errors) diff --git a/scripts/python_boundary/constants.py b/scripts/python_boundary/constants.py index ce4aa5a72d..2c1eac8012 100644 --- a/scripts/python_boundary/constants.py +++ b/scripts/python_boundary/constants.py @@ -6,5 +6,12 @@ NATIVE_MARKER: str = "native" SUPPRESS_NAME: str = "suppress" MODULE_SCOPE: str = "" -STAY_REASONS: tuple[str, ...] = ("user-code", "adapter", "executor", "public-api") +STAY_RULES: dict[int, str] = { + 1: "executor or adapter runtime and the shapes it consumes", + 2: "thin shim that invokes user Python", + 3: "public facade", + 4: "glue around a Python-only third-party library", +} +STAY_KEYS: frozenset[str] = frozenset({"path", "rule", "note"}) +NOTE_RULE_PATTERN: str = r"\b[Rr]ule (\d+)\b" ESCAPING_PATH_PARTS: frozenset[str] = frozenset({".", ".."}) diff --git a/scripts/python_boundary/models.py b/scripts/python_boundary/models.py index 0abf0f0f5b..86a2ba6d93 100644 --- a/scripts/python_boundary/models.py +++ b/scripts/python_boundary/models.py @@ -7,11 +7,12 @@ @dataclass(frozen=True) class StayEntry: - """A module or package that stays Python, and why.""" + """A module or package that stays Python, the rule that keeps it, and the evidence.""" path: str - reason: str + rule: object note: str + unknown_keys: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/__init__.py b/src/sqlbuild/__init__.py index d0c8b749a6..e69de29bb2 100644 --- a/src/sqlbuild/__init__.py +++ b/src/sqlbuild/__init__.py @@ -1 +0,0 @@ -"""SQLBuild package.""" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4a5eeb5dc1..5bfbae85ff 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,5 +1,6 @@ """Private SQLBuild native engine bindings.""" +import logging import threading from collections import Counter from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set @@ -13,7 +14,7 @@ from enum import StrEnum from pathlib import Path, PurePath from threading import Event from types import TracebackType -from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict +from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -26,47 +27,26 @@ from sqlbuild.adapter.contract.models import ( ExpressionInferenceProfile, QueryResult, RelationInfo, + RetentionRequest, RowDiffSampling, RowDiffTolerances, ) from sqlbuild.adapter.contract.types import ( AdapterExecute, FrameworkType, + MigrationTransfer, TablePromotionMode, TypeDialect, ) -from sqlbuild.adapter.relations.models import InspectionQueryRecord +from sqlbuild.adapter.relations.models import InspectionQueryRecord, StatementMetadataEffect from sqlbuild.cli.compile_reuse.models import CompileReuseRequest -from sqlbuild.compiler.auditing.models import ( - AuditGateIdentity, - MeasurementContract, - MeasurementThresholds, -) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, AuditOutcome, AuditRunScope, AuditSeverity, -) -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledLineageSourceFact, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, - CompilerDiagnostic, - CompileSqlReference, - DeclarationResolutionContext, - InferredColumn, - LoadedMacro, - MacroContext, - ModelInputBuildContext, - StaticMacroExport, - _native, + ThresholdOperator, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -78,142 +58,24 @@ from sqlbuild.compiler.compile.types import ( CompiledResourceType, CompiledSeedView, CompiledSourceView, + ExternalSqlReferenceResolver, FunctionLanguage, RelationKeyRow, SourceInput, SqlTestMode, TypedSqlValueRenderer, ) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredCheckFunction, - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredEventExporterDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMacroFile, - DiscoveredMaterializationFile, - DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredProviderUsage, - DiscoveredPythonNodeFunctions, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, - EnumDeclaration, - SqlTestParameterDeclaration, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import ( InferredNullability, ) -from sqlbuild.compiler.migrations.models import ( - ColumnMigrationEvent, - MigrationEvent, - MigrationRelation, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.migrations.types import ( - MigrationDecision, - MigrationPromotion, - OldNameViewEventType, -) -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineResult, - ProjectGraph, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ChainStep, - CloneSourcePlanEntry, - ColumnMigrationPlanEntry, - CursorBounds, - CursorInputEvidence, - CursorInputRelation, - CursorOverrides, - EmptyCursorInputDecision, - FunctionPlanEntry, - FutureCursorSafetyEvidence, - MaximumStartPolicyInputs, - MaximumStartSafetyEvidence, - ModelCursorSnapshot, - ModelMigrationPlanEntry, - ModelPlanEntry, - OldNameView, - OldNameViewPlanEntry, - PlanOutput, - RetentionPlanEntry, - ScenarioExecutionPlan, - ScenarioFixturePlan, - ScenarioRelationMap, - ScenarioRelationPlan, - SchemaFinding, - SeedPlanEntry, - SourceLoadPlanEntry, - SqlTestPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, - ScenarioArtifactKind, -) -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.compiler.references.models import ( ProjectRelation, ProjectRelationIndex, RelationName, ) -from sqlbuild.compiler.references.types import ( - ExternalSqlReferenceResolver, - HardCodedRelationOwnerKind, - SqlReferenceKind, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationExplanation, - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibility, - MovePreview, - ResourceIdentity, - ScopeBrowseResult, - ScopeDiagnostic, - ScopeIndex, - ScopeListResult, - ScopeLookup, - ScopeReport, - ScopeReportFilters, - ScopeTargetQuery, - UsageRecord, - VisibilityRecord, - VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ScopeKind, -) -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord, -) -from sqlbuild.compiler.sql_analysis.models import ( - SqlLexicalSyntax, -) from sqlbuild.cost.models import CostResourceContext, RunCostSummary, StatementExecutionTelemetry from sqlbuild.cost.types import CostStatus -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, Duration, TimestampValue from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.models import ProcessResourceUsage from sqlbuild.executor.auditing.models import ( @@ -251,26 +113,25 @@ from sqlbuild.executor.testing.models import ( SqlTestExecutionResult, StepResult, ) -from sqlbuild.integrations.dbt.models import ( - ResolvedDbtProfileOutput, +from sqlbuild.microbatches.models import ( + MicrobatchCoverageProjection, + MicrobatchEvent, + MicrobatchInterval, + MicrobatchScope, + ReplayRequirementProjection, ) -from sqlbuild.integrations.dbt.types import DbtInvoker -from sqlbuild.microbatches.models import MicrobatchInterval -from sqlbuild.presentation.types import TransientLineOwner +from sqlbuild.microbatches.types import MicrobatchRecordType from sqlbuild.provider.classes.container import ProviderContainer +from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.python_nodes.models import ( + AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef, ) from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.rule_engine.models import ( - Finding, - Rule, - RulesConfig, - RulesResult, -) +from sqlbuild.rule_engine.types import RuleCheck, RuleOptionValue from sqlbuild.runtime.compute_logs.models import ( CaptureByteCounts, CaptureInventory, @@ -297,6 +158,7 @@ from sqlbuild.runtime.execution_history.types import ( ) from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.models import DiagnosticLog, LifecycleEvent +from sqlbuild.runtime.observability.types import JSONValue from sqlbuild.runtime.output_capture.models import ( BoundCommandOutputSink, CommandOutputCaptureSummary, @@ -304,48 +166,70 @@ from sqlbuild.runtime.output_capture.models import ( OutputCaptureContext, ) from sqlbuild.runtime.output_capture.types import CommandOutputStream -from sqlbuild.spec.contracts.models import ( - CostConfig, - DbtConfig, - DefaultsConfig, - ExecutionLimitsConfig, - FutureCursorsConfig, - LoaderDestinationParts, - LocalConfig, - LocalDbtConfig, - ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, - ScenarioConfig, - SchemaAuditInstance, - SchemaColumn, - SchemaDynamicColumnFamily, - SchemaModelEntry, - SchemaSeedEntry, - SettingsConfig, - SnapshotsConfig, - SourceColumnEntry, - SourceEntry, - SourceFreshnessAgePolicy, - SourceLocation, - StartCursorsConfig, - TargetConfig, -) -from sqlbuild.spec.contracts.types import ( - EventExportSeverity, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - SourceFreshnessStrategy, - SourceFreshnessValueKind, - SourceWriteStrategy, - WarehouseGroup, -) -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import ( - StateSqlValueType, -) BUILD_IDENTITY: str +MICROBATCH_START_SENTINEL: str +MICROBATCH_END_SENTINEL: str +MIGRATION_FINGERPRINT_METADATA_KEY: str +SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str +SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str +PROJECT_CONFIG_FILENAME: str +PYTHON_INIT_MODULE_STEM: str +PYTHON_NODE_ROOT: str +SETTINGS_SECTION: str +TABLE_PROMOTION_MODE_SETTING_KEY: str +STAGED_PROMOTION_MODE: str +STAGE_CAPTURE_DIR_ENV_VAR: str +COMPILE_REUSE_DISABLE_ENV_VAR: str +SQLBUILD_QUERY_TAG_APP: str +STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] +STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] +GRAIN_ORDER: dict[CursorGrain, int] +INVOCATION_CONTEXT_ENV: str +COMMAND_OUTPUT_RECORD_TYPE: str +COMMAND_OUTPUT_LOSS_RECORD_TYPE: str +INTEGRATION_RESULT_PATH_ENV: str +FINGERPRINT_TABLE_NAME: str +SOURCE_FRESHNESS_TABLE_NAME: str +MIGRATION_TABLE_NAME: str +COLUMN_MIGRATION_TABLE_NAME: str +OLD_NAME_VIEW_TABLE_NAME: str +COLUMN_NODE_TYPE: str +COLUMN_NODE_NAME: str +COLUMN_TARGET_DATABASE: str +COLUMN_TARGET_SCHEMA: str +COLUMN_TARGET_NAME: str +COLUMN_RUN_ID: str +COLUMN_DEFINITION_HASH: str +COLUMN_VERSION_HASH: str +COLUMN_SCHEMA_FINGERPRINT: str +COLUMN_DEFINITION_B64: str +COLUMN_METADATA_JSON_B64: str +COLUMN_TIMESTAMP: str +NODE_TYPE_MODEL: str +NODE_TYPE_SEED: str +AUDIT_GATE_METADATA_KEY: str +CUSTOM_HOST_RUNTIME_VERSION: str +CUSTOM_HOST_LAUNCH_MODULE: str +CUSTOM_RULE_PROJECT_SUBJECT: str +CONTRACT_ENFORCED: str +LAUNCH_RECORD_OPTION: str +ZERO_DAY_CURSOR_DURATION: str +AST_CTE_KIND: str +SQL_TEXT_FIELD: str +SQL_DIGEST_FIELD: str +SQL_PRIVATE_RECORD_FIELD: str +MIN_CUSTOM_RULE_TEST_CASES: str +DEFAULT_PAGE_LIMIT: int +MAX_PAGE_LIMIT: int +CURRENT_EVENT_LOG_SCHEMA_VERSION: int +CURRENT_RUN_STORAGE_SCHEMA_VERSION: int +COMPUTE_LOG_FORMAT_VERSION: int +DEFAULT_RETENTION_COUNT: int +MAX_READ_BYTES: int +CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int +MIGRATION_WRITE_ATTEMPTS: int +CUSTOM_HOST_PROTOCOL_VERSION: int def plan_sql_tests( *, @@ -489,6 +373,21 @@ type NativeSqlTestPlanRow = tuple[ type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] +type ProjectProvider = Provider +type SqlScalar = str | int | bool | float | Decimal | None +type SqlValuePayload = SqlScalar | tuple[object, ...] + +class TransientLineOwner(Protocol): + """A renderer that owns the transient line currently drawn in place on a terminal.""" + + def clear_transient_line(self) -> None: ... + def redraw_transient_line(self) -> None: ... + +class DbtInvoker(Protocol): + """Callable dbt process invocation contract.""" + + def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... + class FingerprintWriteProgress(Protocol): def __call__(self, *, completed: int, total: int) -> None: ... @@ -726,7 +625,7 @@ class CompileInputsView(Protocol): @property def declaration_scope(self) -> DeclarationScopeBuild | None: ... @property - def native_project(self) -> _native.NativeCompiledProject | None: ... + def native_project(self) -> NativeCompiledProject | None: ... class ModelInput(Protocol): """One model's compile input: a native owner view, or a hand-built `CompileModelInput`.""" @@ -836,11 +735,9 @@ class FixtureColumnInferences: ) -> FixtureColumnInference | None: """Return the inference for one fixture query, computing each distinct query once.""" -class NativeSqlTestArtifact: +class NativeSqlTestArtifact(NativeRecord): """One native-planned SQL-test artifact and its distinct planning error messages.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - sql: str model_names: tuple[str, ...] error_messages: tuple[str, ...] @@ -849,21 +746,17 @@ class NativeSqlTestArtifact: ) -> None: ... def __replace__(self, **changes: Any) -> NativeSqlTestArtifact: ... -class RelationFixtureDiagnostic: +class RelationFixtureDiagnostic(NativeRecord): """One statically provable invalid fixture condition.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - key: FixtureKey message: str def __init__(self, key: FixtureKey, message: str) -> None: ... def __replace__(self, **changes: Any) -> RelationFixtureDiagnostic: ... -class RelationFixtureCompletion: +class RelationFixtureCompletion(NativeRecord): """Completed fixture SQL plus metadata reused by SQL-test validation.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - fixture_sql_by_key: dict[FixtureKey, str] inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] expected_types: Mapping[FixtureKey, dict[str, str]] @@ -877,22 +770,18 @@ class RelationFixtureCompletion: ) -> None: ... def __replace__(self, **changes: Any) -> RelationFixtureCompletion: ... -class FixtureColumnMetadata: +class FixtureColumnMetadata(NativeRecord): """Known type and nullability for one relation fixture column.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - name: str type: str | None nullable: bool | None def __init__(self, name: str, type: str | None, nullable: bool | None) -> None: ... def __replace__(self, **changes: Any) -> FixtureColumnMetadata: ... -class FixtureRelationMetadata: +class FixtureRelationMetadata(NativeRecord): """Known columns and whether they form an authoritative relation shape.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - columns: tuple[FixtureColumnMetadata, ...] authoritative_names: bool def __init__( @@ -900,11 +789,9 @@ class FixtureRelationMetadata: ) -> None: ... def __replace__(self, **changes: Any) -> FixtureRelationMetadata: ... -class RelationFixturePlanningContext: +class RelationFixturePlanningContext(NativeRecord): """Project-wide fixture metadata shared across test and scenario planning.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models_by_name: dict[str, CompiledModelView] relations: dict[FixtureKey, FixtureRelationMetadata] authoritative_columns: dict[FixtureKey, frozenset[str]] @@ -920,33 +807,27 @@ class RelationFixturePlanningContext: ) -> None: ... def __replace__(self, **changes: Any) -> RelationFixturePlanningContext: ... -class SelectorExpansion: +class SelectorExpansion(NativeRecord): """One selector split into graph expansion flags and core text.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - core: str upstream: bool downstream: bool def __init__(self, core: str, upstream: bool = ..., downstream: bool = ...) -> None: ... def __replace__(self, **changes: Any) -> SelectorExpansion: ... -class ParsedScenarioArtifactName: +class ParsedScenarioArtifactName(NativeRecord): """Parsed physical name for one planner-owned scenario artifact.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - hash_prefix: str kind: str logical_name: str def __init__(self, hash_prefix: str, kind: str, logical_name: str) -> None: ... def __replace__(self, **changes: Any) -> ParsedScenarioArtifactName: ... -class MissingUpstream: +class MissingUpstream(NativeRecord): """One upstream dependency missing from both scope and warehouse.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - key: CompiledObjectKey required_by: tuple[CompiledObjectKey, ...] def __init__( @@ -954,11 +835,9 @@ class MissingUpstream: ) -> None: ... def __replace__(self, **changes: Any) -> MissingUpstream: ... -class WarehouseFingerprints: +class WarehouseFingerprints(NativeRecord): """Latest direct fingerprints grouped by node type.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models: dict[str, Fingerprint] functions: dict[str, Fingerprint] seeds: dict[str, Fingerprint] @@ -976,11 +855,9 @@ class WarehouseFingerprints: ) -> None: ... def __replace__(self, **changes: Any) -> WarehouseFingerprints: ... -class CursorSnapshotScope: +class CursorSnapshotScope(NativeRecord): """Execution selection used for cursor validation and runtime producer resolution.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_keys: frozenset[CompiledObjectKey] runtime_producer_keys: frozenset[CompiledObjectKey] invocation_time: datetime | None @@ -996,11 +873,9 @@ class CursorSnapshotScope: ) -> None: ... def __replace__(self, **changes: Any) -> CursorSnapshotScope: ... -class ReferenceRename: +class ReferenceRename(NativeRecord): """One model rename that downstream references may have been rewritten for.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - new_name: str origin_name: str recorded_at: datetime | None @@ -1009,11 +884,9 @@ class ReferenceRename: ) -> None: ... def __replace__(self, **changes: Any) -> ReferenceRename: ... -class WarehouseSnapshot: +class WarehouseSnapshot(NativeRecord): """Frozen point-in-time picture of warehouse state for planning.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - existing_relations: dict[str, RelationInfo] existing_columns: dict[str, tuple[ColumnInfo, ...]] fingerprints: WarehouseFingerprints @@ -1041,21 +914,17 @@ class WarehouseSnapshot: ) -> None: ... def __replace__(self, **changes: Any) -> WarehouseSnapshot: ... -class SelectionStalenessNodeKey: +class SelectionStalenessNodeKey(NativeRecord): """Neutral node identity for shared selection-aware staleness classification.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - resource_type: str name: str def __init__(self, resource_type: str, name: str) -> None: ... def __replace__(self, **changes: Any) -> SelectionStalenessNodeKey: ... -class SelectionStalenessGraph: +class SelectionStalenessGraph(NativeRecord): """Neutral graph inputs for selected-model stale warning classification.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - upstream_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]] selected_model_names: frozenset[str] run_model_names: frozenset[str] @@ -1077,31 +946,25 @@ class SelectionStalenessGraph: ) -> None: ... def __replace__(self, **changes: Any) -> SelectionStalenessGraph: ... -class SelectionStalenessWarning: +class SelectionStalenessWarning(NativeRecord): """Neutral stale warning classification for one selected model.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str trigger_names: tuple[str, ...] def __init__(self, model_name: str, trigger_names: tuple[str, ...]) -> None: ... def __replace__(self, **changes: Any) -> SelectionStalenessWarning: ... -class BackfillResult: +class BackfillResult(NativeRecord): """Resolved backfill action from a change detection policy.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - action: BackfillAction duration: str | None def __init__(self, action: BackfillAction, duration: str | None = ...) -> None: ... def __replace__(self, **changes: Any) -> BackfillResult: ... -class ChangeDetectionResult: +class ChangeDetectionResult(NativeRecord): """Per-model output from change detection and policy resolution.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str change_kind: ChangeKind query_changed: bool @@ -1131,11 +994,9 @@ class ChangeDetectionResult: ) -> None: ... def __replace__(self, **changes: Any) -> ChangeDetectionResult: ... -class UnitTestSelectorSplit: +class UnitTestSelectorSplit(NativeRecord): """Resource selectors left after unit-test selectors moved into a test selection.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - select: tuple[str, ...] exclude: tuple[str, ...] sql_test_selection: SqlTestSelection @@ -1147,11 +1008,9 @@ class UnitTestSelectorSplit: ) -> None: ... def __replace__(self, **changes: Any) -> UnitTestSelectorSplit: ... -class ModelPlanContext: +class ModelPlanContext(NativeRecord): """Relation lookups and source columns for building one model plan entry.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_locations: dict[str, CompiledRelationLocation] models_by_name: dict[str, CompiledModelView] seed_locations: dict[str, CompiledRelationLocation] @@ -1181,11 +1040,9 @@ class ModelPlanContext: ) -> None: ... def __replace__(self, **changes: Any) -> ModelPlanContext: ... -class CursorOverridePair: +class CursorOverridePair(NativeRecord): """Resolved per-model cursor start/end overrides for plan entry construction.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - start_cursor_override: str | None end_cursor_override: str | None def __init__( @@ -1193,11 +1050,9 @@ class CursorOverridePair: ) -> None: ... def __replace__(self, **changes: Any) -> CursorOverridePair: ... -class PlanEntryBuildInputs: +class PlanEntryBuildInputs(NativeRecord): """Blocked models and cursor overrides for plan entry building.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - source_freshness_blocked_model_names: frozenset[str] external_blocked_model_names: frozenset[str] start_cursor_override: str | None @@ -1223,21 +1078,17 @@ class PlanEntryBuildInputs: ) -> None: ... def __replace__(self, **changes: Any) -> PlanEntryBuildInputs: ... -class FunctionChangeResult: +class FunctionChangeResult(NativeRecord): """Per-function output from change detection.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - fingerprint_sql: str reason: PlanReason def __init__(self, fingerprint_sql: str, reason: PlanReason = ...) -> None: ... def __replace__(self, **changes: Any) -> FunctionChangeResult: ... -class PlannerChangeResults: +class PlannerChangeResults(NativeRecord): """Change detection output for selected planner resources.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models: dict[str, ChangeDetectionResult] functions: dict[str, FunctionChangeResult] def __init__( @@ -1245,30 +1096,24 @@ class PlannerChangeResults: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerChangeResults: ... -class ResolvedModelAction: +class ResolvedModelAction(NativeRecord): """Effective model change and backfill decided from the model's own change.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - change: ChangeDetectionResult backfill: BackfillResult def __init__(self, change: ChangeDetectionResult, backfill: BackfillResult) -> None: ... def __replace__(self, **changes: Any) -> ResolvedModelAction: ... -class PlannerResolvedActions: +class PlannerResolvedActions(NativeRecord): """Per-model planning decisions keyed by model name.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - models: dict[str, ResolvedModelAction] def __init__(self, models: dict[str, ResolvedModelAction]) -> None: ... def __replace__(self, **changes: Any) -> PlannerResolvedActions: ... -class DirectModelVersionIdentities: +class DirectModelVersionIdentities(NativeRecord): """Current direct model version identity values by model name.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - function_local_hashes: dict[str, str] seed_version_hashes: dict[str, str] seed_metadata_jsons: dict[str, str] @@ -1286,11 +1131,9 @@ class DirectModelVersionIdentities: ) -> None: ... def __replace__(self, **changes: Any) -> DirectModelVersionIdentities: ... -class PlannerModelEntryResults: +class PlannerModelEntryResults(NativeRecord): """Model plan-entry phase output.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - entries: tuple[ModelPlanEntry, ...] warnings: tuple[PlanWarning, ...] def __init__( @@ -1298,11 +1141,9 @@ class PlannerModelEntryResults: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerModelEntryResults: ... -class SchemaAction: +class SchemaAction(NativeRecord): """One concrete schema change action to apply to a target table.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - kind: SchemaActionKind column_name: str column_type: str | None @@ -1311,11 +1152,9 @@ class SchemaAction: ) -> None: ... def __replace__(self, **changes: Any) -> SchemaAction: ... -class PlanWarning: +class PlanWarning(NativeRecord): """One warning or error produced during plan resolution.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str | None severity: WarningSeverity message: str @@ -1329,11 +1168,9 @@ class PlanWarning: ) -> None: ... def __replace__(self, **changes: Any) -> PlanWarning: ... -class ColumnRenameHint: +class ColumnRenameHint(NativeRecord): """A removed column that an added column may have been renamed from.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str added_column: str candidate_columns: tuple[str, ...] @@ -1350,11 +1187,9 @@ class ColumnRenameHint: def message(self) -> str: """The one-line hint shown beside the added column.""" -class QueryProjection: +class QueryProjection(NativeRecord): """One named top-level output column of a query and its canonical expression.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - name: str expression: str references: frozenset[str] @@ -1367,11 +1202,9 @@ class QueryProjection: def key(self) -> str: """The case-insensitive output name.""" -class QueryShape: +class QueryShape(NativeRecord): """Canonical top-level projections and the rest of one query, ignoring formatting.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - projections: tuple[QueryProjection, ...] body: str alias_clauses: str @@ -1391,11 +1224,9 @@ class QueryShape: ) -> None: ... def __replace__(self, **changes: Any) -> QueryShape: ... -class PlanOutputExtras: +class PlanOutputExtras(NativeRecord): """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - seed_version_hashes: dict[str, str] | None seed_metadata_jsons: dict[str, str] | None seed_plan_reasons: dict[str, PlanReason] | None @@ -1409,11 +1240,9 @@ class PlanOutputExtras: ) -> None: ... def __replace__(self, **changes: Any) -> PlanOutputExtras: ... -class PlannedSqlTests: +class PlannedSqlTests(NativeRecord): """SQL test entries planned for one selection; reusable only for the same selection.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - selected_keys: frozenset[CompiledObjectKey] sql_test_selection: SqlTestSelection entries: tuple[SqlTestPlanEntry, ...] @@ -1427,11 +1256,9 @@ class PlannedSqlTests: ) -> None: ... def __replace__(self, **changes: Any) -> PlannedSqlTests: ... -class NativeSqlTestPlan: +class NativeSqlTestPlan(NativeRecord): """Native chain and assertion steps for one SQL test, with SQL when rendered.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - chain: tuple[ChainStep, ...] assertions: tuple[SqlTestAssertionStep, ...] model_names: tuple[str, ...] @@ -1449,11 +1276,9 @@ class NativeSqlTestPlan: ) -> None: ... def __replace__(self, **changes: Any) -> NativeSqlTestPlan: ... -class SqlTestPlanResult: +class SqlTestPlanResult(NativeRecord): """One SQL test's plan entry and warnings, or the fixture diagnostics preventing it.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - entry: SqlTestPlanEntry | None warnings: tuple[PlanWarning, ...] fixture_diagnostics: tuple[str, ...] @@ -1465,11 +1290,9 @@ class SqlTestPlanResult: ) -> None: ... def __replace__(self, **changes: Any) -> SqlTestPlanResult: ... -class ScenarioGraphPlan: +class ScenarioGraphPlan(NativeRecord): """Inferred graph slice and fixture boundaries for one SQL scenario.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - key: CompiledObjectKey name: str target_model_names: tuple[str, ...] @@ -1497,31 +1320,25 @@ class ScenarioGraphPlan: ) -> None: ... def __replace__(self, **changes: Any) -> ScenarioGraphPlan: ... -class ScenarioArtifactIdentity: +class ScenarioArtifactIdentity(NativeRecord): """Logical identity for one scenario-owned physical artifact.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - kind: ScenarioArtifactKind | str logical_name: str def __init__(self, kind: ScenarioArtifactKind | str, logical_name: str) -> None: ... def __replace__(self, **changes: Any) -> ScenarioArtifactIdentity: ... -class ScenarioArtifactName: +class ScenarioArtifactName(NativeRecord): """Resolved physical relation name for one scenario artifact.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - identity: ScenarioArtifactIdentity physical_name: str def __init__(self, identity: ScenarioArtifactIdentity, physical_name: str) -> None: ... def __replace__(self, **changes: Any) -> ScenarioArtifactName: ... -class PlanProviderUsage: +class PlanProviderUsage(NativeRecord): """Provider usage metadata scoped to planned work.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - provider_name: str consumer_kind: str consumer_name: str @@ -1539,11 +1356,9 @@ class PlanProviderUsage: ) -> None: ... def __replace__(self, **changes: Any) -> PlanProviderUsage: ... -class PlannerRuntime: +class PlannerRuntime(NativeRecord): """Resolved execution environment shared by planner phases.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - project: CompiledProjectView adapter: BaseAdapter connection: Any @@ -1563,11 +1378,9 @@ class PlannerRuntime: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerRuntime: ... -class PlannerScopeResolution: +class PlannerScopeResolution(NativeRecord): """Resolved planner scopes for one plan build.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - selected_scope: PlannerScope stale_warning_scope: PlannerScope inspection_scope: PlannerScope @@ -1579,11 +1392,9 @@ class PlannerScopeResolution: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... -class PlannerWarehouseState: +class PlannerWarehouseState(NativeRecord): """Warehouse snapshot and inspection relations gathered once per plan.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - snapshot: WarehouseSnapshot inspection_relations: PlannerRelationsContext migration_entries: tuple[ModelMigrationPlanEntry, ...] @@ -1607,11 +1418,9 @@ class PlannerWarehouseState: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerWarehouseState: ... -class ModelMigrationRequest: +class ModelMigrationRequest(NativeRecord): """One destination model whose data should come from an earlier relation.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model: CompiledModelView discovery: MigrationDiscovery raw_origin: str | None @@ -1631,11 +1440,9 @@ class ModelMigrationRequest: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationRequest: ... -class ModelMigrationDeclaration: +class ModelMigrationDeclaration(NativeRecord): """One project model's explicit migrate_from, resolved to a relation.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_name: str origin_location: CompiledRelationLocation origin_model: str | None @@ -1644,11 +1451,9 @@ class ModelMigrationDeclaration: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationDeclaration: ... -class ModelMigrationDiscovery: +class ModelMigrationDiscovery(NativeRecord): """Manual and automatically discovered migration requests for one plan.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - requests: tuple[ModelMigrationRequest, ...] warnings: tuple[PlanWarning, ...] destination_fingerprints: dict[str, tuple[str, ...]] @@ -1660,21 +1465,17 @@ class ModelMigrationDiscovery: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationDiscovery: ... -class MigrationCompatibilityResult: +class MigrationCompatibilityResult(NativeRecord): """Compatibility status plus human-readable blocking findings.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - status: MigrationCompatibility findings: tuple[str, ...] def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... -class ModelMigrationPlanning: +class ModelMigrationPlanning(NativeRecord): """Migration decisions plus the warehouse snapshot as it will look after moves.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - snapshot: WarehouseSnapshot entries: tuple[ModelMigrationPlanEntry, ...] warnings: tuple[PlanWarning, ...] @@ -1686,11 +1487,9 @@ class ModelMigrationPlanning: ) -> None: ... def __replace__(self, **changes: Any) -> ModelMigrationPlanning: ... -class OldNameViewPlanning: +class OldNameViewPlanning(NativeRecord): """Old-name steps for this build plus every compatibility view still recorded.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - entries: tuple[OldNameViewPlanEntry, ...] views: tuple[OldNameView, ...] warnings: tuple[PlanWarning, ...] @@ -1702,11 +1501,9 @@ class OldNameViewPlanning: ) -> None: ... def __replace__(self, **changes: Any) -> OldNameViewPlanning: ... -class ColumnMigrationPlanning: +class ColumnMigrationPlanning(NativeRecord): """Column rename decisions plus the warehouse snapshot as it will look after renames.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - snapshot: WarehouseSnapshot entries: tuple[ColumnMigrationPlanEntry, ...] hints: tuple[ColumnRenameHint, ...] @@ -1720,11 +1517,9 @@ class ColumnMigrationPlanning: ) -> None: ... def __replace__(self, **changes: Any) -> ColumnMigrationPlanning: ... -class PlannerIdentityContext: +class PlannerIdentityContext(NativeRecord): """Expected version identities for inspection and stale-warning scopes.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - version_identities: DirectModelVersionIdentities stale_warning_identities: DirectModelVersionIdentities def __init__( @@ -1734,11 +1529,9 @@ class PlannerIdentityContext: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... -class PlannerScopePruningResult: +class PlannerScopePruningResult(NativeRecord): """Scope pruning outcome with pruned scopes, actions, and staleness state.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - inspection_scope: PlannerScope execution_scope: PlannerScope resolved_actions: PlannerResolvedActions @@ -1754,11 +1547,9 @@ class PlannerScopePruningResult: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerScopePruningResult: ... -class PlannerChangeReconciliation: +class PlannerChangeReconciliation(NativeRecord): """Write-hash-honest changes merged back into resolved actions.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - changes: PlannerChangeResults resolved_actions: PlannerResolvedActions def __init__( @@ -1766,11 +1557,9 @@ class PlannerChangeReconciliation: ) -> None: ... def __replace__(self, **changes: Any) -> PlannerChangeReconciliation: ... -class PlannerEntryResults: +class PlannerEntryResults(NativeRecord): """Model plan entries for one plan build.""" - __dataclass_fields__: ClassVar[dict[str, Any]] - model_entry_results: PlannerModelEntryResults def __init__(self, model_entry_results: PlannerModelEntryResults) -> None: ... def __replace__(self, **changes: Any) -> PlannerEntryResults: ... @@ -3121,6 +2910,8 @@ def validate_preview_target( def normalize_dbt_profile_connection( *, resolved: ResolvedDbtProfileOutput ) -> NormalizedDbtProfileConnection: ... +def coerce_dbt_profile_bool(value: object) -> bool: ... +def coerce_dbt_profile_number(value: object) -> int | float: ... def build_dbt_init_project_toml( *, project_name: str, @@ -4201,6 +3992,25 @@ def format_seed_execution_json( *, results: tuple[SeedExecutionResult, ...], plan: PlanOutput ) -> str: ... def format_load_execution_json(*, results: tuple[LoadExecutionResult, ...]) -> str: ... +def build_ingestr_command( + *, + source_entry: SourceEntry, + adapter_name: str, + connection_config: dict[str, object], + destination_name: str, + vars: dict[str, object], + environment: str | None, + run_id: str, + is_reload: bool, +) -> tuple[str, ...]: ... +def format_ingestr_command(command: Sequence[str], /) -> str: ... +def dlt_destination_arguments( + *, + adapter_name: str, + connection_config: Mapping[str, object], + destination_config: Mapping[str, object] | None, + dataset_name: str | None, +) -> tuple[str, dict[str, Any], dict[str, Any], str | None]: ... def format_clone_execution_json( *, result: CloneExecutionResult, resource_types_by_name: Mapping[str, str] ) -> str: ... @@ -5233,6 +5043,111 @@ def normalize_sql_value( limits: SqlValueLimits | None = None, ) -> SqlValue: ... def render_state_sql_literal(*, value: object | None, declared_type: StateSqlValueType) -> str: ... +def format_diagnostics_console_record(record: logging.LogRecord, *, use_color: bool) -> str: ... +def interpreter_identity() -> tuple[str, str, str, str]: ... +def project_microbatch_coverage( + *, + events: tuple[MicrobatchEvent, ...], + expected_intervals: tuple[MicrobatchInterval, ...], + cursor_type: str, +) -> MicrobatchCoverageProjection: ... +def project_replay_requirement( + *, + requirement: MicrobatchEvent, + current_model_version_hash: str, + expected_intervals: tuple[MicrobatchInterval, ...], + coverage: MicrobatchCoverageProjection, + cursor_type: str, +) -> ReplayRequirementProjection: ... +def latest_active_replay_requirement( + *, events: tuple[MicrobatchEvent, ...], current_model_version_hash: str +) -> MicrobatchEvent | None: ... +def deterministic_microbatch_event_id( + *, + scope: MicrobatchScope, + record_type: MicrobatchRecordType, + partition_start: str | None, + partition_end: str | None, + completion_reason: str, +) -> str: ... +def build_microbatch_insert_many_sql( + *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_microbatch_existing_event_ids_sql( + *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_microbatch_read_scope_sql( + *, scope: MicrobatchScope, render_qualified_name: Callable[..., str | None] +) -> str: ... +def build_microbatch_create_table_sql( + *, + database: str | None, + schema: str, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def build_microbatch_create_index_sqls( + *, + database: str | None, + schema: str, + render_identifier: Callable[[str], str], + render_qualified_name: Callable[..., str | None], +) -> tuple[str, ...]: ... + +CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: int +CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int +LIFECYCLE_EVENT_SCHEMA_VERSIONS: frozenset[int] +HOOK_PHASES: frozenset[str] +HOOK_TYPES: frozenset[str] +OPERATION_ADAPTERS: frozenset[str] +OPERATION_KINDS: frozenset[str] +OPERATION_METADATA_FIELDS: frozenset[str] +OPERATION_NAMES: frozenset[str] +OPERATION_PHASES: frozenset[str] +OPERATION_SCOPES: frozenset[str] +OPERATION_STRATEGIES: frozenset[str] +OPERATION_TARGET_KINDS: frozenset[str] +RESOURCE_SKIP_CODES: frozenset[str] +RESOURCE_SKIP_MODES: frozenset[str] + +def lifecycle_event_catalogued(*, schema_version: object, event_type: object) -> bool: ... +def lifecycle_event_terminal(*, schema_version: int, event_type: str) -> bool: ... +def validate_lifecycle_schema_version(*, value: object) -> None: ... +def freeze_lifecycle_json(*, value: object, path: str) -> JSONValue: ... +def validate_lifecycle_required_text(*, value: object, field_name: str) -> None: ... +def validate_lifecycle_optional_text(*, value: object, field_name: str) -> None: ... +def validate_lifecycle_timestamp(*, value: object) -> None: ... +def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None: ... +def validate_lifecycle_event(*, event: LifecycleEvent) -> None: ... +def validate_lifecycle_invocation_sequence(*, value: object) -> None: ... +def validate_diagnostic_log(*, record: DiagnosticLog) -> None: ... +def validate_run_bounds( + *, selected_count: int, configured_concurrency: int, worker_count: int +) -> None: ... +def validated_operation_metadata(*, metadata: Mapping[str, int] | None) -> dict[str, int]: ... +def validated_operation_hook_fields( + *, + hook_phase: str | None, + hook_index: int | None, + hook_type: str | None, + hook_name: str | None, +) -> dict[str, JSONValue]: ... +def validate_lifecycle_producer(*, producer: str, producer_version: str | None) -> None: ... +def render_state_table_create_sql( + *, + qualified_name: str, + columns: tuple[str, ...], + column_types: Mapping[str, StateSqlValueType], + required_columns: frozenset[str], + render_framework_type: Callable[[FrameworkType], str], +) -> str: ... +def render_insert_source_freshness_records_sql( + *, + database: str | None, + schema: str, + records: tuple[SourceFreshnessRecord, ...], + render_qualified_name: Callable[..., str | None], +) -> str: ... def model_header_declarations( header: tuple[object | None, object | None], model_name: str, relative_path: Path, / ) -> tuple[tuple[object, ...], tuple[object, ...]]: ... @@ -5458,9 +5373,18 @@ def aggregate_build_result( source_audit_results: tuple[AuditExecutionResult, ...], end_audit_results: tuple[AuditExecutionResult, ...], ) -> BuildExecutionResult: ... -def resolve_dag_selection( - *, dag: Mapping[str, Any], select: str, exclude: str | None = None -) -> frozenset[str]: ... +def dagster_asset_plans( + dag: Mapping[str, Any], / +) -> list[tuple[dict[str, Any], list[dict[str, Any]], set[str]]]: ... +def dagster_check_plans( + dag: Mapping[str, Any], /, *, include_scenarios: bool, only_scenarios: bool +) -> list[tuple[Mapping[str, Any], bool, list[Mapping[str, Any]], str | None]]: ... +def dagster_selected_asset_nodes( + dag: Mapping[str, Any], /, *, select: str, exclude: str | None = None +) -> list[dict[str, Any]]: ... +def rivers_asset_plans( + dag: Mapping[str, Any], / +) -> list[tuple[dict[str, Any], list[Mapping[str, Any]]]]: ... def load_sqlbuild_dag( dag: Mapping[str, Any] | str | Path, *, error_class: type[Exception] ) -> Mapping[str, Any]: ... @@ -5585,6 +5509,9 @@ def apply_deferred_locations( selected_keys: frozenset[CompiledObjectKey], ) -> tuple[dict[str, CompiledRelationLocation], dict[str, CompiledRelationLocation]]: ... def substitute_placeholder_defaults(sql: str, placeholders: list[tuple[str, str]], /) -> str: ... +def statement_metadata_effect(sql: str, /) -> StatementMetadataEffect: ... +def relation_name_key(name: str, /) -> str: ... +def qualified_relation_name_key(qualified: str, /) -> str | None: ... def fixture_projection_facts( sql: str, dialect: str | None, / ) -> tuple[list[tuple[str, str]], list[str], list[str]]: ... @@ -5705,6 +5632,15 @@ def digest_files(paths: list[str]) -> list[str | None]: ... def normalize_type( type_sql: str, dialect: str ) -> tuple[tuple[str, str, str | None, str | None, str | None], str | None]: ... +def infer_first_arg_nullability( + args: Sequence[InferredNullability | str], +) -> InferredNullability: ... +def infer_conditional_result_nullability( + args: Sequence[InferredNullability | str], +) -> InferredNullability: ... +def infer_safe_cast_nullability( + args: Sequence[InferredNullability | str], +) -> InferredNullability: ... # Native analysis: model analysis session. type _AnalysisDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] @@ -5853,6 +5789,22 @@ def parse_audit_instances( error_class: type[Exception], null_as_empty: bool, ) -> tuple[SchemaAuditInstance, ...]: ... +def retry_policy_exceptions( + *, + max_attempts: int, + retry_on: type[BaseException] | Iterable[type[BaseException]], + initial_delay_seconds: float, + backoff_multiplier: float, + max_delay_seconds: float | None, + max_elapsed_seconds: float | None, +) -> tuple[type[BaseException], ...]: ... +def retry_delay_seconds(*, retry_policy: RetryPolicy, retry_index: int) -> float: ... +def validate_audit_case( + *, name: str, definition: str, minimum_samples: int | None, evidence_limit: int | None +) -> None: ... +def normalize_hook_reads( + value: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef], +) -> tuple[SqlResourceRef, ...]: ... def optional_named_string( *, raw_value: object | None, @@ -6353,6 +6305,13 @@ def dagster_failure( def dagster_live_failure( *, asset: IntegrationAssetResult, envelope: IntegrationResultEnvelope, command: list[str] ) -> tuple[list[str], dict[str, Any]]: ... +def dagster_default_asset_key_path(node: Mapping[str, Any]) -> list[str]: ... +def dagster_default_group_name(node: Mapping[str, Any]) -> str: ... +def dagster_default_tags(node: Mapping[str, Any]) -> dict[str, str]: ... +def dagster_default_metadata(node: Mapping[str, Any]) -> dict[str, Any]: ... +def dagster_default_description(node: Mapping[str, Any]) -> str: ... +def dagster_default_check_name(check: Mapping[str, Any]) -> str: ... +def dagster_default_check_metadata(check: Mapping[str, Any]) -> dict[str, Any]: ... def build_janitor_plan( *, project: CompiledProjectView, @@ -6372,6 +6331,8 @@ def build_janitor_plan( def build_janitor_archive_name( *, original_name: str, archived_at: datetime, identifier_limit: int, kind: str | None = None ) -> str: ... +def build_query_diff_artifact_name(*, run_id: str, side: str) -> str: ... +def parse_query_diff_artifact_name(name: str, /) -> tuple[str, str] | None: ... def render_janitor_completion( *, result: JanitorExecutionResult, use_color: bool = False ) -> str: ... @@ -6686,6 +6647,14 @@ def write_source_freshness_records( records: tuple[SourceFreshnessRecord, ...], renderers: SourceFreshnessRenderers, ) -> None: ... +def append_eligible_direct_source_freshness_records( + *, + plan: PlanOutput, + result: object, + adapter: object, + connection_config: dict[str, object], + run_id: str, +) -> None: ... def observe_configured_source_freshness( *, adapter: StrictAdapter, connection: object, source: SourceEntry, observed_at: datetime ) -> SourceFreshnessObservation: ... @@ -6717,6 +6686,32 @@ def build_direct_source_freshness_propagation_result( ) -> DirectSourceFreshnessPropagationResult: ... def redact_diagnostic_text(value: str) -> str: ... def redact_diagnostic_value(*, name: str, value: object) -> object: ... +def sql_log_record( + *, + context: Mapping[str, object] | None, + identity: Mapping[str, str | None] | None, + sql: str, + action: str, + intent: str | None, + query_id: str | None, +) -> tuple[str, dict[str, object]]: ... +def debug_event_record_fields( + *, + context: Mapping[str, object] | None, + event_fields: Mapping[str, object], + identity: Mapping[str, str | None] | None, +) -> dict[str, object]: ... +def route_diagnostic_log_record( + *, + record: logging.LogRecord, + internal_only: bool, + allow_internal: bool, + include_sql_text: bool, +) -> dict[str, object] | None: ... +def safe_diagnostic_extra_fields(record: logging.LogRecord) -> dict[str, object]: ... +def compute_diagnostic_log_fields( + *, record: logging.LogRecord, invocation_id: str, include_sql_text: bool +) -> dict[str, object] | None: ... def sql_rule_lint_key(*, config: LintConfig) -> str: ... def sql_model_rule_keys( *, lint_key: str, codes: list[str], models: list[tuple[str, str, str, str | None]] @@ -7043,6 +7038,31 @@ def get_config_string_tuple( ) -> tuple[str, ...] | None: ... def get_config_cursor_bound(*, values: Mapping[str, object], key: str) -> str | None: ... def custom_rule_import_closure(*, rule: Rule, project_dir: Path) -> tuple[Path, ...]: ... +def allowed_custom_rule_import(module: str) -> bool: ... +def verify_custom_rules(*, rules: Iterable[Rule], project_dir: Path) -> None: ... +def finding_position( + *, path: Path, line: int | None, column: int | None, node: object +) -> tuple[int, int]: ... +def rule_option_toml_value(value: object) -> str: ... +def validate_project_relative(path: str) -> None: ... +def project_tree_paths(root: Path) -> tuple[ProjectPath, ...]: ... +def project_tree_file_path(root: Path, path: str) -> str: ... +def project_tree_child_nodes( + nodes: Iterable[ProjectPath], path: str | PurePath +) -> tuple[ProjectPath, ...]: ... +def project_tree_descendant_nodes( + nodes: Iterable[ProjectPath], path: str | PurePath +) -> tuple[ProjectPath, ...]: ... +def project_tree_glob_nodes( + nodes: Iterable[ProjectPath], pattern: str +) -> tuple[ProjectPath, ...]: ... +def tracked_text_file_values(nodes: Iterable[ProjectPath]) -> tuple[str, ...]: ... +def project_tree_relative_parts( + *, root: Path, path: str | PurePath, under: str | PurePath +) -> tuple[str, ...]: ... +def startup_input_stamps( + *, directories: Iterable[str], files: Iterable[str], prefixes: Iterable[str] +) -> list[tuple[str, int | None, int | None]]: ... def custom_rule_implementation_fingerprint( *, rule: Rule, project_dir: Path, import_closure: tuple[Path, ...] | None = None ) -> str: ... @@ -11395,7 +11415,7 @@ class DeclarationScopeResolver(NativeRecord): project_dir: Path | None lookup: ScopeLookup projection: DeclarationRuntimeProjection - native_contexts: _native.NativeDeclarationContexts + native_contexts: NativeDeclarationContexts resource_specific: frozenset[ResourceIdentity] | None contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] def __init__( @@ -11403,7 +11423,7 @@ class DeclarationScopeResolver(NativeRecord): project_dir: Path | None, lookup: ScopeLookup, projection: DeclarationRuntimeProjection, - native_contexts: _native.NativeDeclarationContexts, + native_contexts: NativeDeclarationContexts, resource_specific: frozenset[ResourceIdentity] | None = ..., contexts_by_directory: dict[tuple[str, str], DeclarationResolutionContext] = ..., ) -> None: ... @@ -11745,7 +11765,7 @@ class CompileProjectInputs(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None scope_index: ScopeIndex declaration_scope: DeclarationScopeBuild | None - native_project: _native.NativeCompiledProject | None + native_project: NativeCompiledProject | None def __init__( self, project_config: ProjectConfig, @@ -11775,7 +11795,7 @@ class CompileProjectInputs(NativeRecord): external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., scope_index: ScopeIndex = ..., declaration_scope: DeclarationScopeBuild | None = ..., - native_project: _native.NativeCompiledProject | None = ..., + native_project: NativeCompiledProject | None = ..., ) -> None: ... class CompiledSource(NativeRecord): @@ -12091,14 +12111,14 @@ class NativeModelConfigSession(NativeRecord): """The native config builder, validator, and inherited storage policies for one compile.""" path_defaults: dict[str, dict[str, object]] - builder: _native.NativeModelConfigBuilder - validator: _native.NativeModelValidator + builder: NativeModelConfigBuilder + validator: NativeModelValidator inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]] def __init__( self, path_defaults: dict[str, dict[str, object]], - builder: _native.NativeModelConfigBuilder, - validator: _native.NativeModelValidator, + builder: NativeModelConfigBuilder, + validator: NativeModelValidator, inherited_storage: dict[str | None, tuple[ResolvedTimeTravelRetention, ResolvedTableType]], ) -> None: ... @@ -13362,3 +13382,5226 @@ class CustomRulesOutcome(NativeRecord): custom_ms: int, custom_cpu_ms: int = ..., ) -> None: ... + +class ResourceKind(StrEnum): + """Kinds of authored resources represented in a scope index.""" + + MODEL = "model" + TEST = "test" + SCENARIO = "scenario" + FUNCTION = "function" + SOURCE = "source" + SEED = "seed" + +class DeclarationKind(StrEnum): + """Independent public declaration namespaces.""" + + MACRO = "macro" + ENUM = "enum" + CONSTANT = "constant" + AUDIT = "audit" + SINGULAR_AUDIT = "singular_audit" + SCHEMA = "schema" + SQL_HOOK = "sql_hook" + PYTHON_HOOK = "python_hook" + +class ScopeKind(StrEnum): + """Compiler-enforced declaration visibility tiers.""" + + GLOBAL = "global" + INHERITED = "inherited" + LOCAL = "local" + PRIVATE = "private" + +class OwnershipRootKind(StrEnum): + """How an ownership root participates in authored resource scope.""" + + RESOURCE = "resource" + GLOBAL = "global" + +class UsageKind(StrEnum): + """Why one indexed identity consumes a declaration.""" + + RUNTIME = "runtime" + GENERATED = "generated" + DECLARATION_DEPENDENCY = "declaration_dependency" + +class GrantKind(StrEnum): + """Compiler relationships that grant additional public vocabulary.""" + + EXPECTED_MODEL = "expected_model" + TESTED_MACRO = "tested_macro" + +class VisibilityReason(StrEnum): + """Positive explanations emitted by the canonical resolver.""" + + GLOBAL = "global" + INHERITED_ANCESTOR = "inherited_ancestor" + LOCAL_OWNER = "local_owner" + PRIVATE_OWNER = "private_owner" + EXPECTED_MODEL = "expected_model" + TESTED_MACRO = "tested_macro" + +class InaccessibleReason(StrEnum): + """Stable explanations for declarations outside effective scope.""" + + LOCAL_BOUNDARY = "local_boundary" + SIBLING_SCOPE = "sibling_scope" + DESCENDANT_SCOPE = "descendant_scope" + UNRELATED_SCOPE = "unrelated_scope" + PRIVATE_OWNER = "private_owner" + UNSUPPORTED_RESOURCE_KIND = "unsupported_resource_kind" + +class ScopeDiagnosticSeverity(StrEnum): + """Severity of a stable scope diagnostic.""" + + ERROR = "error" + WARNING = "warning" + +class ScopeDiagnosticCode(StrEnum): + """Stable diagnostic identities shared by future scope consumers.""" + + UNKNOWN_TARGET = "S001" + UNQUALIFIED_TARGET = "S002" + DUPLICATE_DECLARATION = "S003" + INVALID_DECLARATION_NAME = "S004" + RESERVED_DECLARATION_NAME = "S005" + INACCESSIBLE_DECLARATION = "S006" + LOCAL_NEEDED_BY_DESCENDANT = "S007" + OVER_BROAD_INHERITED = "S008" + REQUIRES_GLOBAL_PLACEMENT = "S009" + UNUSED_DECLARATION = "S010" + INVALID_PROSPECTIVE_PATH = "S011" + NESTED_DECLARATION_ROOT = "S012" + INCOMPLETE_USAGE = "S013" + INVALID_CURSOR = "S014" + MACRO_MODULE_IMPORT = "S015" + INVALID_MACRO_DEPENDENCY = "S016" + MACRO_DEPENDENCY_CYCLE = "S017" + DUPLICATE_RESOURCE = "S018" + RESOURCE_PARSE_ERROR = "S019" + DECLARATION_PARSE_ERROR = "S020" + MACRO_PARSE_ERROR = "S021" + RELATIONSHIP_PARSE_ERROR = "S022" + CONFIG_PARSE_ERROR = "S023" + OVER_BROAD_GLOBAL = "S024" + +class ResourceIdentity(NativeRecord): + """Stable kind-qualified identity of one authored resource.""" + + kind: ResourceKind + name: str + def __init__( + self, + kind: ResourceKind, + name: str, + ) -> None: ... + +class DeclarationIdentity(NativeRecord): + """Stable declaration identity, including a private declaration owner.""" + + kind: DeclarationKind + name: str + owner: ResourceIdentity | None + def __init__( + self, + kind: DeclarationKind, + name: str, + owner: ResourceIdentity | None = ..., + ) -> None: ... + +class OwnershipRoot(NativeRecord): + """Normalized project-relative directory that owns resource scope.""" + + path: str + kind: OwnershipRootKind + resource_kind: ResourceKind | None + def __init__( + self, + path: str, + kind: OwnershipRootKind = ..., + resource_kind: ResourceKind | None = ..., + ) -> None: ... + +class ResourceRecord(NativeRecord): + """One resource participating in lexical declaration scope.""" + + identity: ResourceIdentity + path: str + ownership_root: OwnershipRoot + def __init__( + self, + identity: ResourceIdentity, + path: str, + ownership_root: OwnershipRoot, + ) -> None: ... + +class MacroMetadata(NativeRecord): + """Safe macro metadata required by scope and dependency inspection.""" + + parameters: tuple[str, ...] + dependencies: tuple[DeclarationIdentity, ...] + source_digest: str + def __init__( + self, + parameters: tuple[str, ...] = ..., + dependencies: tuple[DeclarationIdentity, ...] = ..., + source_digest: str = ..., + ) -> None: ... + +class EnumMemberMetadata(NativeRecord): + """One value-free authored enum member identity.""" + + name: str + def __init__( + self, + name: str, + ) -> None: ... + +class EnumMetadata(NativeRecord): + """Safe enum metadata containing member names and their shared scalar type.""" + + members: tuple[EnumMemberMetadata, ...] + scalar_type: str + def __init__( + self, + members: tuple[EnumMemberMetadata, ...], + scalar_type: str, + ) -> None: ... + +class ConstantMetadata(NativeRecord): + """Value-free normalized metadata for one typed constant.""" + + logical_type: str + collection_kind: str | None + item_count: int | None + nullable: bool + render_as: str | None + def __init__( + self, + logical_type: str, + collection_kind: str | None = ..., + item_count: int | None = ..., + nullable: bool = ..., + render_as: str | None = ..., + ) -> None: ... + +class DeclarationRecord(NativeRecord): + """One declaration definition and its compiler-owned scope metadata.""" + + identity: DeclarationIdentity + path: str + line: int + column: int + scope: ScopeKind + ownership_root: OwnershipRoot + owning_path: str | None + macro: MacroMetadata | None + enum: EnumMetadata | None + constant: ConstantMetadata | None + def __init__( + self, + identity: DeclarationIdentity, + path: str, + line: int, + column: int, + scope: ScopeKind, + ownership_root: OwnershipRoot, + owning_path: str | None = ..., + macro: MacroMetadata | None = ..., + enum: EnumMetadata | None = ..., + constant: ConstantMetadata | None = ..., + ) -> None: ... + +class UsageRecord(NativeRecord): + """A runtime or declaration dependency edge.""" + + consumer: ResourceIdentity | DeclarationIdentity + declaration: DeclarationIdentity + kind: UsageKind + through: ResourceIdentity | DeclarationIdentity | None + enum_member: str | None + def __init__( + self, + consumer: ResourceIdentity | DeclarationIdentity, + declaration: DeclarationIdentity, + kind: UsageKind = ..., + through: ResourceIdentity | DeclarationIdentity | None = ..., + enum_member: str | None = ..., + ) -> None: ... + +class GrantRecord(NativeRecord): + """A compiler relationship granting a public declaration to a resource.""" + + resource: ResourceIdentity + declaration: DeclarationIdentity + through: ResourceIdentity | DeclarationIdentity + kind: GrantKind + def __init__( + self, + resource: ResourceIdentity, + declaration: DeclarationIdentity, + through: ResourceIdentity | DeclarationIdentity, + kind: GrantKind = ..., + ) -> None: ... + +class VisibilityRecord(NativeRecord): + """A resolved positive visibility fact with stable provenance.""" + + resource: ResourceIdentity + declaration: DeclarationIdentity + reason: VisibilityReason + through: ResourceIdentity | DeclarationIdentity | None + def __init__( + self, + resource: ResourceIdentity, + declaration: DeclarationIdentity, + reason: VisibilityReason, + through: ResourceIdentity | DeclarationIdentity | None = ..., + ) -> None: ... + +class InaccessibleRecord(NativeRecord): + """A resolved negative visibility fact with stable provenance.""" + + resource: ResourceIdentity + declaration: DeclarationIdentity + reason: InaccessibleReason + def __init__( + self, + resource: ResourceIdentity, + declaration: DeclarationIdentity, + reason: InaccessibleReason, + ) -> None: ... + +class ScopeDiagnostic(NativeRecord): + """Stable diagnostic retained even when an index is partial.""" + + code: ScopeDiagnosticCode + message: str + severity: ScopeDiagnosticSeverity + path: str | None + line: int | None + column: int | None + declaration: DeclarationIdentity | None + resource: ResourceIdentity | None + def __init__( + self, + code: ScopeDiagnosticCode, + message: str, + severity: ScopeDiagnosticSeverity = ..., + path: str | None = ..., + line: int | None = ..., + column: int | None = ..., + declaration: DeclarationIdentity | None = ..., + resource: ResourceIdentity | None = ..., + ) -> None: ... + +class ScopeCompleteness(NativeRecord): + """Independent completeness facts for partially invalid projects.""" + + discovery: bool + static_visibility: bool + runtime_usage: bool + relationships: bool + placement: bool + promotion_impact: bool + def __init__( + self, + discovery: bool = ..., + static_visibility: bool = ..., + runtime_usage: bool = ..., + relationships: bool = ..., + placement: bool = ..., + promotion_impact: bool = ..., + ) -> None: ... + @property + def complete(self) -> bool: ... + +class ScopeIndex(NativeRecord): + """Canonical deterministic facts used by every future scope consumer.""" + + ownership_roots: tuple[OwnershipRoot, ...] + resources: tuple[ResourceRecord, ...] + declarations: tuple[DeclarationRecord, ...] + usages: tuple[UsageRecord, ...] + grants: tuple[GrantRecord, ...] + visibility: tuple[VisibilityRecord, ...] + inaccessible: tuple[InaccessibleRecord, ...] + diagnostics: tuple[ScopeDiagnostic, ...] + completeness: ScopeCompleteness + def __init__( + self, + ownership_roots: tuple[OwnershipRoot, ...] = ..., + resources: tuple[ResourceRecord, ...] = ..., + declarations: tuple[DeclarationRecord, ...] = ..., + usages: tuple[UsageRecord, ...] = ..., + grants: tuple[GrantRecord, ...] = ..., + visibility: tuple[VisibilityRecord, ...] = ..., + inaccessible: tuple[InaccessibleRecord, ...] = ..., + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + completeness: ScopeCompleteness = ..., + ) -> None: ... + +class DeclarationVisibilityIndex(NativeRecord): + """Declaration positions grouped by the scope facts that make them visible.""" + + identities: tuple[DeclarationIdentity, ...] + global_positions: tuple[int, ...] + private_positions: Mapping[ResourceIdentity, tuple[int, ...]] + local_positions: Mapping[str, tuple[int, ...]] + inherited_positions: Mapping[str, tuple[int, ...]] + def __init__( + self, + identities: tuple[DeclarationIdentity, ...], + global_positions: tuple[int, ...], + private_positions: Mapping[ResourceIdentity, tuple[int, ...]], + local_positions: Mapping[str, tuple[int, ...]], + inherited_positions: Mapping[str, tuple[int, ...]], + ) -> None: ... + +class ScopeLookup(NativeRecord): + """Immutable indexes over one canonical ``ScopeIndex``.""" + + index: ScopeIndex + resources: Mapping[ResourceIdentity, tuple[ResourceRecord, ...]] + resources_by_path: Mapping[str, tuple[ResourceRecord, ...]] + declarations: Mapping[DeclarationIdentity, tuple[DeclarationRecord, ...]] + usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] + usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] + grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] + visibility_index: DeclarationVisibilityIndex + native: object | None + def __init__( + self, + index: ScopeIndex, + resources: Mapping[ResourceIdentity, tuple[ResourceRecord, ...]], + resources_by_path: Mapping[str, tuple[ResourceRecord, ...]], + declarations: Mapping[DeclarationIdentity, tuple[DeclarationRecord, ...]], + usages_by_consumer: Mapping[ + ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...] + ], + usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]], + grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]], + visibility_index: DeclarationVisibilityIndex, + native: object | None = ..., + ) -> None: ... + +class ScopeTargetQuery(NativeRecord): + """Non-throwing qualified-identity or path lookup result.""" + + value: str | ResourceIdentity | DeclarationIdentity + matches: tuple[ResourceRecord, ...] + declaration_matches: tuple[DeclarationRecord, ...] + def __init__( + self, + value: str | ResourceIdentity | DeclarationIdentity, + matches: tuple[ResourceRecord, ...] = ..., + declaration_matches: tuple[DeclarationRecord, ...] = ..., + ) -> None: ... + @property + def unknown(self) -> bool: ... + +class VisibilityResolution(NativeRecord): + """Complete static visibility classification for one queried target.""" + + target: ScopeTargetQuery + visible: tuple[VisibilityRecord, ...] + inaccessible: tuple[InaccessibleRecord, ...] + def __init__( + self, + target: ScopeTargetQuery, + visible: tuple[VisibilityRecord, ...] = ..., + inaccessible: tuple[InaccessibleRecord, ...] = ..., + ) -> None: ... + +class DeclarationVisibility(NativeRecord): + """Visible facts and inaccessible declaration identities for one queried target.""" + + target: ScopeTargetQuery + visible: tuple[VisibilityRecord, ...] + inaccessible: tuple[DeclarationIdentity, ...] + def __init__( + self, + target: ScopeTargetQuery, + visible: tuple[VisibilityRecord, ...] = ..., + inaccessible: tuple[DeclarationIdentity, ...] = ..., + ) -> None: ... + +class ScopeReportFilters(NativeRecord): + """Orthogonal declaration filters applied in their documented fixed order.""" + + sections: tuple[str, ...] + include_nearby: bool + defined_under: str | None + kinds: tuple[DeclarationKind, ...] + glob: str | None + used_only: bool + dependency_depth: int + sort: str + cursor: str | None + page_size: int + nearby_depth: int + globals: str + def __init__( + self, + sections: tuple[str, ...] = ..., + include_nearby: bool = ..., + defined_under: str | None = ..., + kinds: tuple[DeclarationKind, ...] = ..., + glob: str | None = ..., + used_only: bool = ..., + dependency_depth: int = ..., + sort: str = ..., + cursor: str | None = ..., + page_size: int = ..., + nearby_depth: int = ..., + globals: str = ..., + ) -> None: ... + +class ScopeSourceLocation(NativeRecord): + """Safe project-relative source location.""" + + path: str + line: int + column: int + def __init__( + self, + path: str, + line: int, + column: int, + ) -> None: ... + +class VisibilityProvenance(NativeRecord): + """Why a declaration is visible or inaccessible.""" + + reason: str + through: str | None + def __init__( + self, + reason: str, + through: str | None = ..., + ) -> None: ... + +class DeclarationReport(NativeRecord): + """Value-free declaration projection used by every report section.""" + + identity: str + kind: str + name: str + owner: str | None + definition: SourceLocation + scope: str + owning_path: str | None + visibility: VisibilityProvenance | None + inaccessible_reason: str | None + metadata: tuple[tuple[str, object], ...] + consumers: tuple[str, ...] + dependencies: tuple[str, ...] + grants: tuple[str, ...] + required_scope: str | None + required_path: str | None + promotion_impact: tuple[str, ...] + def __init__( + self, + identity: str, + kind: str, + name: str, + owner: str | None, + definition: SourceLocation, + scope: str, + owning_path: str | None, + visibility: VisibilityProvenance | None = ..., + inaccessible_reason: str | None = ..., + metadata: tuple[tuple[str, object], ...] = ..., + consumers: tuple[str, ...] = ..., + dependencies: tuple[str, ...] = ..., + grants: tuple[str, ...] = ..., + required_scope: str | None = ..., + required_path: str | None = ..., + promotion_impact: tuple[str, ...] = ..., + ) -> None: ... + +class ScopeChainEntry(NativeRecord): + """One exact or inherited lexical scope in deterministic path order.""" + + kind: str + path: str + declaration_count: int + def __init__( + self, + kind: str, + path: str, + declaration_count: int, + ) -> None: ... + +class ScopeSection(NativeRecord): + """Pagination and completeness metadata for one report section.""" + + name: str + total: int + returned: int + collapsed: bool + collapsed_count: int + truncated: bool + complete: bool + next_cursor: str | None + cursor: str | None + page_size: int + def __init__( + self, + name: str, + total: int, + returned: int, + collapsed: bool = ..., + collapsed_count: int = ..., + truncated: bool = ..., + complete: bool = ..., + next_cursor: str | None = ..., + cursor: str | None = ..., + page_size: int = ..., + ) -> None: ... + +class ScopeResourceReport(NativeRecord): + """Resolved existing resource or prospective authored path.""" + + target: str + identity: str | None + path: str | None + prospective: bool + directory: bool + duplicate_count: int + def __init__( + self, + target: str, + identity: str | None, + path: str | None, + prospective: bool = ..., + directory: bool = ..., + duplicate_count: int = ..., + ) -> None: ... + +class DeclarationExplanation(NativeRecord): + """Detailed explanation for one qualified declaration.""" + + declaration: DeclarationReport | None + complete: bool + def __init__( + self, + declaration: DeclarationReport | None, + complete: bool, + ) -> None: ... + +class MovePreview(NativeRecord): + """Pure visibility delta for moving one existing resource.""" + + resource: str + destination: str + new_ownership_root: str | None + retained: tuple[DeclarationReport, ...] + gained: tuple[DeclarationReport, ...] + lost: tuple[DeclarationReport, ...] + invalidated_usages: tuple[str, ...] + private_retained: tuple[DeclarationReport, ...] + relationship_retained: tuple[DeclarationReport, ...] + complete: bool + def __init__( + self, + resource: str, + destination: str, + new_ownership_root: str | None, + retained: tuple[DeclarationReport, ...] = ..., + gained: tuple[DeclarationReport, ...] = ..., + lost: tuple[DeclarationReport, ...] = ..., + invalidated_usages: tuple[str, ...] = ..., + private_retained: tuple[DeclarationReport, ...] = ..., + relationship_retained: tuple[DeclarationReport, ...] = ..., + complete: bool = ..., + ) -> None: ... + +class ScopeFolder(NativeRecord): + """One declaration-definition folder with exact recursive counts.""" + + path: str + name: str + descendant_count: int + child_count: int + used_count: int + kind_counts: tuple[tuple[str, int], ...] + def __init__( + self, + path: str, + name: str, + descendant_count: int, + child_count: int, + used_count: int, + kind_counts: tuple[tuple[str, int], ...], + ) -> None: ... + +class ScopeBrowseResult(NativeRecord): + """Folder-first browse result.""" + + folder: str + folders: tuple[ScopeFolder, ...] + diagnostics: tuple[ScopeDiagnostic, ...] + complete: bool + def __init__( + self, + folder: str, + folders: tuple[ScopeFolder, ...], + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + complete: bool = ..., + ) -> None: ... + +class ScopeListResult(NativeRecord): + """Recursive declaration list beneath one definition folder.""" + + folder: str + declarations: tuple[DeclarationReport, ...] + section: ScopeSection + diagnostics: tuple[ScopeDiagnostic, ...] + def __init__( + self, + folder: str, + declarations: tuple[DeclarationReport, ...], + section: ScopeSection, + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + ) -> None: ... + +class ScopeReport(NativeRecord): + """Canonical payload shared by all non-rendering scope consumers.""" + + resource: ScopeResourceReport + scope_chain: tuple[ScopeChainEntry, ...] + available: tuple[DeclarationReport, ...] + used: tuple[DeclarationReport, ...] + relationship_scope: tuple[DeclarationReport, ...] + nearby_unavailable: tuple[DeclarationReport, ...] + filters: ScopeReportFilters + sections: tuple[ScopeSection, ...] + explanation: DeclarationExplanation | None + move_preview: MovePreview | None + diagnostics: tuple[ScopeDiagnostic, ...] + complete: bool + def __init__( + self, + resource: ScopeResourceReport, + scope_chain: tuple[ScopeChainEntry, ...] = ..., + available: tuple[DeclarationReport, ...] = ..., + used: tuple[DeclarationReport, ...] = ..., + relationship_scope: tuple[DeclarationReport, ...] = ..., + nearby_unavailable: tuple[DeclarationReport, ...] = ..., + filters: ScopeReportFilters = ..., + sections: tuple[ScopeSection, ...] = ..., + explanation: DeclarationExplanation | None = ..., + move_preview: MovePreview | None = ..., + diagnostics: tuple[ScopeDiagnostic, ...] = ..., + complete: bool = ..., + ) -> None: ... + +class EventExportSeverity(StrEnum): + """Severity used to filter exported lifecycle events.""" + + DEBUG = "debug" + INFO = "info" + WARNING = "warning" + ERROR = "error" + +class WarehouseGroup(StrEnum): + """Command group that selects a target's default warehouse.""" + + BUILD = "build" + QUERY = "query" + +class SourceWriteStrategy(StrEnum): + APPEND = "append" + DELETE_INSERT = "delete_insert" + MERGE = "merge" + TABLE = "table" + +class FutureCursorAction(StrEnum): + """Response when an effective cursor lies beyond the future horizon.""" + + CAP = "cap" + ERROR = "error" + +class MicrobatchLimitAction(StrEnum): + """Response when one model plans more than the configured batch limit.""" + + CAP_FROM_END = "cap_from_end" + CAP_FROM_START = "cap_from_start" + ERROR = "error" + WARN = "warn" + +class ColumnContractMode(StrEnum): + """Default contract behavior for model column declarations.""" + + IMPLICIT = "implicit" + EXPLICIT = "explicit" + +class SourceFreshnessStrategy(StrEnum): + ADAPTER = "adapter" + COLUMN = "column" + SQL = "sql" + +class SourceFreshnessValueKind(StrEnum): + TIMESTAMP = "timestamp" + INTEGER = "integer" + STRING = "string" + +class TimeTravelRetentionSource(StrEnum): + """Authored layer that supplied the effective retention policy.""" + + TARGET = "target" + MATERIALIZATION = "materialization" + MODEL = "model" + +class TableType(StrEnum): + """Snowflake managed table type.""" + + PERMANENT = "permanent" + TRANSIENT = "transient" + +class TableTypeSource(StrEnum): + """Authored layer that supplied the effective table type.""" + + DEFAULT = "default" + TARGET = "target" + MATERIALIZATION = "materialization" + MODEL = "model" + +class TableTypeDowngradePolicy(StrEnum): + """Permanent-to-transient conversion policy.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + +class MissingMigrationOriginPolicy(StrEnum): + """Policy for a declared migrate_from whose origin does not exist in a target.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + +class RetentionDecreasePolicy(StrEnum): + """Policy for lowering live time-travel retention.""" + + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + ALLOW = "allow" + +class ClonePolicy(NativeRecord): + """Environment clone policy.""" + + allow_as_clone_origin: bool + allow_as_clone_destination: bool + def __init__( + self, + allow_as_clone_origin: bool = ..., + allow_as_clone_destination: bool = ..., + ) -> None: ... + +class LocalClonePolicy(NativeRecord): + """Local environment clone policy overrides.""" + + allow_as_clone_origin: bool | None + allow_as_clone_destination: bool | None + def __init__( + self, + allow_as_clone_origin: bool | None = ..., + allow_as_clone_destination: bool | None = ..., + ) -> None: ... + +class AuthoredTimeTravelRetention(NativeRecord): + """One authored time-travel retention policy.""" + + desired_days: int | None + unmanaged: bool + def __init__( + self, + desired_days: int | None = ..., + unmanaged: bool = ..., + ) -> None: ... + +class MaterializationRetentionDefaults(NativeRecord): + """Retention defaults for one built-in materialization.""" + + time_travel_retention: AuthoredTimeTravelRetention | None + table_type: TableType | None + def __init__( + self, + time_travel_retention: AuthoredTimeTravelRetention | None = ..., + table_type: TableType | None = ..., + ) -> None: ... + +class MaterializationDefaultsConfig(NativeRecord): + """Project retention defaults by materialization type.""" + + table: MaterializationRetentionDefaults + incremental: MaterializationRetentionDefaults + snapshot: MaterializationRetentionDefaults + def __init__( + self, + table: MaterializationRetentionDefaults = ..., + incremental: MaterializationRetentionDefaults = ..., + snapshot: MaterializationRetentionDefaults = ..., + ) -> None: ... + +class ResolvedTimeTravelRetention(NativeRecord): + """Effective retention policy attached to a compiled model.""" + + desired_days: int | None + unmanaged: bool + source: TimeTravelRetentionSource | None + def __init__( + self, + desired_days: int | None = ..., + unmanaged: bool = ..., + source: TimeTravelRetentionSource | None = ..., + ) -> None: ... + +class ResolvedTableType(NativeRecord): + """Effective Snowflake table type attached to a compiled model.""" + + value: TableType + source: TableTypeSource + declared: bool + def __init__( + self, + value: TableType = ..., + source: TableTypeSource = ..., + declared: bool = ..., + ) -> None: ... + +class ExecutionLimitsConfig(NativeRecord): + """Optional target-scoped build execution limits.""" + + max_models: int | None + max_duration: str | None + max_duration_seconds: int | None + remediation: str | None + def __init__( + self, + max_models: int | None = ..., + max_duration: str | None = ..., + max_duration_seconds: int | None = ..., + remediation: str | None = ..., + ) -> None: ... + +class TargetDiffConfig(NativeRecord): + """Optional target-scoped data-diff limits.""" + + max_full_rows: int | Literal["unlimited"] | None + def __init__( + self, + max_full_rows: int | Literal["unlimited"] | None = ..., + ) -> None: ... + +class TargetWarehousesConfig(NativeRecord): + """Optional per-command-group default warehouses for one target.""" + + build: str | None + query: str | None + def __init__( + self, + build: str | None = ..., + query: str | None = ..., + ) -> None: ... + +class TargetConfig(NativeRecord): + """One named target configuration.""" + + connection: dict[str, object] + connection_name: str | None + vars: dict[str, str] + database: str | None + schema: str | None + loader_schema: str | None + defer_sources_to: str | None + defer_clone_from: str | None + clone: ClonePolicy + compile_cache: bool | None + time_travel_retention: AuthoredTimeTravelRetention | None + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] + owns_time_travel_retention_namespace: bool + default_table_type: TableType | None + table_type_downgrade: TableTypeDowngradePolicy + time_travel_retention_decrease: RetentionDecreasePolicy + missing_migration_origin: MissingMigrationOriginPolicy + execution_limits: ExecutionLimitsConfig + diff: TargetDiffConfig + warehouses: TargetWarehousesConfig + def __init__( + self, + connection: dict[str, object] = ..., + connection_name: str | None = ..., + vars: dict[str, str] = ..., + database: str | None = ..., + schema: str | None = ..., + loader_schema: str | None = ..., + defer_sources_to: str | None = ..., + defer_clone_from: str | None = ..., + clone: ClonePolicy = ..., + compile_cache: bool | None = ..., + time_travel_retention: AuthoredTimeTravelRetention | None = ..., + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] = ..., + owns_time_travel_retention_namespace: bool = ..., + default_table_type: TableType | None = ..., + table_type_downgrade: TableTypeDowngradePolicy = ..., + time_travel_retention_decrease: RetentionDecreasePolicy = ..., + missing_migration_origin: MissingMigrationOriginPolicy = ..., + execution_limits: ExecutionLimitsConfig = ..., + diff: TargetDiffConfig = ..., + warehouses: TargetWarehousesConfig = ..., + ) -> None: ... + +class LocalTargetConfig(NativeRecord): + """Local developer overrides for one named target.""" + + connection: dict[str, object] + connection_name: str | None + vars: dict[str, str] + database: str | None + schema: str | None + loader_schema: str | None + defer_sources_to: str | None + defer_clone_from: str | None + clone: LocalClonePolicy + compile_cache: bool | None + time_travel_retention: AuthoredTimeTravelRetention | None + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] | None + owns_time_travel_retention_namespace: bool | None + default_table_type: TableType | None + table_type_downgrade: TableTypeDowngradePolicy | None + time_travel_retention_decrease: RetentionDecreasePolicy | None + missing_migration_origin: MissingMigrationOriginPolicy | None + execution_limits: ExecutionLimitsConfig + diff: TargetDiffConfig + warehouses: TargetWarehousesConfig + def __init__( + self, + connection: dict[str, object] = ..., + connection_name: str | None = ..., + vars: dict[str, str] = ..., + database: str | None = ..., + schema: str | None = ..., + loader_schema: str | None = ..., + defer_sources_to: str | None = ..., + defer_clone_from: str | None = ..., + clone: LocalClonePolicy = ..., + compile_cache: bool | None = ..., + time_travel_retention: AuthoredTimeTravelRetention | None = ..., + time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] + | None = ..., + owns_time_travel_retention_namespace: bool | None = ..., + default_table_type: TableType | None = ..., + table_type_downgrade: TableTypeDowngradePolicy | None = ..., + time_travel_retention_decrease: RetentionDecreasePolicy | None = ..., + missing_migration_origin: MissingMigrationOriginPolicy | None = ..., + execution_limits: ExecutionLimitsConfig = ..., + diff: TargetDiffConfig = ..., + warehouses: TargetWarehousesConfig = ..., + ) -> None: ... + +class SettingsConfig(NativeRecord): + """Global feature toggles.""" + + sql_analysis: bool + require_sql_analysis: bool + query_change_tracking: bool + column_contract_mode: ColumnContractMode + concurrency: int + auto_load_sources: bool + microbatch_concurrency: bool + microbatch_unaccounted_partition_policy: str + table_promotion_mode: str | None + default_audit_severity: AuditSeverity | None + default_audit_run_scope: str | None + def __init__( + self, + sql_analysis: bool = ..., + require_sql_analysis: bool = ..., + query_change_tracking: bool = ..., + column_contract_mode: ColumnContractMode = ..., + concurrency: int = ..., + auto_load_sources: bool = ..., + microbatch_concurrency: bool = ..., + microbatch_unaccounted_partition_policy: str = ..., + table_promotion_mode: str | None = ..., + default_audit_severity: AuditSeverity | None = ..., + default_audit_run_scope: str | None = ..., + ) -> None: ... + @property + def sql_validation(self) -> bool: ... + +class ScopesConfig(NativeRecord): + """Declaration scope policy configuration.""" + + enforce_placement: bool + def __init__( + self, + enforce_placement: bool = ..., + ) -> None: ... + +class ReferencesConfig(NativeRecord): + """Explicit-reference enforcement policy configuration.""" + + enforce_explicit: bool + def __init__( + self, + enforce_explicit: bool = ..., + ) -> None: ... + +class CostConfig(NativeRecord): + """Snowflake compute estimate configuration.""" + + usd_per_credit: Decimal + usd_per_credit_is_default: bool + def __init__( + self, + usd_per_credit: Decimal = ..., + usd_per_credit_is_default: bool = ..., + ) -> None: ... + +class ConstantsConfig(NativeRecord): + """Project-wide constant rendering configuration.""" + + collection_rendering: CollectionRendering + def __init__( + self, + collection_rendering: CollectionRendering = ..., + ) -> None: ... + +class FutureCursorsConfig(NativeRecord): + """Safety policy for automatically discovered future cursor watermarks.""" + + max_distance: str | None + action: FutureCursorAction + def __init__( + self, + max_distance: str | None = ..., + action: FutureCursorAction = ..., + ) -> None: ... + +class StartCursorsConfig(NativeRecord): + """Safety policy for automatically discovered incremental starts.""" + + max_ahead: str | None + action: FutureCursorAction + def __init__( + self, + max_ahead: str | None = ..., + action: FutureCursorAction = ..., + ) -> None: ... + +class CursorsConfig(NativeRecord): + """Project-wide cursor safety configuration.""" + + future: FutureCursorsConfig + start: StartCursorsConfig + def __init__( + self, + future: FutureCursorsConfig = ..., + start: StartCursorsConfig = ..., + ) -> None: ... + +class MicrobatchLimitsConfig(NativeRecord): + """Per-model microbatch planning safety limit.""" + + max_batches: int | None + action: MicrobatchLimitAction + def __init__( + self, + max_batches: int | None = ..., + action: MicrobatchLimitAction = ..., + ) -> None: ... + +class MicrobatchesConfig(NativeRecord): + """Project-wide microbatch configuration.""" + + limits: MicrobatchLimitsConfig + def __init__( + self, + limits: MicrobatchLimitsConfig = ..., + ) -> None: ... + +class DefaultsConfig(NativeRecord): + """Project-wide resource defaults.""" + + materialized: str | None + database: str | None + schema: str | None + seed_database: str | None + seed_schema: str | None + function_database: str | None + function_schema: str | None + incremental_strategy: str | None + incremental_mode: str | None + microbatch_strategy: str | None + cursor_watermark_mode: str | None + merge_exclude_columns: tuple[str, ...] + full_refresh: bool | None + append_cursor_inclusive: object | None + cursor_start: object | None + cursor_end: object | None + cursor_start_max_ahead: str | None + cursor_start_max_action: str | None + cursor_future_max_distance: str | None + cursor_future_action: str | None + lookback: str | None + batch_size: str | int | None + batch_concurrency: int | None + max_microbatches: int | None + unaccounted_partition_policy: str | None + replay_on_change: str | None + row_diff_exclude_columns: tuple[str, ...] + row_diff_tolerances: dict[str, object] + row_diff_sample_rows: int | None + row_diff_sample_seed: int | None + tags: tuple[str, ...] + contract: str | None + pre_hooks: object | None + post_hooks: object | None + def __init__( + self, + materialized: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + seed_database: str | None = ..., + seed_schema: str | None = ..., + function_database: str | None = ..., + function_schema: str | None = ..., + incremental_strategy: str | None = ..., + incremental_mode: str | None = ..., + microbatch_strategy: str | None = ..., + cursor_watermark_mode: str | None = ..., + merge_exclude_columns: tuple[str, ...] = ..., + full_refresh: bool | None = ..., + append_cursor_inclusive: object | None = ..., + cursor_start: object | None = ..., + cursor_end: object | None = ..., + cursor_start_max_ahead: str | None = ..., + cursor_start_max_action: str | None = ..., + cursor_future_max_distance: str | None = ..., + cursor_future_action: str | None = ..., + lookback: str | None = ..., + batch_size: str | int | None = ..., + batch_concurrency: int | None = ..., + max_microbatches: int | None = ..., + unaccounted_partition_policy: str | None = ..., + replay_on_change: str | None = ..., + row_diff_exclude_columns: tuple[str, ...] = ..., + row_diff_tolerances: dict[str, object] = ..., + row_diff_sample_rows: int | None = ..., + row_diff_sample_seed: int | None = ..., + tags: tuple[str, ...] = ..., + contract: str | None = ..., + pre_hooks: object | None = ..., + post_hooks: object | None = ..., + ) -> None: ... + +class DiffConfig(NativeRecord): + """Project-wide data-diff configuration.""" + + query_artifact_ttl: str + def __init__( + self, + query_artifact_ttl: str = ..., + ) -> None: ... + +class JanitorConfig(NativeRecord): + """Janitor command defaults.""" + + enabled: bool + retention_days: int | None + archive_retention_days: int + direct_state_history_versions: int + delete_tracked_only: bool + exclude_patterns: tuple[str, ...] + def __init__( + self, + enabled: bool = ..., + retention_days: int | None = ..., + archive_retention_days: int = ..., + direct_state_history_versions: int = ..., + delete_tracked_only: bool = ..., + exclude_patterns: tuple[str, ...] = ..., + ) -> None: ... + +class MigrationsConfig(NativeRecord): + """Model migration defaults.""" + + old_name_views: str | None + def __init__( + self, + old_name_views: str | None = ..., + ) -> None: ... + +class SnapshotsConfig(NativeRecord): + """Snapshot materialization safety defaults.""" + + current_state_full_refresh: str + historical_full_refresh: str + schema_change: str + wildcard_check_schema_change: str + def __init__( + self, + current_state_full_refresh: str = ..., + historical_full_refresh: str = ..., + schema_change: str = ..., + wildcard_check_schema_change: str = ..., + ) -> None: ... + +class ScenarioSnapshotLimitsConfig(NativeRecord): + """Scenario snapshot capture safety limits.""" + + max_rows_per_relation: int | None + max_total_rows: int | None + max_bytes_per_relation: int | None + max_total_bytes: int | None + def __init__( + self, + max_rows_per_relation: int | None = ..., + max_total_rows: int | None = ..., + max_bytes_per_relation: int | None = ..., + max_total_bytes: int | None = ..., + ) -> None: ... + +class ScenarioConfig(NativeRecord): + """Scenario command configuration.""" + + run_namespace: str | None + local_type_overrides: dict[str, dict[str, str]] + snapshot_limits: ScenarioSnapshotLimitsConfig + def __init__( + self, + run_namespace: str | None = ..., + local_type_overrides: dict[str, dict[str, str]] = ..., + snapshot_limits: ScenarioSnapshotLimitsConfig = ..., + ) -> None: ... + +class DbtConfig(NativeRecord): + """dbt interop configuration.""" + + project_dir: str | None + profiles_dir: str | None + target: str | None + target_path: str | None + vars: dict[str, object] + def __init__( + self, + project_dir: str | None = ..., + profiles_dir: str | None = ..., + target: str | None = ..., + target_path: str | None = ..., + vars: dict[str, object] = ..., + ) -> None: ... + +class LocalDbtConfig(NativeRecord): + """Local dbt interop configuration overrides.""" + + target: str | None + vars: dict[str, object] + def __init__( + self, + target: str | None = ..., + vars: dict[str, object] = ..., + ) -> None: ... + +class LifecycleEventSinkFilterConfig(NativeRecord): + """Destination-neutral runtime narrowing for lifecycle-event sinks.""" + + event_kinds: frozenset[str] | None + min_severity: EventExportSeverity | None + def __init__( + self, + event_kinds: frozenset[str] | None = ..., + min_severity: EventExportSeverity | None = ..., + ) -> None: ... + +class LifecycleEventSinksConfig(NativeRecord): + """Global and named lifecycle-event sink runtime filters and delivery settings.""" + + defaults: LifecycleEventSinkFilterConfig + named: dict[str, LifecycleEventSinkFilterConfig] + shutdown_timeout: str | None + shutdown_timeout_seconds: int | None + def __init__( + self, + defaults: LifecycleEventSinkFilterConfig = ..., + named: dict[str, LifecycleEventSinkFilterConfig] = ..., + shutdown_timeout: str | None = ..., + shutdown_timeout_seconds: int | None = ..., + ) -> None: ... + +class SinksConfig(NativeRecord): + """Typed destination sink configuration.""" + + lifecycle: LifecycleEventSinksConfig + def __init__( + self, + lifecycle: LifecycleEventSinksConfig = ..., + ) -> None: ... + +class ProjectConfig(NativeRecord): + """Shared project configuration loaded from sqlbuild_project.toml.""" + + name: str + adapter: str + default_target: str | None + connection: dict[str, object] + connections: dict[str, dict[str, object]] + settings: SettingsConfig + scopes: ScopesConfig + references: ReferencesConfig + cost: CostConfig + constants: ConstantsConfig + cursors: CursorsConfig + microbatches: MicrobatchesConfig + defaults: DefaultsConfig + diff: DiffConfig + materialization_defaults: MaterializationDefaultsConfig + path_defaults: dict[str, dict[str, object]] + vars: dict[str, str] + targets: dict[str, TargetConfig] + janitor: JanitorConfig + migrations: MigrationsConfig + snapshots: SnapshotsConfig + scenario: ScenarioConfig + dbt: DbtConfig + sinks: SinksConfig + def __init__( + self, + name: str, + adapter: str, + default_target: str | None = ..., + connection: dict[str, object] = ..., + connections: dict[str, dict[str, object]] = ..., + settings: SettingsConfig = ..., + scopes: ScopesConfig = ..., + references: ReferencesConfig = ..., + cost: CostConfig = ..., + constants: ConstantsConfig = ..., + cursors: CursorsConfig = ..., + microbatches: MicrobatchesConfig = ..., + defaults: DefaultsConfig = ..., + diff: DiffConfig = ..., + materialization_defaults: MaterializationDefaultsConfig = ..., + path_defaults: dict[str, dict[str, object]] = ..., + vars: dict[str, str] = ..., + targets: dict[str, TargetConfig] = ..., + janitor: JanitorConfig = ..., + migrations: MigrationsConfig = ..., + snapshots: SnapshotsConfig = ..., + scenario: ScenarioConfig = ..., + dbt: DbtConfig = ..., + sinks: SinksConfig = ..., + ) -> None: ... + +class LocalConfig(NativeRecord): + """Local developer overrides from sqlbuild_local.toml.""" + + target: str | None + adapter: str | None + connection: dict[str, object] + connections: dict[str, dict[str, object]] + targets: dict[str, LocalTargetConfig] + settings: SettingsConfig + setting_overrides: frozenset[str] + vars: dict[str, str] + dbt: LocalDbtConfig + scenario: ScenarioConfig + def __init__( + self, + target: str | None = ..., + adapter: str | None = ..., + connection: dict[str, object] = ..., + connections: dict[str, dict[str, object]] = ..., + targets: dict[str, LocalTargetConfig] = ..., + settings: SettingsConfig = ..., + setting_overrides: frozenset[str] = ..., + vars: dict[str, str] = ..., + dbt: LocalDbtConfig = ..., + scenario: ScenarioConfig = ..., + ) -> None: ... + +class SourceLocation(NativeRecord): + """An authored source location for compiler diagnostics.""" + + path: Path + line: int + column: int + end_line: int | None + end_column: int | None + def __init__( + self, + path: Path, + line: int, + column: int, + end_line: int | None = ..., + end_column: int | None = ..., + ) -> None: ... + +class SeedCsvSettings(NativeRecord): + """CSV reader settings for a seed file.""" + + delimiter: str | None + quotechar: str | None + doublequote: bool | None + escapechar: str | None + skipinitialspace: bool | None + lineterminator: str | None + encoding: str | None + na_values: tuple[object, ...] | dict[str, tuple[object, ...]] | None + keep_default_na: bool | None + def __init__( + self, + delimiter: str | None = ..., + quotechar: str | None = ..., + doublequote: bool | None = ..., + escapechar: str | None = ..., + skipinitialspace: bool | None = ..., + lineterminator: str | None = ..., + encoding: str | None = ..., + na_values: tuple[object, ...] | dict[str, tuple[object, ...]] | None = ..., + keep_default_na: bool | None = ..., + ) -> None: ... + +class SchemaAuditInstance(NativeRecord): + """One audit instance attached to model, column, or seed metadata.""" + + definition_name: str + arguments: dict[str, object] + name: str | None + description: str | None + severity: AuditSeverity | None + run_scope: str | None + always_run: bool + thresholds: MeasurementThresholds | None + minimum_samples: int | None + evidence_limit: int | None + location: SourceLocation | None + def __init__( + self, + definition_name: str, + arguments: dict[str, object] = ..., + name: str | None = ..., + description: str | None = ..., + severity: AuditSeverity | None = ..., + run_scope: str | None = ..., + always_run: bool = ..., + thresholds: MeasurementThresholds | None = ..., + minimum_samples: int | None = ..., + evidence_limit: int | None = ..., + location: SourceLocation | None = ..., + ) -> None: ... + +class SchemaColumn(NativeRecord): + """One declared model, seed, or source column entry.""" + + name: str + type: str | None + nullable: bool | None + description: str | None + meta: dict[str, object] + audits: tuple[SchemaAuditInstance, ...] + location: SourceLocation | None + migrate_from: str | None + def __init__( + self, + name: str, + type: str | None = ..., + nullable: bool | None = ..., + description: str | None = ..., + meta: dict[str, object] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + location: SourceLocation | None = ..., + migrate_from: str | None = ..., + ) -> None: ... + +class SchemaDynamicColumnFamily(NativeRecord): + """One declared family of columns generated by a runtime-dynamic pivot.""" + + name: str + pivot_column: str + value_column: str + aggregate: str + type: str + name_pattern: str | None + def __init__( + self, + name: str, + pivot_column: str, + value_column: str, + aggregate: str, + type: str, + name_pattern: str | None = ..., + ) -> None: ... + +class SchemaModelEntry(NativeRecord): + """One model metadata entry normalized from MODEL(...).""" + + name: str + model_schema: str | None + description: str | None + type_enforcement: bool | None + meta: dict[str, object] + columns: tuple[SchemaColumn, ...] + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] + audits: tuple[SchemaAuditInstance, ...] + tags: tuple[str, ...] + def __init__( + self, + name: str, + model_schema: str | None = ..., + description: str | None = ..., + type_enforcement: bool | None = ..., + meta: dict[str, object] = ..., + columns: tuple[SchemaColumn, ...] = ..., + dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + tags: tuple[str, ...] = ..., + ) -> None: ... + +class SchemaSeedEntry(NativeRecord): + """One seed metadata entry from seed YAML.""" + + name: str + description: str | None + database: str | None + schema: str | None + meta: dict[str, object] + csv_settings: SeedCsvSettings + columns: tuple[SchemaColumn, ...] + tags: tuple[str, ...] + audits: tuple[SchemaAuditInstance, ...] + def __init__( + self, + name: str, + description: str | None = ..., + database: str | None = ..., + schema: str | None = ..., + meta: dict[str, object] = ..., + csv_settings: SeedCsvSettings = ..., + columns: tuple[SchemaColumn, ...] = ..., + tags: tuple[str, ...] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + ) -> None: ... + +class IntegrationLoaderConfig(NativeRecord): + """Typed configuration for one declarative integration loader.""" + + kind: str + config: object + def __init__( + self, + kind: str, + config: object, + ) -> None: ... + +class SourceFreshnessAgePolicy(NativeRecord): + """Age-based source freshness warning/error thresholds.""" + + warn_after: str | None + error_after: str | None + def __init__( + self, + warn_after: str | None = ..., + error_after: str | None = ..., + ) -> None: ... + +class SourceFreshnessConfig(NativeRecord): + """Configured source freshness observation for planning.""" + + strategy: SourceFreshnessStrategy + value_kind: SourceFreshnessValueKind | None + column: str | None + query: str | None + filter: str | None + lag_tolerance: str | None + age_policy: SourceFreshnessAgePolicy | None + def __init__( + self, + strategy: SourceFreshnessStrategy, + value_kind: SourceFreshnessValueKind | None = ..., + column: str | None = ..., + query: str | None = ..., + filter: str | None = ..., + lag_tolerance: str | None = ..., + age_policy: SourceFreshnessAgePolicy | None = ..., + ) -> None: ... + +class SourceColumnEntry(NativeRecord): + """One source column entry from sources/*.yml.""" + + name: str + type: str | None + nullable: bool | None + description: str | None + meta: dict[str, object] + audits: tuple[SchemaAuditInstance, ...] + def __init__( + self, + name: str, + type: str | None = ..., + nullable: bool | None = ..., + description: str | None = ..., + meta: dict[str, object] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + ) -> None: ... + +class SourceEntry(NativeRecord): + """One source declaration from sources/*.yml.""" + + name: str + database: str | None + schema: str | None + table: str | None + loader: str | None + managed: bool + integration_loader: IntegrationLoaderConfig | None + freshness: SourceFreshnessConfig | None + write_strategy: SourceWriteStrategy | None + load_batch_size: int | None + cursor_column: str | None + unique_key: tuple[str, ...] + expression: str | None + description: str | None + type_enforcement: bool | None + contract: str | None + meta: dict[str, object] + columns: tuple[SourceColumnEntry, ...] + audits: tuple[SchemaAuditInstance, ...] + def __init__( + self, + name: str, + database: str | None = ..., + schema: str | None = ..., + table: str | None = ..., + loader: str | None = ..., + managed: bool = ..., + integration_loader: IntegrationLoaderConfig | None = ..., + freshness: SourceFreshnessConfig | None = ..., + write_strategy: SourceWriteStrategy | None = ..., + load_batch_size: int | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + expression: str | None = ..., + description: str | None = ..., + type_enforcement: bool | None = ..., + contract: str | None = ..., + meta: dict[str, object] = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + audits: tuple[SchemaAuditInstance, ...] = ..., + ) -> None: ... + +class LoaderDestinationParts(NativeRecord): + """Parsed warehouse relation parts for a loader destination.""" + + database: str | None + schema: str | None + table: str + def __init__( + self, + database: str | None, + schema: str | None, + table: str, + ) -> None: ... + +class MeasurementThresholdBound(NativeRecord): + """One directional threshold bound for a measurement audit.""" + + operator: ThresholdOperator + limit: float | None + lower: float | None + upper: float | None + def __init__( + self, + operator: ThresholdOperator, + limit: float | None = ..., + lower: float | None = ..., + upper: float | None = ..., + ) -> None: ... + +class MeasurementThresholds(NativeRecord): + """Warning and error policy for a measurement audit.""" + + warn: MeasurementThresholdBound | None + error: MeasurementThresholdBound | None + def __init__( + self, + warn: MeasurementThresholdBound | None = ..., + error: MeasurementThresholdBound | None = ..., + ) -> None: ... + +class MeasurementContract(NativeRecord): + """Columns and sample unit returned by a measurement query.""" + + value_column: str + sample_count_column: str | None + sample_unit: str | None + def __init__( + self, + value_column: str, + sample_count_column: str | None = ..., + sample_unit: str | None = ..., + ) -> None: ... + +class AuditIdentity(NativeRecord): + """Stable identity values for one planned audit binding.""" + + binding_key: str + audit_name: str + definition_fingerprint: str + execution_fingerprint: str + severity: AuditSeverity + run_scope_phase: str + attachment_kind: str + attached_target_name: str | None + attached_column_name: str | None + always_run: bool + def __init__( + self, + binding_key: str, + audit_name: str, + definition_fingerprint: str, + execution_fingerprint: str, + severity: AuditSeverity, + run_scope_phase: str, + attachment_kind: str, + attached_target_name: str | None = ..., + attached_column_name: str | None = ..., + always_run: bool = ..., + ) -> None: ... + +class AuditGateIdentity(NativeRecord): + """Aggregate identity for a model's runtime audit gate.""" + + binding_set_hash: str + blocking_set_hash: str + audits: tuple[AuditIdentity, ...] + def __init__( + self, + binding_set_hash: str, + blocking_set_hash: str, + audits: tuple[AuditIdentity, ...], + ) -> None: ... + +class SqlValueKind(StrEnum): + """Closed logical kinds accepted by the typed SQL value boundary.""" + + STRING = "string" + INTEGER = "integer" + BOOLEAN = "boolean" + FLOAT = "float" + DECIMAL = "decimal" + NULL = "null" + LIST = "list" + SET = "set" + OBJECT = "object" + +class StateSqlValueType(StrEnum): + """Closed physical value kinds used by warehouse state columns.""" + + STRING = "string" + INTEGER = "integer" + BOOLEAN = "boolean" + TIMESTAMP = "timestamp" + TEXT_TIMESTAMP = "text_timestamp" + DATE = "date" + JSON = "json" + +class SqlLogicalType(NativeRecord): + """Logical type inferred for a normalized SQL value.""" + + kind: SqlValueKind + element_type: SqlLogicalType | None + def __init__( + self, + kind: SqlValueKind, + element_type: SqlLogicalType | None = ..., + ) -> None: ... + @property + def display_name(self) -> str: ... + +class SqlValue(NativeRecord): + """One validated, canonical SQL value.""" + + logical_type: SqlLogicalType + value: SqlValuePayload + def __init__( + self, + logical_type: SqlLogicalType, + value: SqlValuePayload, + ) -> None: ... + @property + def kind(self) -> SqlValueKind: ... + +class Fingerprint(NativeRecord): + """One applied fingerprint record from a successful materialization.""" + + node_type: str + node_name: str + target_database: str | None + target_schema: str | None + target_name: str | None + run_id: str + definition_hash: str + schema_fingerprint: str + definition: str + ts: datetime + metadata_json: str + version_hash: str + def __init__( + self, + node_type: str, + node_name: str, + target_database: str | None, + target_schema: str | None, + target_name: str | None, + run_id: str, + definition_hash: str, + schema_fingerprint: str, + definition: str, + ts: datetime, + metadata_json: str = ..., + version_hash: str = ..., + ) -> None: ... + +class FingerprintSet(NativeRecord): + """Latest fingerprints per node name for one target schema.""" + + schema: str + fingerprints: dict[str, Fingerprint] + fingerprints_by_identity: dict[tuple[str, str], Fingerprint] | None + def __init__( + self, + schema: str, + fingerprints: dict[str, Fingerprint], + fingerprints_by_identity: dict[tuple[str, str], Fingerprint] | None = ..., + ) -> None: ... + +class SourceFreshnessRecord(NativeRecord): + """One append-only direct observation whose observed time is always aware UTC.""" + + source_name: str + target_database: str | None + target_schema: str | None + target_name: str | None + run_id: str + strategy: str + value_kind: str + data_version: str | None + data_version_hash: str + observed_at: datetime + def __init__( + self, + source_name: str, + target_database: str | None, + target_schema: str | None, + target_name: str | None, + run_id: str, + strategy: str, + value_kind: str, + data_version: str | None, + data_version_hash: str, + observed_at: datetime, + ) -> None: ... + @property + def identity(self) -> SourceFreshnessIdentity: ... + +class SelectedDbtProfileOutput(NativeRecord): + """Selected dbt profile target output before rendering.""" + + profile_name: str + target_name: str + output: dict[str, object] + def __init__( + self, + profile_name: str, + target_name: str, + output: dict[str, object], + ) -> None: ... + +class ResolvedDbtProfileOutput(NativeRecord): + """Rendered dbt profile target output.""" + + project_dir: Path + profiles_dir: Path + profile_name: str + target_name: str + output: dict[str, object] + def __init__( + self, + project_dir: Path, + profiles_dir: Path, + profile_name: str, + target_name: str, + output: dict[str, object], + ) -> None: ... + +class DiscoveredPythonNode(NativeRecord): + """Shared internal view of a discovered Python DAG node.""" + + kind: PythonNodeKind + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + sql_deps: tuple[SqlResourceRef, ...] + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + loader: DiscoveredPythonLoaderMetadata | None + task: DiscoveredPythonTaskMetadata | None + asset: DiscoveredPythonAssetMetadata | None + check: DiscoveredPythonCheckMetadata | None + identity: PythonNodeIdentity | None + def __init__( + self, + kind: PythonNodeKind, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + sql_deps: tuple[SqlResourceRef, ...] = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + loader: DiscoveredPythonLoaderMetadata | None = ..., + task: DiscoveredPythonTaskMetadata | None = ..., + asset: DiscoveredPythonAssetMetadata | None = ..., + check: DiscoveredPythonCheckMetadata | None = ..., + identity: PythonNodeIdentity | None = ..., + ) -> None: ... + +class PythonNodeGraph(NativeRecord): + """Internal graph inventory for discovered executable Python nodes.""" + + nodes: tuple[DiscoveredPythonNode, ...] + dependency_edges: tuple[PythonNodeDependencyEdge, ...] + upstream_deps: dict[str, tuple[str, ...]] + downstream_deps: dict[str, tuple[str, ...]] + tag_index: dict[str, frozenset[str]] + path_index: dict[str, str] + nodes_by_name: dict[str, DiscoveredPythonNode] + nodes_by_typed_selector: dict[str, DiscoveredPythonNode] + def __init__( + self, + nodes: tuple[DiscoveredPythonNode, ...], + dependency_edges: tuple[PythonNodeDependencyEdge, ...], + upstream_deps: dict[str, tuple[str, ...]], + downstream_deps: dict[str, tuple[str, ...]], + tag_index: dict[str, frozenset[str]], + path_index: dict[str, str], + nodes_by_name: dict[str, DiscoveredPythonNode], + nodes_by_typed_selector: dict[str, DiscoveredPythonNode], + ) -> None: ... + def selected_sql_refs(self, *, selected_names: frozenset[str]) -> frozenset[SqlResourceRef]: ... + +class PythonSqlRunLifecyclePlan(NativeRecord): + """Lifecycle-aware run classification for SQL resources and Python nodes.""" + + ingress_python_node_names: frozenset[str] + ingress_loader_names: frozenset[str] + read_side_python_node_names: frozenset[str] + def __init__( + self, + ingress_python_node_names: frozenset[str], + ingress_loader_names: frozenset[str], + read_side_python_node_names: frozenset[str], + ) -> None: ... + +class PythonIdentityDependency(NativeRecord): + """One first-party Python source object included in a Python node identity.""" + + kind: str + module: str + qualname: str + source_path: str + source_hash: str + source_text: str + def __init__( + self, + kind: str, + module: str, + qualname: str, + source_path: str, + source_hash: str, + source_text: str, + ) -> None: ... + +class PythonNodeIdentity(NativeRecord): + """Read-only identity metadata for a SQLBuild Python node.""" + + node_type: str + node_name: str + source_path: str + source_hash: str + definition_hash: str + version_hash: str + dependencies: tuple[PythonIdentityDependency, ...] + definition_json: str + metadata_json: str + def __init__( + self, + node_type: str, + node_name: str, + source_path: str, + source_hash: str, + definition_hash: str, + version_hash: str, + dependencies: tuple[PythonIdentityDependency, ...] = ..., + definition_json: str = ..., + metadata_json: str = ..., + ) -> None: ... + +class SqlLexicalSyntax(NativeRecord): + """Dialect rules for where SQL quoted text and comments end.""" + + backslash_escape_quotes: frozenset[str] + escape_string_prefix: bool + raw_string_prefix: bool + triple_quoted_strings: bool + nested_block_comments: bool + line_comment_prefixes: frozenset[str] + def __init__( + self, + backslash_escape_quotes: frozenset[str] = ..., + escape_string_prefix: bool = ..., + raw_string_prefix: bool = ..., + triple_quoted_strings: bool = ..., + nested_block_comments: bool = ..., + line_comment_prefixes: frozenset[str] = ..., + ) -> None: ... + @property + def cache_key(self) -> str: ... + @property + def native_mapping(self) -> dict[str, object]: ... + def reads_differently_from_generic(self, sql: str) -> bool: ... + +class TimestampValue(NativeRecord): + """Timestamp cursor value; aware datetimes are normalized to UTC at ingestion.""" + + value: datetime + source_text: str | None + def __init__( + self, + value: datetime, + source_text: str | None = ..., + ) -> None: ... + +class DateValue(NativeRecord): + """Date cursor value which must remain date-formatted when rendered.""" + + value: date + def __init__( + self, + value: date, + ) -> None: ... + +class IntegerValue(NativeRecord): + """Integral cursor value.""" + + value: int + def __init__( + self, + value: int, + ) -> None: ... + +class AlignedInterval(NativeRecord): + """Validated half-open cursor interval.""" + + start: TimestampValue | DateValue | IntegerValue + end: TimestampValue | DateValue | IntegerValue + grain: CursorGrain | None + def __init__( + self, + start: TimestampValue | DateValue | IntegerValue, + end: TimestampValue | DateValue | IntegerValue, + grain: CursorGrain | None, + ) -> None: ... + +class Duration(NativeRecord): + """A calendar-aware duration (years/months plus fixed time units).""" + + years: int + months: int + days: int + hours: int + minutes: int + seconds: int + def __init__( + self, + years: int = ..., + months: int = ..., + days: int = ..., + hours: int = ..., + minutes: int = ..., + seconds: int = ..., + ) -> None: ... + @classmethod + def parse(cls, value: str) -> Duration | None: ... + @property + def is_zero(self) -> bool: ... + @property + def units(self) -> frozenset[str]: ... + def is_single_unit_in(self, allowed_units: frozenset[str]) -> bool: ... + @property + def has_calendar_component(self) -> bool: ... + @property + def fixed_seconds(self) -> int: ... + @property + def total_months(self) -> int: ... + def add_to(self, moment: datetime) -> datetime: ... + def subtract_from(self, moment: datetime) -> datetime: ... + +class MigrationDecision(StrEnum): + """Per-run decision for one declared or discovered model migration.""" + + DONE = "done" + MIGRATE = "migrate" + REDO = "redo" + SUPERSEDED_REPLACE = "superseded_replace" + FORCED_REPLACE = "forced_replace" + CONFLICT = "conflict" + ORIGIN_MISSING = "origin_missing" + RENAMED = "renamed" + @property + def moves_data(self) -> bool: ... + @property + def blocks_build(self) -> bool: ... + @property + def checks_compatibility(self) -> bool: ... + @property + def label(self) -> str: ... + +class MigrationPromotion(StrEnum): + """How a completed migration stage replaces or creates the destination.""" + + SWAP = "swap" + RENAME = "rename" + TRANSACTIONAL_RENAME = "transactional_rename" + @property + def label(self) -> str: ... + +class ColumnMigrationDecision(StrEnum): + """Per-run decision for one declared or detected column rename.""" + + RENAME = "rename" + RECORD = "record" + DONE = "done" + CONFLICT = "conflict" + SOURCE_MISSING = "source_missing" + STILL_PRODUCED = "still_produced" + UNSUPPORTED = "unsupported" + @property + def renames(self) -> bool: ... + @property + def records_event(self) -> bool: ... + @property + def blocks_build(self) -> bool: ... + @property + def label(self) -> str: ... + +class OldNameViewEventType(StrEnum): + """Immutable facts about what happened at a migrated relation's old name.""" + + REQUIRED = "required" + ORIGIN_ARCHIVED = "origin_archived" + VIEW_CREATED = "view_created" + VIEW_DROPPED = "view_dropped" + +class OldNameViewDropReason(StrEnum): + """Why a compatibility view at an old name was dropped.""" + + EXPIRED = "expired" + EARLY = "early" + MISSING = "missing" + +class OldNameViewStatus(StrEnum): + """Projected state of the old-name steps of one recorded move.""" + + PENDING_ARCHIVE = "pending_archive" + PENDING_VIEW = "pending_view" + LIVE = "live" + EXPIRED = "expired" + DROPPED = "dropped" + +class MigrationRelation(NativeRecord): + """One physical relation named by a migration event.""" + + database: str | None + schema: str | None + name: str + def __init__( + self, + database: str | None, + schema: str | None, + name: str, + ) -> None: ... + @property + def identity(self) -> tuple[str | None, str, str]: ... + def matches(self, other: MigrationRelation) -> bool: ... + +class MigrationEvent(NativeRecord): + """One immutable fact that a origin relation was cloned over a destination.""" + + event_id: str + target_name: str | None + origin_model: str | None + origin: MigrationRelation + destination_model: str + destination: MigrationRelation + origin_version_hash: str + discovery: MigrationDiscovery + decision: MigrationDecision + run_id: str + created_at: datetime + def __init__( + self, + event_id: str, + target_name: str | None, + origin_model: str | None, + origin: MigrationRelation, + destination_model: str, + destination: MigrationRelation, + origin_version_hash: str, + discovery: MigrationDiscovery, + decision: MigrationDecision, + run_id: str, + created_at: datetime, + ) -> None: ... + def mentions(self, relation: MigrationRelation) -> bool: ... + +class ColumnMigrationEvent(NativeRecord): + """One immutable fact that a column of a model relation was renamed in place.""" + + event_id: str + target_name: str | None + model_name: str + relation: MigrationRelation + origin_column: str + destination_column: str + discovery: MigrationDiscovery + decision: ColumnMigrationDecision + run_id: str + created_at: datetime + def __init__( + self, + event_id: str, + target_name: str | None, + model_name: str, + relation: MigrationRelation, + origin_column: str, + destination_column: str, + discovery: MigrationDiscovery, + decision: ColumnMigrationDecision, + run_id: str, + created_at: datetime, + ) -> None: ... + def mentions(self, *, relation: MigrationRelation, columns: frozenset[str]) -> bool: ... + def renamed(self, *, origin_column: str, destination_column: str) -> bool: ... + +class OldNameViewEvent(NativeRecord): + """One immutable fact about the old name of one recorded move.""" + + event_id: str + target_name: str | None + event_type: OldNameViewEventType + migration_event_id: str + destination_model: str + old: MigrationRelation + new: MigrationRelation + run_id: str + created_at: datetime + view_retention: str | None + archive_name: str | None + column_aliases: tuple[tuple[str, str], ...] + expires_at: datetime | None + drop_reason: OldNameViewDropReason | None + grants_copied: tuple[str, ...] | None + view_sql: str | None + def __init__( + self, + event_id: str, + target_name: str | None, + event_type: OldNameViewEventType, + migration_event_id: str, + destination_model: str, + old: MigrationRelation, + new: MigrationRelation, + run_id: str, + created_at: datetime, + view_retention: str | None = ..., + archive_name: str | None = ..., + column_aliases: tuple[tuple[str, str], ...] = ..., + expires_at: datetime | None = ..., + drop_reason: OldNameViewDropReason | None = ..., + grants_copied: tuple[str, ...] | None = ..., + view_sql: str | None = ..., + ) -> None: ... + +class OldNameViewHistory(NativeRecord): + """The recorded move and every old-name fact that references it.""" + + move: MigrationEvent + required: OldNameViewEvent + archived: OldNameViewEvent | None + created: OldNameViewEvent | None + dropped: OldNameViewEvent | None + def __init__( + self, + move: MigrationEvent, + required: OldNameViewEvent, + archived: OldNameViewEvent | None = ..., + created: OldNameViewEvent | None = ..., + dropped: OldNameViewEvent | None = ..., + ) -> None: ... + def status(self, *, now: datetime) -> OldNameViewStatus: ... + @property + def old(self) -> MigrationRelation: ... + @property + def new(self) -> MigrationRelation: ... + +class BackfillAction(StrEnum): + FULL = "full" + BOUNDED = "bounded" + FORWARD_ONLY = "forward" + +class OnSchemaChange(StrEnum): + IGNORE = "ignore" + FAIL = "fail" + APPEND_NEW_COLUMNS = "append_new_columns" + SYNC_ALL_COLUMNS = "sync_all_columns" + +class ContractPolicy(StrEnum): + NONE = "none" + ENFORCED = "enforced" + +class RetentionDirection(StrEnum): + INCREASE = "increase" + DECREASE = "decrease" + MIXED = "mixed" + MATCH = "match" + APPLY_AFTER_CREATE = "apply_after_create" + +class RetentionPlanPhase(StrEnum): + PRE = "pre" + POST = "post" + AFTER_CREATE = "after_create" + NONE = "none" + +class PlanAction(StrEnum): + CREATE_VIEW = "create_view" + CREATE_TABLE = "create_table" + INCREMENTAL_APPEND = "incremental_append" + INCREMENTAL_DELETE_INSERT = "incremental_delete_insert" + INCREMENTAL_MERGE = "incremental_merge" + SNAPSHOT = "snapshot" + LOAD_SEED = "load_seed" + SKIP = "skip" + CUSTOM = "custom" + +class PlanReason(StrEnum): + FIRST_RUN = "first_run" + RENAMED = "renamed" + FULL_REFRESH = "full_refresh" + QUERY_CHANGED = "query_changed" + FUNCTION_CHANGED = "function_changed" + CONFIG_CHANGED = "config_changed" + SCHEMA_CHANGED = "schema_changed" + NORMAL_INCREMENTAL = "normal_incremental" + NO_CHANGE = "no_change" + DISABLED = "disabled" + SOURCE_FRESHNESS_ERROR = "source_freshness_error" + EXTERNAL_UPSTREAM_FAILED = "external_upstream_failed" + +class IncrementalStrategy(StrEnum): + APPEND = "append" + DELETE_INSERT = "delete_insert" + MERGE = "merge" + +class MicrobatchStrategy(StrEnum): + """How an ordinary microbatch range is selected.""" + + ROLLING_WINDOW = "rolling_window" + WATERMARK = "watermark" + +class IncrementalMode(StrEnum): + FULL = "full" + MICROBATCH = "microbatch" + +class SnapshotStrategy(StrEnum): + TIMESTAMP = "timestamp" + CHECK = "check" + +class HistoricalInput(StrEnum): + SNAPSHOT = "snapshot" + CHANGES = "changes" + +class InitialValidFrom(StrEnum): + UPDATED_AT = "updated_at" + OBSERVED_AT = "observed_at" + EXECUTION_TIME = "execution_time" + +class SnapshotSchemaChangePolicy(StrEnum): + DENY = "deny" + REQUIRE_CONFIRMATION = "require_confirmation" + APPEND_NEW_COLUMNS = "append_new_columns" + +class CursorType(StrEnum): + TIMESTAMP = "timestamp" + INTEGER = "integer" + +class CursorGrain(StrEnum): + SECOND = "second" + MINUTE = "minute" + HOUR = "hour" + DAY = "day" + MONTH = "month" + YEAR = "year" + +class MaterializationType(StrEnum): + VIEW = "view" + TABLE = "table" + INCREMENTAL = "incremental" + SNAPSHOT = "snapshot" + SEED = "seed" + CUSTOM = "custom" + @classmethod + def is_table_backed(cls, *, materialized: object | None) -> bool: ... + +class ScenarioArtifactKind(StrEnum): + SOURCE = "source" + REF = "ref" + SEED = "seed" + DBT_REF = "dbt_ref" + MODEL = "model" + +class PlannerInputError(ValueError): + """Raised when planner inputs cannot be resolved safely.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class NativeSqlTestPlanningError(PlannerInputError): + """Raised when the native SQL-test planning boundary returns invalid data.""" + +class FutureCursorSafetyError(PlannerInputError): + """Raised when an effective cursor exceeds the configured future limit.""" + +class MaximumAutomaticStartError(PlannerInputError): + """Raised when automatic cursor recovery would exceed its configured horizon.""" + + evidence: Any + +class CursorOverrides(NativeRecord): + """Cursor override values from CLI flags; untyped `start`/`end` await planner typing.""" + + start_ts: CursorScalar | None + end_ts: CursorScalar | None + start_int: CursorScalar | None + end_int: CursorScalar | None + start: str | None + end: str | None + def __init__( + self, + *, + start_ts: str | CursorScalar | None = None, + end_ts: str | CursorScalar | None = None, + start_int: str | CursorScalar | None = None, + end_int: str | CursorScalar | None = None, + start: str | None = None, + end: str | None = None, + ) -> None: ... + @property + def has_untyped(self) -> bool: ... + def typed_as(self, *, cursor_type: CursorType) -> CursorOverrides: ... + +class EmptyCursorInputDecision(NativeRecord): + """Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.""" + + input_names: tuple[str, ...] + waiting_on_empty_inputs: bool + def __init__( + self, + input_names: tuple[str, ...] = ..., + waiting_on_empty_inputs: bool = ..., + ) -> None: ... + +class ModelCursorSnapshot(NativeRecord): + """Cursor MIN/MAX values gathered from warehouse for one incremental model.""" + + target_max: CursorScalar | None + upstream_mins: tuple[CursorScalar, ...] + upstream_maxes: tuple[CursorScalar, ...] + physical_target_max: CursorScalar | None + target_eligible_max: CursorScalar | None + target_relation: str | None + destination_cursor_column: str | None + input_evidence: tuple[CursorInputEvidence, ...] + expected_watermark_count: int + unavailable_watermark_tags: tuple[str, ...] + cursor_watermark_mode: CursorWatermarkMode + upstream_terminal_starts: tuple[CursorScalar, ...] + upstream_terminal_ends: tuple[CursorScalar, ...] + upstream_end_inputs: tuple[tuple[CursorScalar | None, CursorScalar | None], ...] + upstream_availability_ends: tuple[CursorScalar, ...] + empty_input_names: tuple[str, ...] + unreadable_input_names: tuple[str, ...] + def __init__(self, **values: Any) -> None: ... + @property + def watermarks_available(self) -> bool: ... + +class CursorBounds(NativeRecord): + """Effective cursor start and end values for one incremental model.""" + + start: Bound + end: Bound + future_safety: FutureCursorSafetyEvidence | None + maximum_start_safety: MaximumStartSafetyEvidence | None + def __init__( + self, + *, + start: Bound | str, + end: Bound | str, + future_safety: FutureCursorSafetyEvidence | None = None, + maximum_start_safety: MaximumStartSafetyEvidence | None = None, + ) -> None: ... + +class CursorInputEvidence(NativeRecord): + """Observed bounds for one physical cursor input.""" + + relation: str + cursor_column: str + minimum: CursorScalar | None + maximum: CursorScalar + def __init__( + self, + *, + relation: str, + cursor_column: str, + minimum: CursorScalar | str | None, + maximum: CursorScalar | str, + ) -> None: ... + +class FutureCursorSafetyEvidence(NativeRecord): + """Structured evidence for one applied future-cursor cap.""" + + action: FutureCursorAction + max_distance: str + invocation_time: CursorScalar + discovered_start: CursorScalar + discovered_end: CursorScalar + applied_start: CursorScalar + applied_end: CursorScalar + maximum_allowed_start: CursorScalar + maximum_allowed_end: CursorScalar + future_start_detected: bool + future_end_detected: bool + determining_relation: str | None + determining_cursor_column: str | None + inputs: tuple[CursorInputEvidence, ...] + def __init__(self, **values: Any) -> None: ... + +class MaximumStartSafetyEvidence(NativeRecord): + """Structured evidence for an automatic-start eligibility decision.""" + + action: FutureCursorAction + max_ahead: str + invocation_time: CursorScalar + physical_target_max: CursorScalar + highest_eligible_target_max: CursorScalar | None + effective_start: CursorScalar + maximum_allowed_start: CursorScalar + target_relation: str + cursor_column: str + def __init__( + self, + *, + action: FutureCursorAction, + max_ahead: str, + invocation_time: CursorScalar | str, + physical_target_max: CursorScalar | str, + highest_eligible_target_max: CursorScalar | str | None, + effective_start: CursorScalar | str, + maximum_allowed_start: CursorScalar | str, + target_relation: str, + cursor_column: str, + ) -> None: ... + +class CursorInputRelation(NativeRecord): + """One cursor-bearing input relation for runtime range discovery.""" + + relation: str + cursor_column: str + cursor_grain: CursorGrain | None + is_model_backed: bool + is_runtime_produced: bool + terminal_cursor_start: CursorScalar | None + terminal_cursor_end: CursorScalar | None + input_name: str | None + input_kind: GraphResourceKind | None + def __init__( + self, + *, + relation: str, + cursor_column: str, + cursor_grain: CursorGrain | str | None = None, + is_model_backed: bool = False, + is_runtime_produced: bool = False, + terminal_cursor_start: CursorScalar | str | None = None, + terminal_cursor_end: CursorScalar | str | None = None, + input_name: str | None = None, + input_kind: GraphResourceKind | None = None, + ) -> None: ... + @property + def is_runtime_owned(self) -> bool: ... + +class MaximumStartPolicyInputs(NativeRecord): + """Effective policy and materialization safety inputs for automatic starts.""" + + config: StartCursorsConfig | None + invocation_time: datetime | None + incremental_strategy: str | None + incremental_mode: str | None + def __init__( + self, + config: StartCursorsConfig | None = ..., + invocation_time: datetime | None = ..., + incremental_strategy: str | None = ..., + incremental_mode: str | None = ..., + ) -> None: ... + +class SchemaFinding(NativeRecord): + """One detected schema difference between expected and warehouse columns.""" + + kind: SchemaChangeKind + column_name: str + source: SchemaColumnSource + expected_type: str | None + actual_type: str | None + def __init__( + self, + kind: SchemaChangeKind, + column_name: str, + source: SchemaColumnSource, + expected_type: str | None = ..., + actual_type: str | None = ..., + ) -> None: ... + +class RetentionPlanEntry(NativeRecord): + """Warehouse retention work independent of model identity actions.""" + + request: RetentionRequest + model_names: tuple[str, ...] + actual_days: int | None + effective_days: int | None + source: str + direction: RetentionDirection + phase: RetentionPlanPhase + statements: tuple[str, ...] + irreversible_warning: str | None + decrease_policy: str + def __init__( + self, + request: RetentionRequest, + model_names: tuple[str, ...], + actual_days: int | None, + effective_days: int | None, + source: str, + direction: RetentionDirection, + phase: RetentionPlanPhase, + statements: tuple[str, ...] = ..., + irreversible_warning: str | None = ..., + decrease_policy: str = ..., + ) -> None: ... + @property + def decreases(self) -> bool: ... + +class TableTypePlanEntry(NativeRecord): + """Snowflake table-type work independent of model identity actions.""" + + model_name: str + destination: CompiledRelationLocation + copy_name: str + desired_type: str + actual_type: str | None + source: str + downgrade: bool + downgrade_policy: str + irreversible_warning: str | None + def __init__( + self, + model_name: str, + destination: CompiledRelationLocation, + copy_name: str, + desired_type: str, + actual_type: str | None, + source: str, + downgrade: bool, + downgrade_policy: str, + irreversible_warning: str | None = ..., + ) -> None: ... + +class ModelMigrationPlanEntry(NativeRecord): + """One declared or discovered model migration and its per-run decision.""" + + model_name: str + discovery: MigrationDiscovery + decision: MigrationDecision + compatibility: MigrationCompatibility + origin_model: str | None + origin: CompiledRelationLocation + destination: CompiledRelationLocation + target_name: str | None + origin_version_hash: str + origin_is_transient: bool + stage_is_transient: bool | None + transfer: MigrationTransfer | None + transfer_fallback: MigrationTransfer | None + promotion: MigrationPromotion | None + completed_at: datetime | None + compatibility_findings: tuple[str, ...] + message: str | None + origin_tracked: bool + missing_origin_policy: MissingMigrationOriginPolicy + def __init__( + self, + model_name: str, + discovery: MigrationDiscovery, + decision: MigrationDecision, + compatibility: MigrationCompatibility, + origin_model: str | None, + origin: CompiledRelationLocation, + destination: CompiledRelationLocation, + target_name: str | None, + origin_version_hash: str = ..., + origin_is_transient: bool = ..., + stage_is_transient: bool | None = ..., + transfer: MigrationTransfer | None = ..., + transfer_fallback: MigrationTransfer | None = ..., + promotion: MigrationPromotion | None = ..., + completed_at: datetime | None = ..., + compatibility_findings: tuple[str, ...] = ..., + message: str | None = ..., + origin_tracked: bool = ..., + missing_origin_policy: MissingMigrationOriginPolicy = ..., + ) -> None: ... + @property + def origin_missing(self) -> bool: ... + @property + def origin_hidden(self) -> bool: ... + @property + def storage_transition(self) -> str | None: ... + @property + def target_label(self) -> str: ... + @property + def blocks_build(self) -> bool: ... + +class OldNameViewPlanEntry(NativeRecord): + """What one build does at the old name of one migrated model.""" + + model_name: str + origin: CompiledRelationLocation + destination: CompiledRelationLocation + action: OldNameViewAction + target_name: str | None + retention: str | None + expires_at: datetime | None + column_aliases: tuple[tuple[str, str], ...] + stores_history: bool + migration_event_id: str | None + records_requirement: bool + reason: str | None + grants_copied: int | None + grants_supported: bool + archived: bool + def __init__( + self, + model_name: str, + origin: CompiledRelationLocation, + destination: CompiledRelationLocation, + action: OldNameViewAction, + target_name: str | None, + retention: str | None = ..., + expires_at: datetime | None = ..., + column_aliases: tuple[tuple[str, str], ...] = ..., + stores_history: bool = ..., + migration_event_id: str | None = ..., + records_requirement: bool = ..., + reason: str | None = ..., + grants_copied: int | None = ..., + grants_supported: bool = ..., + archived: bool = ..., + ) -> None: ... + @property + def runs_steps(self) -> bool: ... + +class OldNameView(NativeRecord): + """One compatibility view recorded at an old name and not recorded as dropped.""" + + destination_model: str + old: CompiledRelationLocation + new: CompiledRelationLocation + expires_at: datetime | None + column_aliases: tuple[tuple[str, str], ...] + migration_event_id: str + target_name: str | None + name_reused_by: str | None + def __init__( + self, + destination_model: str, + old: CompiledRelationLocation, + new: CompiledRelationLocation, + expires_at: datetime | None, + column_aliases: tuple[tuple[str, str], ...], + migration_event_id: str, + target_name: str | None, + name_reused_by: str | None = ..., + ) -> None: ... + +class ColumnMigrationPlanEntry(NativeRecord): + """One declared or detected in-place column rename and its per-run decision.""" + + model_name: str + destination: CompiledRelationLocation + origin_column: str + destination_column: str + discovery: MigrationDiscovery + decision: ColumnMigrationDecision + target_name: str | None + completed_at: datetime | None + message: str | None + missing_origin_policy: MissingMigrationOriginPolicy + def __init__( + self, + model_name: str, + destination: CompiledRelationLocation, + origin_column: str, + destination_column: str, + discovery: MigrationDiscovery, + decision: ColumnMigrationDecision, + target_name: str | None, + completed_at: datetime | None = ..., + message: str | None = ..., + missing_origin_policy: MissingMigrationOriginPolicy = ..., + ) -> None: ... + @property + def origin_missing(self) -> bool: ... + @property + def blocks_build(self) -> bool: ... + +class ModelPlanEntry(NativeRecord): + """Per-model execution plan entry with action, reason, and resolved artifacts.""" + + key: CompiledObjectKey + name: str + relative_path: Path + materialization_type: MaterializationType + action: PlanAction + reason: PlanReason + destination: CompiledRelationLocation + fingerprint_query_sql: str + resolved_sql: str + logical_ddl: str + migration_fingerprint: str | None + incremental_strategy: str | None + incremental_mode: str | None + microbatch_strategy: str | None + cursor_watermark_mode: str | None + cursor_column: str | None + cursor_type: str | None + cursor_grain: str | None + cursor_start: str | None + cursor_end: str | None + lookback: str | None + cursor_bounds: CursorBounds | None + empty_cursor_inputs: tuple[str, ...] + waiting_on_empty_inputs: bool + cursor_input_relations: tuple[CursorInputRelation, ...] + batch_size: str | None + batch_concurrency: int + unaccounted_partition_policy: str | None + microbatch_range: CursorBounds | None + microbatch_limit: int | None + declared_microbatch_limit_action: MicrobatchLimitAction | None + microbatch_safety_limit: int | None + microbatch_safety_limit_action: MicrobatchLimitAction | None + microbatch_limit_count: int | None + microbatch_limit_action: MicrobatchLimitAction | None + microbatch_limit_warning: str | None + start_cursor_override: str | None + end_cursor_override: str | None + future_cursor_config: FutureCursorsConfig | None + start_cursor_config: StartCursorsConfig | None + invocation_time: datetime | None + unique_key: tuple[str, ...] + merge_exclude_columns: tuple[str, ...] + snapshot_strategy: str | None + updated_at_column: str | None + check_columns: tuple[str, ...] + observed_at_column: str | None + historical_input: str | None + valid_from_column: str | None + valid_to_column: str | None + initial_valid_from: str | None + invalidate_hard_deletes: bool + snapshot_full_refresh: str | None + snapshot_schema_change: str | None + table_type: str + on_schema_change: OnSchemaChange | None + type_enforcement: bool + declared_columns: tuple[ColumnInfo, ...] + contract_enforced: bool + contract_columns: tuple[ColumnInfo, ...] + contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] + pre_hooks: object + post_hooks: object + previous_query_sql: str | None + fingerprint_metadata_json: str | None + previous_metadata_json: str | None + fingerprint_version_hash: str | None + previous_version_hash: str | None + query_changed: bool + config_changed: bool + schema_actions: tuple[SchemaAction, ...] + schema_findings: tuple[SchemaFinding, ...] + backfill: BackfillResult + changed_functions: tuple[str, ...] + custom_materialization_name: str | None + custom_config: dict[str, object] + custom_placeholders: dict[str, str] + column_rename_hints: tuple[ColumnRenameHint, ...] + def __init__( + self, + key: CompiledObjectKey, + name: str, + relative_path: Path, + materialization_type: MaterializationType, + action: PlanAction, + reason: PlanReason, + destination: CompiledRelationLocation, + fingerprint_query_sql: str, + resolved_sql: str, + logical_ddl: str, + migration_fingerprint: str | None = ..., + incremental_strategy: str | None = ..., + incremental_mode: str | None = ..., + microbatch_strategy: str | None = ..., + cursor_watermark_mode: str | None = ..., + cursor_column: str | None = ..., + cursor_type: str | None = ..., + cursor_grain: str | None = ..., + cursor_start: str | None = ..., + cursor_end: str | None = ..., + lookback: str | None = ..., + cursor_bounds: CursorBounds | None = ..., + empty_cursor_inputs: tuple[str, ...] = ..., + waiting_on_empty_inputs: bool = ..., + cursor_input_relations: tuple[CursorInputRelation, ...] = ..., + batch_size: str | None = ..., + batch_concurrency: int = ..., + unaccounted_partition_policy: str | None = ..., + microbatch_range: CursorBounds | None = ..., + microbatch_limit: int | None = ..., + declared_microbatch_limit_action: MicrobatchLimitAction | None = ..., + microbatch_safety_limit: int | None = ..., + microbatch_safety_limit_action: MicrobatchLimitAction | None = ..., + microbatch_limit_count: int | None = ..., + microbatch_limit_action: MicrobatchLimitAction | None = ..., + microbatch_limit_warning: str | None = ..., + start_cursor_override: str | None = ..., + end_cursor_override: str | None = ..., + future_cursor_config: FutureCursorsConfig | None = ..., + start_cursor_config: StartCursorsConfig | None = ..., + invocation_time: datetime | None = ..., + unique_key: tuple[str, ...] = ..., + merge_exclude_columns: tuple[str, ...] = ..., + snapshot_strategy: str | None = ..., + updated_at_column: str | None = ..., + check_columns: tuple[str, ...] = ..., + observed_at_column: str | None = ..., + historical_input: str | None = ..., + valid_from_column: str | None = ..., + valid_to_column: str | None = ..., + initial_valid_from: str | None = ..., + invalidate_hard_deletes: bool = ..., + snapshot_full_refresh: str | None = ..., + snapshot_schema_change: str | None = ..., + table_type: str = ..., + on_schema_change: OnSchemaChange | None = ..., + type_enforcement: bool = ..., + declared_columns: tuple[ColumnInfo, ...] = ..., + contract_enforced: bool = ..., + contract_columns: tuple[ColumnInfo, ...] = ..., + contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ..., + pre_hooks: object = ..., + post_hooks: object = ..., + previous_query_sql: str | None = ..., + fingerprint_metadata_json: str | None = ..., + previous_metadata_json: str | None = ..., + fingerprint_version_hash: str | None = ..., + previous_version_hash: str | None = ..., + query_changed: bool = ..., + config_changed: bool = ..., + schema_actions: tuple[SchemaAction, ...] = ..., + schema_findings: tuple[SchemaFinding, ...] = ..., + backfill: BackfillResult = ..., + changed_functions: tuple[str, ...] = ..., + custom_materialization_name: str | None = ..., + custom_config: dict[str, object] = ..., + custom_placeholders: dict[str, str] = ..., + column_rename_hints: tuple[ColumnRenameHint, ...] = ..., + ) -> None: ... + +class CloneSourcePlanEntry(NativeRecord): + """One managed source relation selected for target cloning.""" + + key: CompiledObjectKey + name: str + destination: CompiledRelationLocation + def __init__( + self, + key: CompiledObjectKey, + name: str, + destination: CompiledRelationLocation, + ) -> None: ... + +class SeedPlanEntry(NativeRecord): + """Per-seed execution plan entry.""" + + key: CompiledObjectKey + name: str + destination: CompiledRelationLocation + file_path: Path + columns: tuple[ColumnInfo, ...] + csv_settings: SeedCsvSettings + fingerprint_definition: str + fingerprint_version_hash: str + fingerprint_metadata_json: str + action: PlanAction + reason: PlanReason + def __init__( + self, + key: CompiledObjectKey, + name: str, + destination: CompiledRelationLocation, + file_path: Path, + columns: tuple[ColumnInfo, ...], + csv_settings: SeedCsvSettings, + fingerprint_definition: str = ..., + fingerprint_version_hash: str = ..., + fingerprint_metadata_json: str = ..., + action: PlanAction = ..., + reason: PlanReason = ..., + ) -> None: ... + +class SourceLoadPlanEntry(NativeRecord): + """Per-managed-source loader execution plan entry.""" + + key: CompiledObjectKey + name: str + loader: str + destination: str + resource_kind: ExecutionResourceKind + write_strategy: SourceWriteStrategy | None + cursor_column: str | None + unique_key: tuple[str, ...] + is_reload: bool + integration_kind: str | None + def __init__( + self, + key: CompiledObjectKey, + name: str, + loader: str, + destination: str, + resource_kind: ExecutionResourceKind = ..., + write_strategy: SourceWriteStrategy | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + is_reload: bool = ..., + integration_kind: str | None = ..., + ) -> None: ... + +class FunctionPlanEntry(NativeRecord): + """Per-SQL-function execution plan entry.""" + + key: CompiledObjectKey + name: str + relative_path: Path + destination: CompiledRelationLocation + arguments: tuple[object, ...] + returns: str + body_sql: str + fingerprint_query_sql: str + fingerprint_destination: CompiledRelationLocation + return_columns: tuple[FunctionReturnColumn, ...] + language: FunctionLanguage + source_file_path: Path | None + runtime_version: str | None + entry_point: str | None + packages: tuple[str, ...] + previous_query_sql: str | None + reason: PlanReason + def __init__( + self, + key: CompiledObjectKey, + name: str, + relative_path: Path, + destination: CompiledRelationLocation, + arguments: tuple[object, ...], + returns: str, + body_sql: str, + fingerprint_query_sql: str, + fingerprint_destination: CompiledRelationLocation, + return_columns: tuple[FunctionReturnColumn, ...] = ..., + language: FunctionLanguage = ..., + source_file_path: Path | None = ..., + runtime_version: str | None = ..., + entry_point: str | None = ..., + packages: tuple[str, ...] = ..., + previous_query_sql: str | None = ..., + reason: PlanReason = ..., + ) -> None: ... + +class AuditPlanEntry(NativeRecord): + """Per-audit execution plan entry with resolved SQL and scheduling metadata.""" + + key: CompiledObjectKey + name: str + definition_name: str + resolved_sql: str + unresolved_sql: str + attachment_kind: AuditAttachmentKind + severity: AuditSeverity + requested_run_scope: AuditRunScope + effective_run_scope: AuditRunScope + evaluation_mode: AuditEvaluationMode + value_column: str | None + sample_count_column: str | None + sample_unit: str | None + thresholds: MeasurementThresholds | None + minimum_samples: int | None + evidence_resolved_sql: str | None + evidence_unresolved_sql: str | None + evidence_limit: int | None + scope_deps: tuple[CompiledObjectKey, ...] + attached_target_kind: AttachedAuditTargetKind | None + attached_target_name: str | None + attached_column_name: str | None + always_run: bool + description: str | None + reads_outside_target_lineage: bool + def __init__( + self, + key: CompiledObjectKey, + name: str, + definition_name: str, + resolved_sql: str, + unresolved_sql: str, + attachment_kind: AuditAttachmentKind, + severity: AuditSeverity, + requested_run_scope: AuditRunScope, + effective_run_scope: AuditRunScope, + evaluation_mode: AuditEvaluationMode = ..., + value_column: str | None = ..., + sample_count_column: str | None = ..., + sample_unit: str | None = ..., + thresholds: MeasurementThresholds | None = ..., + minimum_samples: int | None = ..., + evidence_resolved_sql: str | None = ..., + evidence_unresolved_sql: str | None = ..., + evidence_limit: int | None = ..., + scope_deps: tuple[CompiledObjectKey, ...] = ..., + attached_target_kind: AttachedAuditTargetKind | None = ..., + attached_target_name: str | None = ..., + attached_column_name: str | None = ..., + always_run: bool = ..., + description: str | None = ..., + reads_outside_target_lineage: bool = ..., + ) -> None: ... + +class ChainStep(NativeRecord): + """One step in a chained test execution sequence.""" + + model_name: str + resolved_sql: str + expected_cte_sql: str | None + lifted_ctes: tuple[tuple[str, str], ...] + comparison_body_sql: str | None + expected_columns: tuple[str, ...] | None + expected_lifted_ctes: tuple[tuple[str, str], ...] + def __init__( + self, + model_name: str, + resolved_sql: str, + expected_cte_sql: str | None = ..., + lifted_ctes: tuple[tuple[str, str], ...] = ..., + comparison_body_sql: str | None = ..., + expected_columns: tuple[str, ...] | None = ..., + expected_lifted_ctes: tuple[tuple[str, str], ...] = ..., + ) -> None: ... + +class SqlTestPlanEntry(NativeRecord): + """Per-test execution plan entry with chained resolution.""" + + key: CompiledObjectKey + name: str + source_path: Path | None + block_index: int | None + parent_name: str | None + case_name: str | None + case_index: int | None + case_fingerprint: str | None + parameter_schema: tuple[SqlTestParameterDeclaration, ...] + parameter_values: tuple[tuple[str, SqlValue], ...] + mock_ref_names: tuple[str, ...] + mock_source_names: tuple[str, ...] + mock_seed_names: tuple[str, ...] + mock_dbt_ref_names: tuple[str, ...] + mock_table_function_names: tuple[str, ...] + chain: tuple[ChainStep, ...] + assertions: tuple[SqlTestAssertionStep, ...] + scope_deps: tuple[CompiledObjectKey, ...] + function_deps: tuple[CompiledObjectKey, ...] + sql_analysis_enabled: bool + def __init__( + self, + key: CompiledObjectKey, + name: str, + source_path: Path | None = ..., + block_index: int | None = ..., + parent_name: str | None = ..., + case_name: str | None = ..., + case_index: int | None = ..., + case_fingerprint: str | None = ..., + parameter_schema: tuple[SqlTestParameterDeclaration, ...] = ..., + parameter_values: tuple[tuple[str, SqlValue], ...] = ..., + mock_ref_names: tuple[str, ...] = ..., + mock_source_names: tuple[str, ...] = ..., + mock_seed_names: tuple[str, ...] = ..., + mock_dbt_ref_names: tuple[str, ...] = ..., + mock_table_function_names: tuple[str, ...] = ..., + chain: tuple[ChainStep, ...] = ..., + assertions: tuple[SqlTestAssertionStep, ...] = ..., + scope_deps: tuple[CompiledObjectKey, ...] = ..., + function_deps: tuple[CompiledObjectKey, ...] = ..., + sql_analysis_enabled: bool = ..., + ) -> None: ... + +class ScenarioRelationMap(NativeRecord): + """Resolved scenario hash prefix and scenario-owned relation names.""" + + scenario_name: str + hash_prefix: str + artifacts: tuple[ScenarioArtifactName, ...] + def __init__( + self, + scenario_name: str, + hash_prefix: str, + artifacts: tuple[ScenarioArtifactName, ...] = ..., + ) -> None: ... + +class ScenarioRelationPlan(NativeRecord): + """Resolved scenario relation locations for model/source/seed ref resolution.""" + + scenario_name: str + relation_map: ScenarioRelationMap + model_locations: dict[str, CompiledRelationLocation] + seed_locations: dict[str, CompiledRelationLocation] + project_source_map: dict[str, SourceEntry] + source_map: dict[str, SourceEntry] + source_fixture_locations: dict[str, CompiledRelationLocation] + ref_fixture_locations: dict[str, CompiledRelationLocation] + seed_fixture_locations: dict[str, CompiledRelationLocation] + dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] + def __init__( + self, + scenario_name: str, + relation_map: ScenarioRelationMap, + model_locations: dict[str, CompiledRelationLocation] = ..., + seed_locations: dict[str, CompiledRelationLocation] = ..., + project_source_map: dict[str, SourceEntry] = ..., + source_map: dict[str, SourceEntry] = ..., + source_fixture_locations: dict[str, CompiledRelationLocation] = ..., + ref_fixture_locations: dict[str, CompiledRelationLocation] = ..., + seed_fixture_locations: dict[str, CompiledRelationLocation] = ..., + dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] = ..., + ) -> None: ... + +class ScenarioFixturePlan(NativeRecord): + """Self-contained fixture SQL planned for scenario materialization.""" + + kind: ScenarioArtifactKind + logical_name: str + destination: CompiledRelationLocation + sql: str + def __init__( + self, + kind: ScenarioArtifactKind, + logical_name: str, + destination: CompiledRelationLocation, + sql: str, + ) -> None: ... + +class ScenarioExpectedExpectationPlan(NativeRecord): + """Expected-output comparison inputs for one scenario target model.""" + + model_name: str + actual_destination: CompiledRelationLocation + expected_sql: str + def __init__( + self, + model_name: str, + actual_destination: CompiledRelationLocation, + expected_sql: str, + ) -> None: ... + +class ScenarioAssertionExpectationPlan(NativeRecord): + """Zero-row assertion SQL for one scenario assertion CTE.""" + + name: str + sql: str + def __init__( + self, + name: str, + sql: str, + ) -> None: ... + +class ScenarioExecutionPlan(NativeRecord): + """Dry-run execution plan for a SQL scenario graph slice.""" + + key: CompiledObjectKey + name: str + graph_plan: ScenarioGraphPlan + relation_plan: ScenarioRelationPlan + fixture_plans: tuple[ScenarioFixturePlan, ...] + seed_entries: tuple[SeedPlanEntry, ...] + function_entries: tuple[FunctionPlanEntry, ...] + model_entries: tuple[ModelPlanEntry, ...] + hook_functions: tuple[DiscoveredHookFunction, ...] + expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] + assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] + def __init__( + self, + key: CompiledObjectKey, + name: str, + graph_plan: ScenarioGraphPlan, + relation_plan: ScenarioRelationPlan, + fixture_plans: tuple[ScenarioFixturePlan, ...] = ..., + seed_entries: tuple[SeedPlanEntry, ...] = ..., + function_entries: tuple[FunctionPlanEntry, ...] = ..., + model_entries: tuple[ModelPlanEntry, ...] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] = ..., + assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] = ..., + ) -> None: ... + +class PlanOutput(NativeRecord): + """Complete execution plan produced by the planner.""" + + execution_order: tuple[CompiledObjectKey, ...] + model_entries: tuple[ModelPlanEntry, ...] + seed_entries: tuple[SeedPlanEntry, ...] + source_load_entries: tuple[SourceLoadPlanEntry, ...] + function_entries: tuple[FunctionPlanEntry, ...] + audit_entries: tuple[AuditPlanEntry, ...] + test_entries: tuple[SqlTestPlanEntry, ...] + available_test_case_names: tuple[str, ...] + selected_keys: frozenset[CompiledObjectKey] + warnings: tuple[PlanWarning, ...] + retention_entries: tuple[RetentionPlanEntry, ...] + table_type_entries: tuple[TableTypePlanEntry, ...] + migration_entries: tuple[ModelMigrationPlanEntry, ...] + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] + old_name_view_entries: tuple[OldNameViewPlanEntry, ...] + old_name_views: tuple[OldNameView, ...] + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + model_locations: dict[str, CompiledRelationLocation] + seed_locations: dict[str, CompiledRelationLocation] + function_locations: dict[str, CompiledRelationLocation] + source_map: dict[str, SourceEntry] + source_read_map: dict[str, SourceEntry] + python_source_read_map: dict[str, SourceEntry] + hook_functions: tuple[DiscoveredHookFunction, ...] + enforce_explicit_references: bool + provider_usages: tuple[PlanProviderUsage, ...] + source_freshness: DirectSourceFreshnessPlanningResult | None + python_identity_fingerprints: dict[tuple[str, str], Fingerprint] + metadata: dict[str, object] + cursor_overrides: CursorOverrides | None + def __init__( + self, + execution_order: tuple[CompiledObjectKey, ...] = ..., + model_entries: tuple[ModelPlanEntry, ...] = ..., + seed_entries: tuple[SeedPlanEntry, ...] = ..., + source_load_entries: tuple[SourceLoadPlanEntry, ...] = ..., + function_entries: tuple[FunctionPlanEntry, ...] = ..., + audit_entries: tuple[AuditPlanEntry, ...] = ..., + test_entries: tuple[SqlTestPlanEntry, ...] = ..., + available_test_case_names: tuple[str, ...] = ..., + selected_keys: frozenset[CompiledObjectKey] = ..., + warnings: tuple[PlanWarning, ...] = ..., + retention_entries: tuple[RetentionPlanEntry, ...] = ..., + table_type_entries: tuple[TableTypePlanEntry, ...] = ..., + migration_entries: tuple[ModelMigrationPlanEntry, ...] = ..., + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = ..., + old_name_view_entries: tuple[OldNameViewPlanEntry, ...] = ..., + old_name_views: tuple[OldNameView, ...] = ..., + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ..., + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ..., + model_locations: dict[str, CompiledRelationLocation] = ..., + seed_locations: dict[str, CompiledRelationLocation] = ..., + function_locations: dict[str, CompiledRelationLocation] = ..., + source_map: dict[str, SourceEntry] = ..., + source_read_map: dict[str, SourceEntry] = ..., + python_source_read_map: dict[str, SourceEntry] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + enforce_explicit_references: bool = ..., + provider_usages: tuple[PlanProviderUsage, ...] = ..., + source_freshness: DirectSourceFreshnessPlanningResult | None = ..., + python_identity_fingerprints: dict[tuple[str, str], Fingerprint] = ..., + metadata: dict[str, object] = ..., + cursor_overrides: CursorOverrides | None = ..., + ) -> None: ... + @property + def python_source_entries(self) -> dict[str, SourceEntry]: ... + +class CompileInputError(ValueError): + """Raised when discovered inputs cannot be attached into a compile view.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class MacroArgumentError(CompileInputError): + """Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.""" + + detail: str + macro_name: str + file_path: Path + offset: int + relative_position: tuple[int, int] + location: SourceLocation | None + def __init__( + self, + *, + detail: str, + help: str, + macro_name: str, + file_path: Path, + offset: int, + relative_position: tuple[int, int], + location: SourceLocation | None = None, + ) -> None: ... + def shifted(self, by: int) -> MacroArgumentError: ... + def located(self, location: SourceLocation) -> MacroArgumentError: ... + +class SqlTestReferenceError(CompileInputError): + """Raised when a SQL-test CTE calls a reference the test query cannot resolve.""" + + location: SourceLocation + def __init__(self, message: str, *, location: SourceLocation, help: str) -> None: ... + +class RulesError(RuntimeError): + """Raised for invalid compiler rules or execution.""" + +class RulesConfigError(RulesError): + """Raised when rules configuration breaks a project policy; help shows the setting to change.""" + + message: str + help: str + def __init__(self, message: str, *, help: str) -> None: ... + +class RuleUsageError(RulesError): + """Raised when a rule or rule test uses the public API incorrectly.""" + +class NonHermeticRuleError(RulesError): + """Raised when a custom rule observes state the Rules cache cannot track.""" + +class RuleAssertionError(RulesError): + """Raised when a custom-rule harness case produces an unexpected result.""" + +class FactDigestError(RulesError): + """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" + +class OpaqueModuleStateError(RulesError): + """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" + +class HostCancelledError(RulesError): + """Raised in a custom-rule host that stops because another host already failed.""" + +class SqlReferenceKind(StrEnum): + REF = "ref" + SEED = "seed" + SOURCE = "source" + DBT_REF = "dbt_ref" + UDF = "udf" + TABLE_FUNCTION = "table_fn" + @property + def function_name(self) -> str: ... + @property + def fixture_cte_prefix(self) -> str: ... + def example_call(self, *args: str, quote: Literal["'", '"'] = "'") -> str: ... + def placeholder_call(self, placeholder: str = "") -> str: ... + +class HardCodedRelationOwnerKind(StrEnum): + """The kind of Python code that names a project relation in SQL.""" + + NODE = "node" + HOOK = "hook" + LOADER = "loader" + +class LoaderConnectionMode(StrEnum): + """How a source loader owns the destination warehouse connection.""" + + SQLBUILD = "sqlbuild" + EXTERNAL = "external" + +class SqlHookEntry(NativeRecord): + """A model lifecycle hook that executes SQL.""" + + statement: str + name: str | None + relative_path: Path | None + definition_sql: str | None + kwargs: dict[str, object] | None + description: str | None + reads: tuple[SqlResourceRef, ...] + def __init__( + self, + statement: str, + name: str | None = ..., + relative_path: Path | None = ..., + definition_sql: str | None = ..., + kwargs: dict[str, object] | None = ..., + description: str | None = ..., + reads: tuple[SqlResourceRef, ...] = ..., + ) -> None: ... + +class PythonHookEntry(NativeRecord): + """A model lifecycle hook that invokes a discovered Python hook.""" + + name: str + kwargs: dict[str, object] + def __init__( + self, + name: str, + kwargs: dict[str, object], + ) -> None: ... + +class EnumDeclaration(NativeRecord): + """One validated public or model-local enum declaration.""" + + name: str + members: tuple[EnumMember, ...] + scalar_type: str + relative_path: Path + model_name: str | None + def __init__( + self, + name: str, + members: tuple[EnumMember, ...], + scalar_type: str, + relative_path: Path, + model_name: str | None = ..., + ) -> None: ... + +class ConstantDeclaration(NativeRecord): + """One validated public or model-local constant declaration.""" + + name: str + value: SqlValue + relative_path: Path + model_name: str | None + render_as: CollectionRendering | None + def __init__( + self, + name: str, + value: SqlValue, + relative_path: Path, + model_name: str | None = ..., + render_as: CollectionRendering | None = ..., + ) -> None: ... + @property + def logical_type(self) -> SqlLogicalType: ... + @property + def scalar_type(self) -> str: ... + +class DiscoveredSourceFile(NativeRecord): + """A discovered source declaration file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + source_entries: tuple[SourceEntry, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + source_entries: tuple[SourceEntry, ...], + ) -> None: ... + +class DiscoveredSqlTestFile(NativeRecord): + """A discovered SQL-native test file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + blocks: tuple[DiscoveredSqlTestBlock, ...] + ownership_root: Path + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + blocks: tuple[DiscoveredSqlTestBlock, ...], + ownership_root: Path = ..., + ) -> None: ... + +class SqlTestParameterDeclaration(NativeRecord): + """One typed parameter declared by a SQL-native test template.""" + + name: str + value_type: SqlValueKind + nullable: bool + def __init__( + self, + name: str, + value_type: SqlValueKind, + nullable: bool = ..., + ) -> None: ... + +class DiscoveredSqlScenarioFile(NativeRecord): + """A discovered SQL-native scenario file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + header_values: dict[str, object] + sql_body: str + name: str + ownership_root: Path + sql_body_span: tuple[int, int] | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + header_values: dict[str, object], + sql_body: str, + name: str, + ownership_root: Path = ..., + sql_body_span: tuple[int, int] | None = ..., + ) -> None: ... + +class DiscoveredMacroFile(NativeRecord): + """A discovered project macro file and its raw contents.""" + + file_path: Path + relative_path: Path + contents: str + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredMaterializationFile(NativeRecord): + """A discovered custom materialization Python file.""" + + file_path: Path + relative_path: Path + name: str + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredProviderUsage(NativeRecord): + """Framework-owned metadata for provider usage by a Python callable surface.""" + + provider_name: str + parameter_name: str + annotation_class_name: str | None + annotation_module: str | None + def __init__( + self, + provider_name: str, + parameter_name: str, + annotation_class_name: str | None = ..., + annotation_module: str | None = ..., + ) -> None: ... + +class DiscoveredLoaderFunction(NativeRecord): + """A discovered project source loader function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + destination: str | None + write_strategy: SourceWriteStrategy | None + cursor_column: str | None + unique_key: tuple[str, ...] + columns: tuple[SourceColumnEntry, ...] + contract: str | None + description: str | None + connection_mode: LoaderConnectionMode + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + destination: str | None = ..., + write_strategy: SourceWriteStrategy | None = ..., + cursor_column: str | None = ..., + unique_key: tuple[str, ...] = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + contract: str | None = ..., + description: str | None = ..., + connection_mode: LoaderConnectionMode = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredTaskFunction(NativeRecord): + """A discovered project task function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + retry: RetryPolicy | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + retry: RetryPolicy | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredAssetFunction(NativeRecord): + """A discovered project asset function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + columns: tuple[SourceColumnEntry, ...] + column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None + retry: RetryPolicy | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + columns: tuple[SourceColumnEntry, ...] = ..., + column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = ..., + retry: RetryPolicy | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredCheckFunction(NativeRecord): + """A discovered project check function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] + severity: PythonCheckSeverity + tags: tuple[str, ...] + group: str | None + description: str | None + meta: dict[str, object] | None + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + depends_on: tuple[Callable[..., object] | SqlResourceRef, ...], + severity: PythonCheckSeverity = ..., + tags: tuple[str, ...] = ..., + group: str | None = ..., + description: str | None = ..., + meta: dict[str, object] | None = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredHookFunction(NativeRecord): + """A discovered project model lifecycle hook function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + description: str | None + reads: tuple[SqlResourceRef, ...] + provider_usages: tuple[DiscoveredProviderUsage, ...] + scope_kind: ScopeKind + ownership_root: Path | None + owning_path: Path | None + declaration_root: Path | None + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + description: str | None = ..., + reads: tuple[SqlResourceRef, ...] = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + scope_kind: ScopeKind = ..., + ownership_root: Path | None = ..., + owning_path: Path | None = ..., + declaration_root: Path | None = ..., + ) -> None: ... + +class DiscoveredAuditFactory(NativeRecord): + """A discovered audit factory and its normalized cases.""" + + name: str + function: Callable[..., object] + file_path: Path + relative_path: Path + line: int + cases: tuple[AuditCase, ...] + def __init__( + self, + name: str, + function: Callable[..., object], + file_path: Path, + relative_path: Path, + line: int, + cases: tuple[AuditCase, ...] = ..., + ) -> None: ... + +class DiscoveredEventExporterDeclaration(NativeRecord): + """A discovered exporter declaration before provider binding.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + event_kinds: frozenset[str] + min_severity: EventExportSeverity + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + event_kinds: frozenset[str], + min_severity: EventExportSeverity, + ) -> None: ... + +class DiscoveredEventExporter(NativeRecord): + """A discovered canonical lifecycle event exporter function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + event_kinds: frozenset[str] + min_severity: EventExportSeverity + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + event_kinds: frozenset[str] = ..., + min_severity: EventExportSeverity = ..., + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredCommandOutputSinkDeclaration(NativeRecord): + """A command-output sink declaration before provider binding.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + streams: frozenset[CommandOutputStream] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + streams: frozenset[CommandOutputStream], + ) -> None: ... + +class DiscoveredCommandOutputSink(NativeRecord): + """A discovered command-output sink function.""" + + file_path: Path + relative_path: Path + name: str + function: Callable[..., object] + streams: frozenset[CommandOutputStream] + provider_usages: tuple[DiscoveredProviderUsage, ...] + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + function: Callable[..., object], + streams: frozenset[CommandOutputStream], + provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + ) -> None: ... + +class DiscoveredProvider(NativeRecord): + """A discovered project provider class and validated settings object.""" + + file_path: Path + relative_path: Path + name: str + provider_class: type[ProjectProvider] + settings: ProjectProvider + def __init__( + self, + file_path: Path, + relative_path: Path, + name: str, + provider_class: type[ProjectProvider], + settings: ProjectProvider, + ) -> None: ... + +class DiscoveredPythonNodeFunctions(NativeRecord): + """Discovered project Python DAG node functions grouped by kind.""" + + loaders: tuple[DiscoveredLoaderFunction, ...] + tasks: tuple[DiscoveredTaskFunction, ...] + assets: tuple[DiscoveredAssetFunction, ...] + checks: tuple[DiscoveredCheckFunction, ...] + audit_factories: tuple[DiscoveredAuditFactory, ...] + def __init__( + self, + loaders: tuple[DiscoveredLoaderFunction, ...] = ..., + tasks: tuple[DiscoveredTaskFunction, ...] = ..., + assets: tuple[DiscoveredAssetFunction, ...] = ..., + checks: tuple[DiscoveredCheckFunction, ...] = ..., + audit_factories: tuple[DiscoveredAuditFactory, ...] = ..., + ) -> None: ... + +class DiscoveredProjectInputs(NativeRecord): + """All raw project inputs discovered from disk before semantic resolution.""" + + project_config: ProjectConfig + local_config: LocalConfig + project_dir: Path | None + model_files: tuple[DiscoveredSqlModelFile, ...] + enum_files: tuple[DiscoveredEnumFile, ...] + constant_files: tuple[DiscoveredConstantFile, ...] + model_schema_files: tuple[DiscoveredModelSchemaFile, ...] + sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] + python_function_files: tuple[DiscoveredPythonFunctionFile, ...] + schema_files: tuple[DiscoveredSchemaFile, ...] + source_files: tuple[DiscoveredSourceFile, ...] + seed_files: tuple[DiscoveredSeedFile, ...] + test_files: tuple[DiscoveredSqlTestFile, ...] + scenario_files: tuple[DiscoveredSqlScenarioFile, ...] + audit_files: tuple[DiscoveredAuditFile, ...] + macro_files: tuple[DiscoveredMacroFile, ...] + materialization_files: tuple[DiscoveredMaterializationFile, ...] + loader_functions: tuple[DiscoveredLoaderFunction, ...] + task_functions: tuple[DiscoveredTaskFunction, ...] + asset_functions: tuple[DiscoveredAssetFunction, ...] + check_functions: tuple[DiscoveredCheckFunction, ...] + audit_factories: tuple[DiscoveredAuditFactory, ...] + hook_functions: tuple[DiscoveredHookFunction, ...] + event_exporters: tuple[DiscoveredEventExporter, ...] + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] + providers: tuple[DiscoveredProvider, ...] + adapter_file: DiscoveredAdapterFile | None + native_session: NativeDiscoverySession | None + def __init__( + self, + project_config: ProjectConfig, + local_config: LocalConfig, + project_dir: Path | None = ..., + model_files: tuple[DiscoveredSqlModelFile, ...] = ..., + enum_files: tuple[DiscoveredEnumFile, ...] = ..., + constant_files: tuple[DiscoveredConstantFile, ...] = ..., + model_schema_files: tuple[DiscoveredModelSchemaFile, ...] = ..., + sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] = ..., + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = ..., + python_function_files: tuple[DiscoveredPythonFunctionFile, ...] = ..., + schema_files: tuple[DiscoveredSchemaFile, ...] = ..., + source_files: tuple[DiscoveredSourceFile, ...] = ..., + seed_files: tuple[DiscoveredSeedFile, ...] = ..., + test_files: tuple[DiscoveredSqlTestFile, ...] = ..., + scenario_files: tuple[DiscoveredSqlScenarioFile, ...] = ..., + audit_files: tuple[DiscoveredAuditFile, ...] = ..., + macro_files: tuple[DiscoveredMacroFile, ...] = ..., + materialization_files: tuple[DiscoveredMaterializationFile, ...] = ..., + loader_functions: tuple[DiscoveredLoaderFunction, ...] = ..., + task_functions: tuple[DiscoveredTaskFunction, ...] = ..., + asset_functions: tuple[DiscoveredAssetFunction, ...] = ..., + check_functions: tuple[DiscoveredCheckFunction, ...] = ..., + audit_factories: tuple[DiscoveredAuditFactory, ...] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + event_exporters: tuple[DiscoveredEventExporter, ...] = ..., + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ..., + providers: tuple[DiscoveredProvider, ...] = ..., + adapter_file: DiscoveredAdapterFile | None = ..., + native_session: NativeDiscoverySession | None = ..., + ) -> None: ... + +class DiscoveredRuntimeExtensions(NativeRecord): + """Command-startup providers, typed sinks, and lifecycle delivery settings.""" + + providers: tuple[DiscoveredProvider, ...] + event_exporters: tuple[DiscoveredEventExporter, ...] + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] + lifecycle_shutdown_timeout_seconds: int | None + def __init__( + self, + providers: tuple[DiscoveredProvider, ...] = ..., + event_exporters: tuple[DiscoveredEventExporter, ...] = ..., + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ..., + lifecycle_shutdown_timeout_seconds: int | None = ..., + ) -> None: ... + +class ProjectGraph(NativeRecord): + """Static compiled project graph held natively; dict views are built only when read.""" + + project: CompiledProjectView + native: NativeProjectGraph + def __init__( + self, + project: CompiledProjectView, + native: NativeProjectGraph, + ) -> None: ... + @property + def lineage_views(self) -> LineageGraphViews: ... + @property + def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... + @property + def downstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... + @property + def tag_index(self) -> dict[str, frozenset[CompiledObjectKey]]: ... + @property + def path_index(self) -> dict[CompiledObjectKey, str]: ... + @property + def all_keys(self) -> dict[str, CompiledObjectKey]: ... + +class CompilePipelineResult(NativeRecord): + """Complete output from the compile-and-plan pipeline.""" + + project: CompiledProjectView + plan_output: PlanOutput + custom_materializations: dict[str, Callable[..., Any]] + python_node_names: frozenset[str] + python_plan_entries: tuple[PythonPlanEntry, ...] + compile_seconds: float | None + planning_seconds: float | None + def __init__( + self, + project: CompiledProjectView, + plan_output: PlanOutput, + custom_materializations: dict[str, Callable[..., Any]] = ..., + python_node_names: frozenset[str] = ..., + python_plan_entries: tuple[PythonPlanEntry, ...] = ..., + compile_seconds: float | None = ..., + planning_seconds: float | None = ..., + ) -> None: ... + +class CompilerDiagnostic(NativeRecord): + """One project diagnostic produced by compile-time checks.""" + + phase: DiagnosticPhase | str + severity: DiagnosticSeverity | str + code: str + message: str + resource_type: CompiledResourceType | str | None + resource_name: str | None + column_name: str | None + path: Path | None + line: int | None + column: int | None + location: SourceLocation | None + related_locations: tuple[RelatedLocation, ...] + help: str | None + notes: tuple[str, ...] + affected_rules: tuple[str, ...] + def __init__( + self, + phase: DiagnosticPhase | str, + severity: DiagnosticSeverity | str, + code: str, + message: str, + resource_type: CompiledResourceType | str | None = ..., + resource_name: str | None = ..., + column_name: str | None = ..., + path: Path | None = ..., + line: int | None = ..., + column: int | None = ..., + location: SourceLocation | None = ..., + related_locations: tuple[RelatedLocation, ...] = ..., + help: str | None = ..., + notes: tuple[str, ...] = ..., + affected_rules: tuple[str, ...] = ..., + ) -> None: ... + @property + def is_error(self) -> bool: ... + +class InferredColumn(NativeRecord): + """One output column inferred from query SQL via sql_analysis parsing.""" + + name: str + type: str | None + nullability: InferredNullability + def __init__( + self, + name: str, + type: str | None = ..., + nullability: InferredNullability = ..., + ) -> None: ... + +class LoadedMacro(NativeRecord): + """One loaded project macro available for compile-time SQL expansion.""" + + name: str + file_path: Path + relative_path: Path + raw_source: str + function: Callable[..., object] + dependencies: tuple[DeclarationIdentity, ...] + def __init__( + self, + name: str, + file_path: Path, + relative_path: Path, + raw_source: str, + function: Callable[..., object], + dependencies: tuple[DeclarationIdentity, ...] = ..., + ) -> None: ... + @property + def injects_context(self) -> bool: ... + +class StaticMacroExport(NativeRecord): + """Value-free AST inventory for one project-owned macro export.""" + + name: str + relative_path: Path + parameters: tuple[str, ...] + line: int + source_digest: str + dependencies: tuple[DeclarationIdentity, ...] + def __init__( + self, + name: str, + relative_path: Path, + parameters: tuple[str, ...], + line: int, + source_digest: str, + dependencies: tuple[DeclarationIdentity, ...] = ..., + ) -> None: ... + +class StaticMacroFault(NativeRecord): + """One value-free static macro inventory fault.""" + + relative_path: Path + message: str + def __init__( + self, + relative_path: Path, + message: str, + ) -> None: ... + +class StaticMacroInventory(NativeRecord): + """AST macro exports and faults collected without module execution.""" + + exports: tuple[StaticMacroExport, ...] + faults: tuple[StaticMacroFault, ...] + def __init__( + self, + exports: tuple[StaticMacroExport, ...] = ..., + faults: tuple[StaticMacroFault, ...] = ..., + ) -> None: ... + +class MacroContext(NativeRecord): + """Expose target settings and caller-visible declarations to a Python SQL macro.""" + + adapter_name: str + sql_analysis_enabled: bool + target_name: str | None + vars: dict[str, object] + constants: Mapping[str, object] + enums: Mapping[str, Mapping[str, str | int]] + _value_renderer: TypedSqlValueRenderer | None + _collection_rendering: CollectionRendering + _constant_declarations: Mapping[str, ConstantDeclaration] + _enforce_explicit_references: bool + def __init__( + self, + adapter_name: str, + sql_analysis_enabled: bool, + target_name: str | None, + vars: dict[str, object] = ..., + constants: Mapping[str, object] = ..., + enums: Mapping[str, Mapping[str, str | int]] = ..., + _value_renderer: TypedSqlValueRenderer | None = ..., + _collection_rendering: CollectionRendering = ..., + _constant_declarations: Mapping[str, ConstantDeclaration] = ..., + _enforce_explicit_references: bool = ..., + ) -> None: ... + def render_constant(self, name: str) -> str: ... + def render_enum_member(self, *, enum_name: str, member_name: str) -> str: ... + +class DeclarationResolutionContext(NativeRecord): + """Declarations visible during one authored SQL expansion.""" + + enums: dict[str, EnumDeclaration] + constants: dict[str, ConstantDeclaration] + inaccessible_enums: dict[str, DeclarationRecord] + inaccessible_constants: dict[str, DeclarationRecord] + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] + macros: dict[str, LoadedMacro] + macro_records: dict[str, DeclarationRecord] + macro_visibility: dict[str, tuple[VisibilityRecord, ...]] + inaccessible_macros: dict[str, DeclarationRecord] + consumer: ResourceIdentity | DeclarationIdentity | None + def __init__( + self, + enums: dict[str, EnumDeclaration] = ..., + constants: dict[str, ConstantDeclaration] = ..., + inaccessible_enums: dict[str, DeclarationRecord] = ..., + inaccessible_constants: dict[str, DeclarationRecord] = ..., + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] = ..., + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] = ..., + macros: dict[str, LoadedMacro] = ..., + macro_records: dict[str, DeclarationRecord] = ..., + macro_visibility: dict[str, tuple[VisibilityRecord, ...]] = ..., + inaccessible_macros: dict[str, DeclarationRecord] = ..., + consumer: ResourceIdentity | DeclarationIdentity | None = ..., + ) -> None: ... + +class ModelInputBuildContext(NativeRecord): + """Run-constant config and macros for building model compile inputs.""" + + effective_vars: dict[str, object] + effective_settings: SettingsConfig + target_config: TargetConfig | None + effective_target_name: str | None + run_id: str + macro_context: MacroContext + loaded_macros: dict[str, LoadedMacro] + value_renderer: TypedSqlValueRenderer + collection_rendering: CollectionRendering + sql_lexical_syntax: SqlLexicalSyntax + public_enums: dict[str, EnumDeclaration] + public_constants: dict[str, ConstantDeclaration] + public_model_schemas: dict[str, ModelSchemaDeclaration] + declaration_resolver: DeclarationScopeResolver | None + def __init__( + self, + effective_vars: dict[str, object], + effective_settings: SettingsConfig, + target_config: TargetConfig | None, + effective_target_name: str | None, + run_id: str, + macro_context: MacroContext, + loaded_macros: dict[str, LoadedMacro], + value_renderer: TypedSqlValueRenderer, + collection_rendering: CollectionRendering, + sql_lexical_syntax: SqlLexicalSyntax, + public_enums: dict[str, EnumDeclaration] = ..., + public_constants: dict[str, ConstantDeclaration] = ..., + public_model_schemas: dict[str, ModelSchemaDeclaration] = ..., + declaration_resolver: DeclarationScopeResolver | None = ..., + ) -> None: ... + @property + def declaration_expansion(self) -> DeclarationExpansionContext: ... + +class CompileSqlReference(NativeRecord): + """One logical SQL reference discovered from compiled SQL text.""" + + ref_kind: SqlReferenceKind | str + ref_name: str + ref_package: str | None + call_argument_count: int | None + def __init__( + self, + ref_kind: SqlReferenceKind | str, + ref_name: str, + ref_package: str | None = ..., + call_argument_count: int | None = ..., + ) -> None: ... + +class CompiledLineageSourceFact(NativeRecord): + """Compact upstream column fact extracted during SQL analysis.""" + + resource_type: CompiledResourceType | str + resource_name: str + column_name: str + def __init__( + self, + resource_type: CompiledResourceType | str, + resource_name: str, + column_name: str, + ) -> None: ... + +class CompactLineageFacts(NativeRecord, Sequence[CompiledLineageColumnFact]): + """Indexed native lineage rows with lazy object projection.""" + + string_pool: tuple[str, ...] + rows: tuple[tuple[int, int, int, tuple[tuple[int, int, int], ...]], ...] + resource_name_indexes: dict[int, int] + resource_names: dict[int, str] + def __init__( + self, + string_pool: tuple[str, ...], + rows: tuple[tuple[int, int, int, tuple[tuple[int, int, int], ...]], ...], + resource_name_indexes: dict[int, int] = ..., + resource_names: dict[int, str] = ..., + ) -> None: ... + @property + def facts(self) -> tuple[CompiledLineageColumnFact, ...]: ... + def __len__(self) -> int: ... + @overload + def __getitem__(self, index: int) -> CompiledLineageColumnFact: ... + @overload + def __getitem__(self, index: slice) -> Sequence[CompiledLineageColumnFact]: ... + def __iter__(self) -> Iterator[CompiledLineageColumnFact]: ... + def __eq__(self, other: object) -> bool: ... + def resource_name(self, index: int) -> str: ... + +class CompiledObjectKey(NativeRecord): + """Stable logical identity for one compiled resource or external dependency.""" + + resource_type: CompiledResourceType | str + name: str + def __init__( + self, + resource_type: CompiledResourceType | str, + name: str, + ) -> None: ... + +class CompileModelInput(NativeRecord): + """One discovered model file with its attached schema metadata, if any.""" + + model_file: DiscoveredSqlModelFile + config: CompileModelConfig + query_sql: str + macro_source_sql: str + references: tuple[CompileSqlReference, ...] + schema_entry: SchemaModelEntry | None + schema_file: DiscoveredSchemaFile | None + sql_validation_enabled: bool + rejected_sql_analysis_opt_out: SourceLocation | None + enum_declarations: tuple[EnumDeclaration, ...] + constant_declarations: tuple[ConstantDeclaration, ...] + enum_columns: dict[str, EnumDeclaration] + macro_deps: tuple[str, ...] + macro_usages: tuple[UsageRecord, ...] + declaration_usages: tuple[UsageRecord, ...] + sql_expansion: CompiledSqlExpansion | None + lint_expansion: CompiledSqlExpansion | None + def __init__( + self, + model_file: DiscoveredSqlModelFile, + config: CompileModelConfig = ..., + query_sql: str = ..., + macro_source_sql: str = ..., + references: tuple[CompileSqlReference, ...] = ..., + schema_entry: SchemaModelEntry | None = ..., + schema_file: DiscoveredSchemaFile | None = ..., + sql_validation_enabled: bool = ..., + rejected_sql_analysis_opt_out: SourceLocation | None = ..., + enum_declarations: tuple[EnumDeclaration, ...] = ..., + constant_declarations: tuple[ConstantDeclaration, ...] = ..., + enum_columns: dict[str, EnumDeclaration] = ..., + macro_deps: tuple[str, ...] = ..., + macro_usages: tuple[UsageRecord, ...] = ..., + declaration_usages: tuple[UsageRecord, ...] = ..., + sql_expansion: CompiledSqlExpansion | None = ..., + lint_expansion: CompiledSqlExpansion | None = ..., + ) -> None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def attached_schema_name(self) -> str | None: ... + +class CompiledRelationLocation(NativeRecord): + """Logical and physical relation location resolved during compile.""" + + database: str | None + schema: str | None + name: str + qualified_name: str | None + logical_schema: str | None + logical_database: str | None + def __init__( + self, + database: str | None, + schema: str | None, + name: str, + qualified_name: str | None, + logical_schema: str | None = ..., + logical_database: str | None = ..., + ) -> None: ... + +class CompiledModel(NativeRecord): + """Planner-ready compiled model metadata.""" + + key: CompiledObjectKey + deps: tuple[CompiledObjectKey, ...] + name: str + relative_path: Path + query_sql: str + config: CompileModelConfig + destination: CompiledRelationLocation + references: tuple[CompileSqlReference, ...] + schema_entry: SchemaModelEntry | None + inferred_columns: tuple[InferredColumn, ...] | None + fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None + fast_lineage_has_star: bool + fast_lineage_star_resolved: bool + authored_sql: str + authored_query_sql: str + output_column_locations: dict[str, SourceLocation] + extract_implicit_alias_columns: bool + macro_deps: tuple[str, ...] + enum_declarations: tuple[EnumDeclaration, ...] + constant_declarations: tuple[ConstantDeclaration, ...] + enum_columns: dict[str, EnumDeclaration] + binding_diagnostics: tuple[CompilerDiagnostic, ...] + binding_validated: bool + dynamic_column_contract: DynamicColumnContractProof | None + unchecked_output_columns: frozenset[str] + rejected_sql_analysis_opt_out: SourceLocation | None + def __init__( + self, + key: CompiledObjectKey, + deps: tuple[CompiledObjectKey, ...], + name: str, + relative_path: Path, + query_sql: str, + config: CompileModelConfig, + destination: CompiledRelationLocation, + references: tuple[CompileSqlReference, ...] = ..., + schema_entry: SchemaModelEntry | None = ..., + inferred_columns: tuple[InferredColumn, ...] | None = ..., + fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = ..., + fast_lineage_has_star: bool = ..., + fast_lineage_star_resolved: bool = ..., + authored_sql: str = ..., + authored_query_sql: str = ..., + output_column_locations: dict[str, SourceLocation] = ..., + extract_implicit_alias_columns: bool = ..., + macro_deps: tuple[str, ...] = ..., + enum_declarations: tuple[EnumDeclaration, ...] = ..., + constant_declarations: tuple[ConstantDeclaration, ...] = ..., + enum_columns: dict[str, EnumDeclaration] = ..., + binding_diagnostics: tuple[CompilerDiagnostic, ...] = ..., + binding_validated: bool = ..., + dynamic_column_contract: DynamicColumnContractProof | None = ..., + unchecked_output_columns: frozenset[str] = ..., + rejected_sql_analysis_opt_out: SourceLocation | None = ..., + ) -> None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def key_row(self) -> tuple[str, str]: ... + @property + def dep_rows(self) -> list[tuple[str, str]]: ... + @property + def destination_row(self) -> tuple[str | None, str | None, str]: ... + @property + def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: ... + @property + def schema_column_rows(self) -> list[tuple[str, str | None]] | None: ... + @property + def has_dynamic_columns(self) -> bool: ... + @property + def output_column_count(self) -> int: ... + +class CompiledAudit(NativeRecord): + """Compiled audit metadata selected by scope dependencies.""" + + key: CompiledObjectKey + scope_deps: tuple[CompiledObjectKey, ...] + name: str + definition_name: str + audit_file: DiscoveredAuditFile + audit_block: DiscoveredAuditBlock + sql_body: str + evaluation_mode: AuditEvaluationMode + measurement_contract: MeasurementContract | None + thresholds: MeasurementThresholds | None + minimum_samples: int | None + measure_sql: str | None + evidence_sql: str | None + evidence_limit: int | None + references: tuple[CompileSqlReference, ...] + attached_target_kind: AttachedAuditTargetKind | None + attached_target_name: str | None + attached_column_name: str | None + severity: AuditSeverity | None + run_scope: str | None + always_run: bool + description: str | None + def __init__( + self, + key: CompiledObjectKey, + scope_deps: tuple[CompiledObjectKey, ...], + name: str, + definition_name: str, + audit_file: DiscoveredAuditFile, + audit_block: DiscoveredAuditBlock, + sql_body: str, + evaluation_mode: AuditEvaluationMode = ..., + measurement_contract: MeasurementContract | None = ..., + thresholds: MeasurementThresholds | None = ..., + minimum_samples: int | None = ..., + measure_sql: str | None = ..., + evidence_sql: str | None = ..., + evidence_limit: int | None = ..., + references: tuple[CompileSqlReference, ...] = ..., + attached_target_kind: AttachedAuditTargetKind | None = ..., + attached_target_name: str | None = ..., + attached_column_name: str | None = ..., + severity: AuditSeverity | None = ..., + run_scope: str | None = ..., + always_run: bool = ..., + description: str | None = ..., + ) -> None: ... + +class CompiledSqlScenario(NativeRecord): + """Compiled SQL-native scenario metadata selected by inferred graph targets.""" + + key: CompiledObjectKey + name: str + scenario_file: DiscoveredSqlScenarioFile + sql_body: str + authored_ctes: tuple[CompileSqlScenarioCte, ...] + expected_ctes: tuple[CompileSqlScenarioCte, ...] + assertion_ctes: tuple[CompileSqlScenarioCte, ...] + source_fixture_names: tuple[str, ...] + ref_fixture_names: tuple[str, ...] + seed_fixture_names: tuple[str, ...] + dbt_ref_fixture_names: tuple[str, ...] + expected_model_names: tuple[str, ...] + assertion_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + target_model_names: tuple[str, ...] + source_path: Path | None + ownership_root: Path | None + def __init__( + self, + key: CompiledObjectKey, + name: str, + scenario_file: DiscoveredSqlScenarioFile, + sql_body: str, + authored_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + expected_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + assertion_ctes: tuple[CompileSqlScenarioCte, ...] = ..., + source_fixture_names: tuple[str, ...] = ..., + ref_fixture_names: tuple[str, ...] = ..., + seed_fixture_names: tuple[str, ...] = ..., + dbt_ref_fixture_names: tuple[str, ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + target_model_names: tuple[str, ...] = ..., + source_path: Path | None = ..., + ownership_root: Path | None = ..., + ) -> None: ... + +class CompiledProject(NativeRecord): + """Planner-ready whole-project compile output.""" + + run_id: str + effective_target_name: str | None + effective_connection: dict[str, object] + effective_vars: dict[str, object] + binding_catalog: Any | None + effective_target_database: str | None + effective_target_schema: str | None + sql_analysis_dialect: str | None + sql_lexical_syntax: SqlLexicalSyntax + compile_cache_dir: Path | None + settings: SettingsConfig + scenario: ScenarioConfig + models: tuple[CompiledModelView, ...] + sources: tuple[CompiledSourceView, ...] + seeds: tuple[CompiledSeedView, ...] + functions: tuple[CompiledFunctionView, ...] + audits: tuple[CompiledAuditView, ...] + sql_tests: tuple[CompiledSqlTest, ...] + sql_scenarios: tuple[CompiledSqlScenario, ...] + loader_functions: tuple[DiscoveredLoaderFunction, ...] + hook_functions: tuple[DiscoveredHookFunction, ...] + enforce_explicit_references: bool + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] + materialization_files: tuple[DiscoveredMaterializationFile, ...] + public_enums: dict[str, EnumDeclaration] + public_constants: dict[str, ConstantDeclaration] + loaded_macros: dict[str, LoadedMacro] + diagnostics: tuple[CompilerDiagnostic, ...] + unmatched_literal_sql_relations: tuple[LiteralSqlRelation, ...] + external_sql_reference_resolver: ExternalSqlReferenceResolver | None + scope_index: ScopeIndex + sql_expansions: dict[Path, CompiledSqlExpansion] + native_session: Any | None + native_project: NativeCompiledProject | None + def __init__( + self, + run_id: str, + effective_target_name: str | None, + effective_connection: dict[str, object], + effective_vars: dict[str, object], + binding_catalog: Any | None = ..., + effective_target_database: str | None = ..., + effective_target_schema: str | None = ..., + sql_analysis_dialect: str | None = ..., + sql_lexical_syntax: SqlLexicalSyntax = ..., + compile_cache_dir: Path | None = ..., + settings: SettingsConfig = ..., + scenario: ScenarioConfig = ..., + models: tuple[CompiledModelView, ...] = ..., + sources: tuple[CompiledSourceView, ...] = ..., + seeds: tuple[CompiledSeedView, ...] = ..., + functions: tuple[CompiledFunctionView, ...] = ..., + audits: tuple[CompiledAuditView, ...] = ..., + sql_tests: tuple[CompiledSqlTest, ...] = ..., + sql_scenarios: tuple[CompiledSqlScenario, ...] = ..., + loader_functions: tuple[DiscoveredLoaderFunction, ...] = ..., + hook_functions: tuple[DiscoveredHookFunction, ...] = ..., + enforce_explicit_references: bool = ..., + sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = ..., + materialization_files: tuple[DiscoveredMaterializationFile, ...] = ..., + public_enums: dict[str, EnumDeclaration] = ..., + public_constants: dict[str, ConstantDeclaration] = ..., + loaded_macros: dict[str, LoadedMacro] = ..., + diagnostics: tuple[CompilerDiagnostic, ...] = ..., + unmatched_literal_sql_relations: tuple[LiteralSqlRelation, ...] = ..., + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = ..., + scope_index: ScopeIndex = ..., + sql_expansions: dict[Path, CompiledSqlExpansion] = ..., + native_session: Any | None = ..., + native_project: NativeCompiledProject | None = ..., + ) -> None: ... + @property + def lineage_graph(self) -> NativeProjectGraph: ... + +class CompiledSqlTest(NativeRecord): + """Compiled SQL-native unit test metadata selected by expected targets.""" + + key: CompiledObjectKey + scope_deps: tuple[CompiledObjectKey, ...] + name: str + test_file: DiscoveredSqlTestFile + test_block: DiscoveredSqlTestBlock + sql_body: str + mode: SqlTestMode + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload + source_path: Path | None + ownership_root: Path | None + block_index: int | None + explicit_name: str | None + parent_name: str | None + case_name: str | None + case_index: int | None + case_fingerprint: str | None + parameter_schema: tuple[SqlTestParameterDeclaration, ...] + parameter_values: tuple[tuple[str, SqlValue], ...] + expected_model_names: tuple[str, ...] + assertion_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + read_helper_names: tuple[str, ...] + reference_target_model_names: tuple[str, ...] + target_model_names: tuple[str, ...] + tested_resources: tuple[CompiledSqlTestResource, ...] + def __init__( + self, + key: CompiledObjectKey, + scope_deps: tuple[CompiledObjectKey, ...], + name: str, + test_file: DiscoveredSqlTestFile, + test_block: DiscoveredSqlTestBlock, + sql_body: str, + mode: SqlTestMode = ..., + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = ..., + source_path: Path | None = ..., + ownership_root: Path | None = ..., + block_index: int | None = ..., + explicit_name: str | None = ..., + parent_name: str | None = ..., + case_name: str | None = ..., + case_index: int | None = ..., + case_fingerprint: str | None = ..., + parameter_schema: tuple[SqlTestParameterDeclaration, ...] = ..., + parameter_values: tuple[tuple[str, SqlValue], ...] = ..., + expected_model_names: tuple[str, ...] = ..., + assertion_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + read_helper_names: tuple[str, ...] = ..., + reference_target_model_names: tuple[str, ...] = ..., + target_model_names: tuple[str, ...] = ..., + tested_resources: tuple[CompiledSqlTestResource, ...] = ..., + ) -> None: ... + +class RuleSubject(StrEnum): + """Compiler-owned unit used to invoke and cache one custom rule.""" + + MODEL = "model" + PROJECT = "project" + +class Model(NativeRecord): + """Stable identity and common metadata for one compiled SQL model.""" + + name: str + path: Path + materialization: str | None + def __init__( + self, + name: str, + path: Path, + materialization: str | None, + ) -> None: ... + +class Project(NativeRecord): + """Stable summary for one compiled project invocation.""" + + model_count: int + target_name: str | None + def __init__( + self, + model_count: int, + target_name: str | None, + ) -> None: ... + +class ProjectPath(NativeRecord): + """One normalized project-relative file or directory.""" + + value: str + is_file: bool + is_dir: bool + def __init__( + self, + value: str, + is_file: bool, + is_dir: bool, + ) -> None: ... + @property + def parts(self) -> tuple[str, ...]: ... + @property + def name(self) -> str: ... + @property + def depth(self) -> int: ... + +class SqlNode(NativeRecord): + """Stable source location for one common SQL structure fact.""" + + kind: str + line: int + column: int + def __init__( + self, + kind: str, + line: int, + column: int, + ) -> None: ... + +class ModelSql(NativeRecord): + """Authored and compiler-expanded SQL documents for one model.""" + + authored: Any + expanded: Any + def __init__( + self, + authored: Any, + expanded: Any, + ) -> None: ... + +class RuleFactViews(NativeRecord): + """Views shared by every invocation in one custom-rule host evaluation.""" + + project: Any + sql: Any + graph: Any + columns: Any + contracts: Any + tests: Any + audits: Any + declarations: Any + def __init__( + self, + project: Any, + sql: Any, + graph: Any, + columns: Any, + contracts: Any, + tests: Any, + audits: Any, + declarations: Any, + ) -> None: ... + +class RuleOption[T](NativeRecord): + """One typed configuration option declared by a rule.""" + + @property + def name(self) -> str: ... + @property + def value_type(self) -> type[T]: ... + @property + def default(self) -> T: ... + @property + def description(self) -> str: ... + @property + def choices(self) -> tuple[T, ...]: ... + @property + def minimum(self) -> int | None: ... + @property + def maximum(self) -> int | None: ... + @property + def minimum_items(self) -> int | None: ... + @property + def item_type(self) -> type[object] | None: ... + def __init__( + self, + name: str, + value_type: type[T], + default: T, + description: str, + choices: tuple[T, ...] = ..., + minimum: int | None = ..., + maximum: int | None = ..., + minimum_items: int | None = ..., + item_type: type[object] | None = ..., + ) -> None: ... + @classmethod + def boolean(cls, *, name: str, default: bool, description: str) -> RuleOption[bool]: ... + @classmethod + def integer( + cls, + *, + name: str, + default: int, + description: str, + minimum: int | None = None, + maximum: int | None = None, + ) -> RuleOption[int]: ... + @classmethod + def string( + cls, *, name: str, default: str, description: str, choices: tuple[str, ...] = () + ) -> RuleOption[str]: ... + @classmethod + def string_list( + cls, + *, + name: str, + default: tuple[str, ...], + description: str, + minimum_items: int | None = None, + ) -> RuleOption[tuple[str, ...]]: ... + @classmethod + def integer_list( + cls, + *, + name: str, + default: tuple[int, ...], + description: str, + minimum_items: int | None = None, + ) -> RuleOption[tuple[int, ...]]: ... + +class Rule(NativeRecord): + """Immutable metadata and implementation for one rule.""" + + code: str + family: str + slug: str + message: str + remediation: str + check: RuleCheck + options: tuple[RuleOption[object], ...] + enabled_by_default: bool + custom: bool + source: str | None + subject: RuleSubject | None + subject_parameter: str | None + context_parameter: str | None + guidance: RuleGuidance | None + def __init__( + self, + code: str, + family: str, + slug: str, + message: str, + remediation: str, + check: RuleCheck, + options: tuple[RuleOption[object], ...] = ..., + enabled_by_default: bool = ..., + custom: bool = ..., + source: str | None = ..., + subject: RuleSubject | None = ..., + subject_parameter: str | None = ..., + context_parameter: str | None = ..., + guidance: RuleGuidance | None = ..., + ) -> None: ... + @property + def project_wide(self) -> bool: ... + +class Finding(NativeRecord): + """One deterministic compiler rules violation.""" + + code: str + path: Path + line: int + column: int + message: str + remediation: str + unevaluated: bool + affected_rules: tuple[str, ...] + fixable: bool + def __init__( + self, + code: str, + path: Path, + line: int, + column: int, + message: str, + remediation: str, + unevaluated: bool = ..., + affected_rules: tuple[str, ...] = ..., + fixable: bool = ..., + ) -> None: ... + +class LayoutConfig(NativeRecord): + levels: tuple[str, ...] + domain_roots: tuple[str, ...] + def __init__( + self, + levels: tuple[str, ...] = ..., + domain_roots: tuple[str, ...] = ..., + ) -> None: ... + +class RulesConfig(NativeRecord): + select: tuple[str, ...] + ignore: tuple[str, ...] + thresholds: dict[str, int] + threshold_overrides: tuple[ThresholdOverride, ...] + rule_options: dict[str, dict[str, RuleOptionValue]] + rule_exceptions: tuple[RuleExemption, ...] + graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] + rule_ignores: tuple[RuleIgnore, ...] + allow_exceptions: bool + select_star_allow: tuple[SelectStarAllow, ...] + domains: tuple[str, ...] + approved_source_tokens: tuple[str, ...] + retired_source_tokens: dict[str, str] + sql_tests: SqlTestRulesConfig + layout: LayoutConfig + cache: RulesCacheConfig + def __init__( + self, + select: tuple[str, ...] = ..., + ignore: tuple[str, ...] = ..., + thresholds: dict[str, int] = ..., + threshold_overrides: tuple[ThresholdOverride, ...] = ..., + rule_options: dict[str, dict[str, RuleOptionValue]] = ..., + rule_exceptions: tuple[RuleExemption, ...] = ..., + graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] = ..., + rule_ignores: tuple[RuleIgnore, ...] = ..., + allow_exceptions: bool = ..., + select_star_allow: tuple[SelectStarAllow, ...] = ..., + domains: tuple[str, ...] = ..., + approved_source_tokens: tuple[str, ...] = ..., + retired_source_tokens: dict[str, str] = ..., + sql_tests: SqlTestRulesConfig = ..., + layout: LayoutConfig = ..., + cache: RulesCacheConfig = ..., + ) -> None: ... + +class RulesResult(NativeRecord): + findings: tuple[Finding, ...] + evaluated_models: int + unevaluated_resources: int + cache_hits: int + cache_misses: int + built_in_ms: int + custom_ms: int + custom_cpu_ms: int + def __init__( + self, + findings: tuple[Finding, ...], + evaluated_models: int, + unevaluated_resources: int = ..., + cache_hits: int = ..., + cache_misses: int = ..., + built_in_ms: int = ..., + custom_ms: int = ..., + custom_cpu_ms: int = ..., + ) -> None: ... + +class RuleFile(NativeRecord): + path: str + source: str + def __init__( + self, + path: str, + source: str, + ) -> None: ... + +class RuleCase(NativeRecord): + description: str + source: str + expected_finding_count: int + path: str + files: tuple[RuleFile, ...] + config: dict[str, object] + def __init__( + self, + description: str, + source: str, + expected_finding_count: int, + path: str = ..., + files: tuple[RuleFile, ...] = ..., + config: dict[str, object] = ..., + ) -> None: ... + +class RuleResult(NativeRecord): + findings: tuple[Finding, ...] + def __init__( + self, + findings: tuple[Finding, ...], + ) -> None: ... + @property + def finding_count(self) -> int: ... + +class CustomRuleEvaluation(NativeRecord): + """Findings and observed fact reads of one rule invocation for one subject.""" + + code: str + subject: str | None + findings: tuple[Finding, ...] + reads: frozenset[tuple[object, ...]] | None + def __init__( + self, + code: str, + subject: str | None, + findings: tuple[Finding, ...], + reads: frozenset[tuple[object, ...]] | None, + ) -> None: ... + +class ModuleStateTrace(NativeRecord): + """What exact module-state walks touched: object and type identities and their decisions.""" + + touched: dict[int, int] + descended_types: frozenset[int] + checked_instances: frozenset[int] + def __init__( + self, + touched: dict[int, int], + descended_types: frozenset[int], + checked_instances: frozenset[int], + ) -> None: ... + +class ModuleStateFingerprint(NativeRecord): + """One exact custom-rule module-state fingerprint and the trace of the walk behind it.""" + + token: str + trace: ModuleStateTrace + def __init__( + self, + token: str, + trace: ModuleStateTrace, + ) -> None: ... + +class CustomRuleRun(NativeRecord): + """Every invocation of one host run plus rules whose reads could not be attributed.""" + + evaluations: tuple[CustomRuleEvaluation, ...] + untracked_codes: frozenset[str] + uncacheable_codes: frozenset[str] + observed: tuple[tuple[tuple[str, ...], str | None], ...] + stateful_codes: frozenset[str] + def __init__( + self, + evaluations: tuple[CustomRuleEvaluation, ...], + untracked_codes: frozenset[str], + uncacheable_codes: frozenset[str] = ..., + observed: tuple[tuple[tuple[str, ...], str | None], ...] = ..., + stateful_codes: frozenset[str] = ..., + ) -> None: ... + +class CustomHostPartition(NativeRecord): + """How one host of a split run detects module state and learns the run was cancelled.""" + + cancelled: Callable[[], bool] + def __init__( + self, + cancelled: Callable[[], bool], + ) -> None: ... diff --git a/src/sqlbuild/adapter/__init__.py b/src/sqlbuild/adapter/__init__.py index cf225709ad..e69de29bb2 100644 --- a/src/sqlbuild/adapter/__init__.py +++ b/src/sqlbuild/adapter/__init__.py @@ -1 +0,0 @@ -"""Adapter interface and base implementation.""" diff --git a/src/sqlbuild/adapter/contract/_helpers/seed_csv.py b/src/sqlbuild/adapter/contract/_helpers/seed_csv.py index ad71e9899b..84daf70c80 100644 --- a/src/sqlbuild/adapter/contract/_helpers/seed_csv.py +++ b/src/sqlbuild/adapter/contract/_helpers/seed_csv.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings _DEFAULT_NULL_VALUE: str = "" diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index cb33a480c7..f915886a37 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -9,6 +9,14 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( HistoricalCheckSnapshotSql, ) @@ -73,17 +81,9 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings from sqlbuild.sql_values.exceptions import SqlValueRenderingError -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind _SCALAR_SQL_VALUE_KINDS: frozenset[SqlValueKind] = frozenset( { @@ -2124,9 +2124,9 @@ def render_create_fingerprint_table_sql( def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: """Render DDL that creates the direct microbatch state table when missing.""" - from sqlbuild.microbatches.main.create_table_sql import build_create_table_sql + from sqlbuild._native import build_microbatch_create_table_sql - return build_create_table_sql( + return build_microbatch_create_table_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index ec8ecff6f5..7d3f607161 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -12,6 +12,13 @@ from types import ModuleType from typing import Any, ClassVar +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract._helpers.seed_csv import get_seed_csv_null_values from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, @@ -78,19 +85,12 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue class DuckDbBackedAdapter(UnkeyedDiffMixin, BaseAdapter): @@ -948,7 +948,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -976,7 +976,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapter/contract/classes/materialization.py b/src/sqlbuild/adapter/contract/classes/materialization.py index bc499048cc..6d86984026 100644 --- a/src/sqlbuild/adapter/contract/classes/materialization.py +++ b/src/sqlbuild/adapter/contract/classes/materialization.py @@ -6,10 +6,10 @@ from pathlib import Path from typing import Any +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, FunctionDefinition from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings class MaterializationMixin(ABC): diff --git a/src/sqlbuild/adapter/contract/classes/microbatch.py b/src/sqlbuild/adapter/contract/classes/microbatch.py index c9766c5fbe..cc55246bc8 100644 --- a/src/sqlbuild/adapter/contract/classes/microbatch.py +++ b/src/sqlbuild/adapter/contract/classes/microbatch.py @@ -39,10 +39,10 @@ def physical_relation_generation( def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: """Render portable DDL for the direct microbatch state table.""" - from sqlbuild.microbatches.main.create_table_sql import build_create_table_sql + from sqlbuild._native import build_microbatch_create_table_sql adapter: Any = cast(Any, self) - return build_create_table_sql( + return build_microbatch_create_table_sql( database=database, schema=schema, render_qualified_name=adapter.render_qualified_name, diff --git a/src/sqlbuild/adapter/contract/classes/snapshot_sql.py b/src/sqlbuild/adapter/contract/classes/snapshot_sql.py index 7187fe778e..a36b24e9c3 100644 --- a/src/sqlbuild/adapter/contract/classes/snapshot_sql.py +++ b/src/sqlbuild/adapter/contract/classes/snapshot_sql.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import InitialValidFrom, SnapshotStrategy from sqlbuild.adapter.contract.models import SnapshotChangeTarget, SnapshotSqlDialect from sqlbuild.adapter.contract.types import SnapshotUpdateStyle -from sqlbuild.compiler.planner.types import InitialValidFrom, SnapshotStrategy class SnapshotSql: diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 4ce382bf8b..7b676a97bb 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -6,6 +6,7 @@ from decimal import Decimal from typing import Any, ClassVar +from sqlbuild._native import SourceFreshnessRecord, SqlValue from sqlbuild.adapter.contract.classes.connection import ConnectionMixin from sqlbuild.adapter.contract.classes.diff import DiffMixin from sqlbuild.adapter.contract.classes.materialization import MaterializationMixin @@ -32,8 +33,6 @@ TablePromotionMode, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.sql_values.models import SqlValue class StrictAdapter( diff --git a/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py b/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py index f93382334c..09abd3c23d 100644 --- a/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py +++ b/src/sqlbuild/adapter/contract/main/normalize_seed_csv_value.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract._helpers.seed_csv import normalize_seed_csv_value as _normalize -from sqlbuild.spec.contracts.models import SeedCsvSettings def normalize_seed_csv_value( diff --git a/src/sqlbuild/adapter/relations/_helpers/ddl_effects.py b/src/sqlbuild/adapter/relations/_helpers/ddl_effects.py deleted file mode 100644 index f78d68623c..0000000000 --- a/src/sqlbuild/adapter/relations/_helpers/ddl_effects.py +++ /dev/null @@ -1,343 +0,0 @@ -"""Classify executed SQL by its effect on cached relation existence and columns.""" - -from __future__ import annotations - -import re - -from sqlbuild.adapter.relations.constants import ( - BACKTICK, - DOUBLE_QUOTE, - IDENTIFIER_QUOTE_PAIRS, - QUOTED_IDENTIFIER_MIN_LENGTH, - STATEMENT_SEPARATOR, -) -from sqlbuild.adapter.relations.models import StatementMetadataEffect - -_LEXEME_START: re.Pattern[str] = re.compile(r"--|/\*|['\"`$\[]") -_BACKSLASH_LITERAL_STOP: re.Pattern[str] = re.compile(r"[\\']") -_DOLLAR_TAG: re.Pattern[str] = re.compile(r"\$[A-Za-z_]*+\$") -_LINE_COMMENT_START: str = "--" -_BLOCK_COMMENT_START: str = "/*" -_BLOCK_COMMENT_END: str = "*/" -_SINGLE_QUOTE: str = "'" -_BACKSLASH: str = "\\" -_DOLLAR: str = "$" -_OPEN_BRACKET: str = "[" -_CLOSE_BRACKET: str = "]" -_NEWLINE: str = "\n" -_EMPTY_LITERAL: str = "''" -_COMMENT_REPLACEMENT: str = " " -_FIRST_WORD: re.Pattern[str] = re.compile(r"[A-Za-z]+") -_IDENTIFIER_PART: str = r'(?:"(?:[^"]|"")*+"|`[^`]*+`|\[[^\]]*+\]|[A-Za-z0-9_][A-Za-z0-9_$]*+)' -_QUALIFIED_NAME: str = ( - rf"(?P{_IDENTIFIER_PART}(?:\s*+\.\s*+{_IDENTIFIER_PART})*+)" - r"(?P(?=\s|\()(?s:.*)|$)" -) -_LOOSE_IDENTIFIER_PART: str = r'(?:"(?:[^"]|"")*+"|`[^`]*+`|\[[^\]]*+\]|[A-Za-z0-9_$-]++)' -_LOOSE_QUALIFIED_NAME: re.Pattern[str] = re.compile( - rf"^\s*+{_LOOSE_IDENTIFIER_PART}(?:\s*+\.\s*+{_LOOSE_IDENTIFIER_PART})*+\s*+$" -) -_RELATION_KIND: str = ( - r"(?:(?:OR\s++REPLACE|LOCAL|GLOBAL|TEMP|TEMPORARY|TRANSIENT|VOLATILE|SECURE|RECURSIVE|" - r"MATERIALIZED|EXTERNAL|DYNAMIC|ICEBERG|HYBRID|UNLOGGED|FOREIGN|STREAMING)\s++)*+" - r"(?:TABLE|VIEW)\s++" -) -_CREATE_RELATION: re.Pattern[str] = re.compile( - rf"^CREATE\s++{_RELATION_KIND}(?:IF\s++NOT\s++EXISTS\s++)?{_QUALIFIED_NAME}", re.IGNORECASE -) -_DROP_RELATION: re.Pattern[str] = re.compile( - rf"^DROP\s++{_RELATION_KIND}(?:IF\s++EXISTS\s++)?{_QUALIFIED_NAME}", re.IGNORECASE -) -_ALTER_RELATION: re.Pattern[str] = re.compile( - rf"^ALTER\s++{_RELATION_KIND}(?:IF\s++EXISTS\s++)?{_QUALIFIED_NAME}", re.IGNORECASE -) -_COPY_INTO_RELATION: re.Pattern[str] = re.compile( - rf"^COPY\s++INTO\s++{_QUALIFIED_NAME}", re.IGNORECASE -) -_SELECT_INTO_RELATION: re.Pattern[str] = re.compile(rf"\bINTO\s++{_QUALIFIED_NAME}", re.IGNORECASE) -_SELECT_INTO_KEYWORD: re.Pattern[str] = re.compile(r"\bINTO\b", re.IGNORECASE) -_SECOND_RELATION: re.Pattern[str] = re.compile( - rf"^\s++(?:RENAME\s++TO|SWAP\s++WITH)\s++{_QUALIFIED_NAME}", re.IGNORECASE -) -_FOLLOWING_WORDS: dict[re.Pattern[str], re.Pattern[str]] = { - _CREATE_RELATION: re.compile( - r"^\s*+(?:$|\(|(?:AS|CLONE|SHALLOW|DEEP|LIKE|COPY|USING|COMMENT|CLUSTER|PARTITION|OPTIONS|WITH|" - r"TBLPROPERTIES|LOCATION|DATA_RETENTION_TIME_IN_DAYS|CHANGE_TRACKING)\b)", - re.IGNORECASE, - ), - _DROP_RELATION: re.compile(r"^\s*+(?:CASCADE|RESTRICT|PURGE)?+\s*+$", re.IGNORECASE), - _ALTER_RELATION: re.compile( - r"^\s++(?:ADD|DROP|ALTER|MODIFY|RENAME|SWAP|SET|UNSET|CLUSTER|RECLUSTER|OWNER|" - r"SUSPEND|RESUME|REFRESH)\b", - re.IGNORECASE, - ), - _COPY_INTO_RELATION: re.compile(r"^\s++FROM\b", re.IGNORECASE), - _SELECT_INTO_RELATION: re.compile(r"^\s++FROM\b", re.IGNORECASE), - _SECOND_RELATION: re.compile(r"^\s*+$"), -} -_CREATE_SCHEMA_IF_NOT_EXISTS: re.Pattern[str] = re.compile( - r"^CREATE\s++SCHEMA\s++IF\s++NOT\s++EXISTS\s++[^\s;]++\s*+$", re.IGNORECASE -) -_RELATION_NEUTRAL_OBJECT: re.Pattern[str] = re.compile( - r"^(?:CREATE|DROP|ALTER)\s++(?:OR\s++REPLACE\s++)?(?:(?:SECURE|TEMP|TEMPORARY|AGGREGATE)\s++)*+" - r"(?:FUNCTION|PROCEDURE|SEQUENCE|STAGE|FILE\s++FORMAT|INDEX|UNIQUE\s++INDEX|WAREHOUSE|MACRO)\b", - re.IGNORECASE, -) -_RENAMING_ALTER: re.Pattern[str] = re.compile( - r"^\s++(?:RENAME\s++TO|SWAP\s++WITH)\b", re.IGNORECASE -) -_QUERY_STATEMENTS: frozenset[str] = frozenset({"SELECT", "WITH"}) -_READ_OR_DATA_STATEMENTS: frozenset[str] = frozenset( - { - "VALUES", - "SHOW", - "DESCRIBE", - "DESC", - "EXPLAIN", - "INSERT", - "UPDATE", - "DELETE", - "MERGE", - "TRUNCATE", - "BEGIN", - "START", - "SAVEPOINT", - "RELEASE", - "SET", - "UNSET", - "GRANT", - "REVOKE", - "COMMENT", - "PUT", - "GET", - "LIST", - "LS", - "REMOVE", - "RM", - "ANALYZE", - } -) -_TRANSACTION_END_STATEMENTS: frozenset[str] = frozenset({"COMMIT", "ROLLBACK", "END", "ABORT"}) -_RELATION_DDL_STATEMENTS: frozenset[str] = frozenset({"CREATE", "DROP", "ALTER", "COPY"}) - - -_NO_EFFECT: StatementMetadataEffect = StatementMetadataEffect() -_ALL_RELATIONS: StatementMetadataEffect = StatementMetadataEffect(invalidates_all=True) -_TRANSACTION_END: StatementMetadataEffect = StatementMetadataEffect(ends_transaction=True) - - -def statement_metadata_effect(sql: str) -> StatementMetadataEffect: - """Return which cached relation names ``sql`` may have changed; ambiguity changes all.""" - - backslash_modes: tuple[bool, ...] = (False, True) if _BACKSLASH in sql else (False,) - bracket_modes: tuple[bool, ...] = (False, True) if _OPEN_BRACKET in sql else (False,) - effects: set[StatementMetadataEffect] = set() - backslash_escapes: bool - for backslash_escapes in backslash_modes: - effects.update( - _code_effect( - code=_code_text( - sql=sql, backslash_escapes=backslash_escapes, bracket_identifiers=brackets - ) - ) - for brackets in bracket_modes - ) - if len(effects) != 1: - return _ALL_RELATIONS - return effects.pop() - - -def relation_name_key(name: str) -> str: - """Return the invalidation key for one unquoted or quoted unqualified relation name.""" - - return _strip_identifier_quotes(name).casefold() - - -def qualified_relation_name_key(qualified: str) -> str | None: - """Return the key for the last part of a qualified name, or None when it cannot be parsed.""" - - if _LOOSE_QUALIFIED_NAME.match(qualified) is None: - return None - parts: list[str] = re.findall(_LOOSE_IDENTIFIER_PART, qualified) - return _part_name_key(parts[-1]) - - -def _code_text(*, sql: str, backslash_escapes: bool, bracket_identifiers: bool) -> str | None: - pieces: list[str] = [] - position: int = 0 - while True: - start: re.Match[str] | None = _LEXEME_START.search(sql, position) - if start is None: - pieces.append(sql[position:]) - return "".join(pieces) - pieces.append(sql[position : start.start()]) - lexeme: tuple[str, int] | None = _read_lexeme( - sql=sql, - start=start, - backslash_escapes=backslash_escapes, - bracket_identifiers=bracket_identifiers, - ) - if lexeme is None: - return None - pieces.append(lexeme[0]) - position = lexeme[1] - - -def _read_lexeme( - *, sql: str, start: re.Match[str], backslash_escapes: bool, bracket_identifiers: bool -) -> tuple[str, int] | None: - token: str = start.group(0) - begin: int = start.start() - if token == _LINE_COMMENT_START: - line_end: int = sql.find(_NEWLINE, begin) - return _COMMENT_REPLACEMENT, (len(sql) if line_end < 0 else line_end) - if token == _BLOCK_COMMENT_START: - return _closed( - text=_COMMENT_REPLACEMENT, end=sql.find(_BLOCK_COMMENT_END, begin + 2), width=2 - ) - if token == _SINGLE_QUOTE: - return _closed( - text=_EMPTY_LITERAL, - end=_literal_end(sql=sql, begin=begin, backslash_escapes=backslash_escapes), - width=1, - ) - if token == _DOLLAR: - return _dollar_lexeme(sql=sql, begin=begin) - if token == _OPEN_BRACKET and not bracket_identifiers: - return token, begin + 1 - closing: str = _CLOSE_BRACKET if token == _OPEN_BRACKET else token - end: int = _quoted_end(sql=sql, begin=begin, quote=closing) - if end < 0: - return None - return sql[begin : end + 1], end + 1 - - -def _closed(*, text: str, end: int, width: int) -> tuple[str, int] | None: - if end < 0: - return None - return text, end + width - - -def _dollar_lexeme(*, sql: str, begin: int) -> tuple[str, int] | None: - tag: re.Match[str] | None = _DOLLAR_TAG.match(sql, begin) - if tag is None: - return _DOLLAR, begin + 1 - return _closed( - text=_EMPTY_LITERAL, end=sql.find(tag.group(0), tag.end()), width=len(tag.group(0)) - ) - - -def _literal_end(*, sql: str, begin: int, backslash_escapes: bool) -> int: - position: int = begin + 1 - while True: - stop: int = _literal_stop(sql=sql, position=position, backslash_escapes=backslash_escapes) - if stop < 0: - return stop - if sql[stop] == _BACKSLASH: - position = stop + 2 - continue - if sql.startswith(_EMPTY_LITERAL, stop): - position = stop + 2 - continue - return stop - - -def _literal_stop(*, sql: str, position: int, backslash_escapes: bool) -> int: - if not backslash_escapes: - return sql.find(_SINGLE_QUOTE, position) - stop: re.Match[str] | None = _BACKSLASH_LITERAL_STOP.search(sql, position) - return -1 if stop is None else stop.start() - - -def _quoted_end(*, sql: str, begin: int, quote: str) -> int: - position: int = begin + 1 - while True: - end: int = sql.find(quote, position) - if end < 0 or quote == _CLOSE_BRACKET or not sql.startswith(quote * 2, end): - return end - position = end + 2 - - -def _code_effect(*, code: str | None) -> StatementMetadataEffect: - if code is None: - return _ALL_RELATIONS - statement: str = code.strip().rstrip(STATEMENT_SEPARATOR).strip().lstrip("(") - first: re.Match[str] | None = _FIRST_WORD.match(statement) - if first is None or STATEMENT_SEPARATOR in statement: - return _ALL_RELATIONS - keyword: str = first.group(0).upper() - if keyword in _QUERY_STATEMENTS: - return _select_into_effect(statement=statement) - if keyword in _READ_OR_DATA_STATEMENTS: - return _NO_EFFECT - if keyword in _TRANSACTION_END_STATEMENTS: - return _TRANSACTION_END - if keyword not in _RELATION_DDL_STATEMENTS: - return _ALL_RELATIONS - if ( - _CREATE_SCHEMA_IF_NOT_EXISTS.match(statement) is not None - or _RELATION_NEUTRAL_OBJECT.match(statement) is not None - ): - return _NO_EFFECT - return _relation_ddl_effect(statement=statement) - - -def _select_into_effect(*, statement: str) -> StatementMetadataEffect: - if _SELECT_INTO_KEYWORD.search(statement) is None: - return _NO_EFFECT - created: re.Match[str] | None = _SELECT_INTO_RELATION.search(statement) - names: frozenset[str] | None = _unambiguous_names(pattern=_SELECT_INTO_RELATION, match=created) - if names is None: - return _ALL_RELATIONS - return StatementMetadataEffect(relation_names=names) - - -def _relation_ddl_effect(*, statement: str) -> StatementMetadataEffect: - pattern: re.Pattern[str] - for pattern in (_CREATE_RELATION, _DROP_RELATION, _ALTER_RELATION, _COPY_INTO_RELATION): - match: re.Match[str] | None = pattern.match(statement) - names: frozenset[str] | None = _unambiguous_names(pattern=pattern, match=match) - if names is not None: - return StatementMetadataEffect(relation_names=names) - return _ALL_RELATIONS - - -def _unambiguous_names( - *, pattern: re.Pattern[str], match: re.Match[str] | None -) -> frozenset[str] | None: - if match is None: - return None - rest: str = match.group("rest") - if _FOLLOWING_WORDS[pattern].match(rest) is None: - return None - name: str = _unqualified_name_key(match.group("name")) - if pattern is not _ALTER_RELATION or _RENAMING_ALTER.match(rest) is None: - return frozenset({name}) - second: frozenset[str] | None = _unambiguous_names( - pattern=_SECOND_RELATION, match=_SECOND_RELATION.match(rest) - ) - if second is None: - return None - return frozenset({name, *second}) - - -def _unqualified_name_key(qualified: str) -> str: - parts: list[str] = re.findall(_IDENTIFIER_PART, qualified) - return _part_name_key(parts[-1]) - - -def _part_name_key(part: str) -> str: - if part.startswith(BACKTICK): - return _strip_identifier_quotes(part).rsplit(".", 1)[-1].casefold() - return relation_name_key(part) - - -def _strip_identifier_quotes(part: str) -> str: - if len(part) < QUOTED_IDENTIFIER_MIN_LENGTH: - return part - edges: tuple[str, str] = (part[0], part[-1]) - if edges == (DOUBLE_QUOTE, DOUBLE_QUOTE): - return part[1:-1].replace(DOUBLE_QUOTE * 2, DOUBLE_QUOTE) - if edges in IDENTIFIER_QUOTE_PAIRS: - return part[1:-1] - return part diff --git a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py index 6f745f81f2..588c1ffb73 100644 --- a/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py +++ b/src/sqlbuild/adapter/relations/classes/relation_metadata_cache.py @@ -6,7 +6,7 @@ from collections.abc import Callable from typing import cast -from sqlbuild.adapter.relations._helpers.ddl_effects import ( +from sqlbuild._native import ( qualified_relation_name_key, relation_name_key, statement_metadata_effect, diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index e68a5951b2..8fa46168db 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -3,10 +3,5 @@ INSPECTION_DESCRIBED_NAME_LIMIT: int = 3 INSPECTION_IN_LIST_LIMIT: int = 50 METADATA_NAME_FILTER_LIMIT: int = 250 -STATEMENT_SEPARATOR: str = ";" -QUOTED_IDENTIFIER_MIN_LENGTH: int = 2 -DOUBLE_QUOTE: str = '"' -IDENTIFIER_QUOTE_PAIRS: frozenset[tuple[str, str]] = frozenset({("`", "`"), ("[", "]")}) RELATION_COLUMNS_LOOKUP: str = "columns" RELATION_EXISTS_LOOKUP: str = "exists" -BACKTICK: str = "`" diff --git a/src/sqlbuild/adapter/state_sql/__init__.py b/src/sqlbuild/adapter/state_sql/__init__.py deleted file mode 100644 index 0163779a0a..0000000000 --- a/src/sqlbuild/adapter/state_sql/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Adapter-owned state SQL rendering.""" diff --git a/src/sqlbuild/adapter/state_sql/_helpers/__init__.py b/src/sqlbuild/adapter/state_sql/_helpers/__init__.py deleted file mode 100644 index ce3f537a3d..0000000000 --- a/src/sqlbuild/adapter/state_sql/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Adapter state SQL implementations.""" diff --git a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py b/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py deleted file mode 100644 index 7edd187e8a..0000000000 --- a/src/sqlbuild/adapter/state_sql/_helpers/source_freshness.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Adapter-owned renderers for source freshness state.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import render_state_sql_literal -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.source_freshness.constants import ( - COLUMN_DATA_VERSION, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, - COLUMN_RUN_ID, - COLUMN_SOURCE_NAME, - COLUMN_STRATEGY, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_NAME, - COLUMN_TARGET_SCHEMA, - COLUMN_VALUE_KIND, - SOURCE_FRESHNESS_COLUMN_TYPES, - SOURCE_FRESHNESS_COLUMNS, - SOURCE_FRESHNESS_TABLE_NAME, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord - - -def render_insert_source_freshness_records_sql( - *, - database: str | None, - schema: str, - records: tuple[SourceFreshnessRecord, ...], - render_qualified_name: Callable[..., str | None], -) -> str: - """Render DML that appends source freshness records.""" - - table_name: str | None = render_qualified_name( - database=database, - schema=schema, - name=SOURCE_FRESHNESS_TABLE_NAME, - ) - if table_name is None: - raise AdapterUserError(message="source freshness table requires a target schema") - values_sql: str = ", ".join(_record_values_sql(record) for record in records) - return ( - f"INSERT INTO {table_name} (" - f"{COLUMN_SOURCE_NAME}, " - f"{COLUMN_TARGET_DATABASE}, " - f"{COLUMN_TARGET_SCHEMA}, " - f"{COLUMN_TARGET_NAME}, " - f"{COLUMN_RUN_ID}, " - f"{COLUMN_STRATEGY}, " - f"{COLUMN_VALUE_KIND}, " - f"{COLUMN_DATA_VERSION}, " - f"{COLUMN_DATA_VERSION_HASH}, " - f"{COLUMN_OBSERVED_AT}" - f") VALUES {values_sql}" - ) - - -def _record_values_sql(record: SourceFreshnessRecord) -> str: - values: tuple[object | None, ...] = ( - record.source_name, - record.target_database, - record.target_schema, - record.target_name, - record.run_id, - record.strategy, - record.value_kind, - record.data_version, - record.data_version_hash, - record.observed_at, - ) - return ( - "(" - + ", ".join( - render_state_sql_literal( - value=value, declared_type=SOURCE_FRESHNESS_COLUMN_TYPES[column] - ) - for column, value in zip(SOURCE_FRESHNESS_COLUMNS, values, strict=True) - ) - + ")" - ) diff --git a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py b/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py deleted file mode 100644 index 5312637c8c..0000000000 --- a/src/sqlbuild/adapter/state_sql/_helpers/table_ddl.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Portable append-only state table DDL implementations.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.sql_values.types import StateSqlValueType - - -def render_state_table_create_sql_impl( - *, - qualified_name: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - required_columns: frozenset[str], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Render create-if-missing DDL for string, integer, and text-timestamp state columns.""" - - rendered_types: dict[StateSqlValueType, str] = { - StateSqlValueType.STRING: render_framework_type(FrameworkType.STRING), - StateSqlValueType.INTEGER: render_framework_type(FrameworkType.INTEGER), - StateSqlValueType.TEXT_TIMESTAMP: render_framework_type(FrameworkType.TIMESTAMP), - } - definitions: list[str] = [] - for column in columns: - required: str = " NOT NULL" if column in required_columns else "" - definitions.append(f"{column} {rendered_types[column_types[column]]}{required}") - return f"CREATE TABLE IF NOT EXISTS {qualified_name} ({', '.join(definitions)})" diff --git a/src/sqlbuild/adapter/state_sql/main/__init__.py b/src/sqlbuild/adapter/state_sql/main/__init__.py deleted file mode 100644 index 810274fc07..0000000000 --- a/src/sqlbuild/adapter/state_sql/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public adapter state SQL entries.""" diff --git a/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py b/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py deleted file mode 100644 index 3434747e8f..0000000000 --- a/src/sqlbuild/adapter/state_sql/main/render_insert_source_freshness_records_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public source freshness insert rendering operation.""" - -from collections.abc import Callable - -from sqlbuild.adapter.state_sql._helpers.source_freshness import ( - render_insert_source_freshness_records_sql as _render_sql, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord - - -def render_insert_source_freshness_records_sql( - *, - database: str | None, - schema: str, - records: tuple[SourceFreshnessRecord, ...], - render_qualified_name: Callable[..., str | None], -) -> str: - """Render DML that appends source freshness records.""" - - return _render_sql( - database=database, - schema=schema, - records=records, - render_qualified_name=render_qualified_name, - ) diff --git a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py b/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py deleted file mode 100644 index e0fe88aac6..0000000000 --- a/src/sqlbuild/adapter/state_sql/main/render_state_table_create_sql.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public append-only state table DDL rendering operation.""" - -from collections.abc import Callable, Mapping - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql._helpers.table_ddl import render_state_table_create_sql_impl -from sqlbuild.sql_values.types import StateSqlValueType - - -def render_state_table_create_sql( - *, - qualified_name: str, - columns: tuple[str, ...], - column_types: Mapping[str, StateSqlValueType], - required_columns: frozenset[str], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Render create-if-missing DDL for one append-only state table.""" - - return render_state_table_create_sql_impl( - qualified_name=qualified_name, - columns=columns, - column_types=column_types, - required_columns=required_columns, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/adapter/type_system/_helpers/inference_rules.py b/src/sqlbuild/adapter/type_system/_helpers/inference_rules.py deleted file mode 100644 index e7d2b22a3b..0000000000 --- a/src/sqlbuild/adapter/type_system/_helpers/inference_rules.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Public expression inference capability for adapters.""" - -from __future__ import annotations - -from sqlbuild.compiler.lineage.types import InferredNullability - - -def first_arg_nullability( - args: tuple[InferredNullability, ...], -) -> InferredNullability: - """Return the first argument's nullability for null-propagating unary functions.""" - - if not args: - return InferredNullability.UNKNOWN - return args[0] - - -def conditional_result_nullability( - args: tuple[InferredNullability, ...], -) -> InferredNullability: - """Infer IF/IFF-style nullability from true/false result arguments.""" - - source_target_and_default_argument_count: int = 3 - if len(args) < source_target_and_default_argument_count: - return InferredNullability.UNKNOWN - result_args: tuple[InferredNullability, InferredNullability] = (args[1], args[2]) - if all(value == InferredNullability.NON_NULL for value in result_args): - return InferredNullability.NON_NULL - if any(value == InferredNullability.NULLABLE for value in result_args): - return InferredNullability.NULLABLE - return InferredNullability.UNKNOWN - - -def safe_cast_nullability( - args: tuple[InferredNullability, ...], -) -> InferredNullability: - """Infer SAFE_CAST/TRY_CAST nullability without assuming conversion succeeds.""" - - if args and args[0] == InferredNullability.NULLABLE: - return InferredNullability.NULLABLE - return InferredNullability.UNKNOWN diff --git a/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py b/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py index 46a0128295..5e18a0780e 100644 --- a/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py +++ b/src/sqlbuild/adapter/type_system/main/conditional_result_nullability.py @@ -1,8 +1,6 @@ """Public adapter conditional nullability inference operation.""" -from sqlbuild.adapter.type_system._helpers.inference_rules import ( - conditional_result_nullability as _conditional_result_nullability, -) +from sqlbuild._native import infer_conditional_result_nullability from sqlbuild.compiler.lineage.types import InferredNullability @@ -11,4 +9,4 @@ def conditional_result_nullability( ) -> InferredNullability: """Infer IF/IFF-style nullability from true and false result arguments.""" - return _conditional_result_nullability(args) + return infer_conditional_result_nullability(args) diff --git a/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py b/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py index 3117238ff8..1e4e38ceee 100644 --- a/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py +++ b/src/sqlbuild/adapter/type_system/main/first_arg_nullability.py @@ -1,12 +1,10 @@ """Public adapter unary nullability inference operation.""" -from sqlbuild.adapter.type_system._helpers.inference_rules import ( - first_arg_nullability as _first_arg_nullability, -) +from sqlbuild._native import infer_first_arg_nullability from sqlbuild.compiler.lineage.types import InferredNullability def first_arg_nullability(args: tuple[InferredNullability, ...]) -> InferredNullability: """Return the first argument's nullability for null-propagating unary functions.""" - return _first_arg_nullability(args) + return infer_first_arg_nullability(args) diff --git a/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py b/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py index 5dddd886e9..03829b08e0 100644 --- a/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py +++ b/src/sqlbuild/adapter/type_system/main/safe_cast_nullability.py @@ -1,12 +1,10 @@ """Public adapter safe-cast nullability inference operation.""" -from sqlbuild.adapter.type_system._helpers.inference_rules import ( - safe_cast_nullability as _safe_cast_nullability, -) +from sqlbuild._native import infer_safe_cast_nullability from sqlbuild.compiler.lineage.types import InferredNullability def safe_cast_nullability(args: tuple[InferredNullability, ...]) -> InferredNullability: """Infer safe-cast nullability without assuming conversion succeeds.""" - return _safe_cast_nullability(args) + return infer_safe_cast_nullability(args) diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 4601cc88e8..e0356b7fd3 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -10,6 +10,14 @@ from pathlib import Path from typing import Any, ClassVar, cast +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( _COLLECTION_SQL_VALUE_KINDS, BaseAdapter, @@ -98,9 +106,6 @@ ) from sqlbuild.adapter.relations.main.invalidate_cached_relations import invalidate_cached_relations from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) @@ -135,14 +140,9 @@ TIMESTAMP_TYPE_TOKEN, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind class BigQueryAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): @@ -402,7 +402,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -434,7 +434,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index a15d065224..00caaef22e 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -13,6 +13,13 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -96,9 +103,6 @@ get_columns_for_relations_bulk, ) from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) @@ -126,12 +130,8 @@ UNDEFINED_TABLE_SQLSTATE, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue class DatabricksAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): @@ -498,7 +498,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -530,7 +530,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py index 971c445f7d..7294fd39c3 100644 --- a/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py +++ b/src/sqlbuild/adapters/duckdb/classes/duckdb_adapter.py @@ -62,9 +62,9 @@ def physical_relation_generation( def render_create_microbatch_state_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: - from sqlbuild.microbatches.main.create_index_sqls import build_create_index_sqls + from sqlbuild._native import build_microbatch_create_index_sqls - return build_create_index_sqls( + return build_microbatch_create_index_sqls( database=database, schema=schema, render_identifier=self.render_identifier, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 5cd919f515..793d5fef84 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -10,6 +10,13 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -87,9 +94,6 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.adapters.postgres._helpers.grants import ( @@ -104,12 +108,8 @@ TABLE_FUNCTION_RETURN_TYPE, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue class PostgresAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): @@ -1521,7 +1521,7 @@ def render_create_fingerprint_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -1539,9 +1539,9 @@ def render_create_fingerprint_index_sqls( def render_create_microbatch_state_index_sqls( self, *, database: str | None, schema: str ) -> tuple[str, ...]: - from sqlbuild.microbatches.main.create_index_sqls import build_create_index_sqls + from sqlbuild._native import build_microbatch_create_index_sqls - return build_create_index_sqls( + return build_microbatch_create_index_sqls( database=database, schema=schema, render_identifier=self.render_identifier, @@ -1571,7 +1571,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -1608,7 +1608,7 @@ def render_create_source_freshness_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -1783,7 +1783,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index c20afa3e80..c6affbc356 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -12,7 +12,15 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import collect_snowflake_cost +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + TableType, + collect_snowflake_cost, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _encode_typed_json, @@ -103,9 +111,6 @@ SchemaColumnListing, SchemaRelationListing, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( conditional_result_nullability, ) @@ -151,16 +156,11 @@ VIEW_RELATION_TYPE_TOKEN, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.spec.contracts.types import TableType -from sqlbuild.sql_values.models import SqlValue _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" @@ -730,7 +730,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -762,7 +762,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py index 8b5d0fd682..94d6b96685 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_cursor.py @@ -12,8 +12,7 @@ from typing import Any from uuid import uuid4 -from sqlbuild._native import CostContext, record_statement -from sqlbuild.cost.constants import SQLBUILD_QUERY_TAG_APP +from sqlbuild._native import SQLBUILD_QUERY_TAG_APP, CostContext, record_statement from sqlbuild.cost.models import CostResourceContext, StatementExecutionTelemetry from sqlbuild.runtime.execution_limits.main.current_execution_deadline import ( current_execution_deadline, diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index ccfda8d0b2..af95fd3413 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -10,6 +10,14 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import ( + SeedCsvSettings, + SourceFreshnessRecord, + SqlLexicalSyntax, + SqlValue, + SqlValueKind, + render_insert_source_freshness_records_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import ( BaseAdapter, _build_names_filter, @@ -89,9 +97,6 @@ from sqlbuild.adapter.relations.main.get_columns_for_relations import ( get_columns_for_relations_bulk, ) -from sqlbuild.adapter.state_sql.main.render_insert_source_freshness_records_sql import ( - render_insert_source_freshness_records_sql, -) from sqlbuild.adapters.sqlserver._helpers.grants import ( render_sqlserver_view_grants, render_sqlserver_view_revokes, @@ -109,13 +114,8 @@ USER_TABLE_OBJECT_TYPE, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind def _render_tsql_distinct_condition(*, left: str, right: str) -> str: @@ -314,7 +314,7 @@ def render_create_fingerprint_table_sql( database: str | None, schema: str, ) -> str: - from sqlbuild.compiler.fingerprints.constants import ( + from sqlbuild._native import ( COLUMN_DEFINITION_B64, COLUMN_DEFINITION_HASH, COLUMN_METADATA_JSON_B64, @@ -361,10 +361,10 @@ def render_create_fingerprint_table_sql( ) def render_create_microbatch_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import build_microbatch_create_table_sql from sqlbuild.microbatches.constants import MICROBATCH_TABLE_NAME - from sqlbuild.microbatches.main.create_table_sql import build_create_table_sql - create_sql: str = build_create_table_sql( + create_sql: str = build_microbatch_create_table_sql( database=database, schema=schema, render_qualified_name=self.render_qualified_name, @@ -383,7 +383,7 @@ def render_create_fingerprint_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -428,7 +428,7 @@ def render_prune_fingerprint_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME + from sqlbuild._native import FINGERPRINT_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -463,7 +463,7 @@ def render_create_source_freshness_index_sqls( database: str | None, schema: str, ) -> tuple[str, ...]: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, @@ -655,8 +655,7 @@ def render_create_janitor_event_table_sql(self, *, database: str | None, schema: ) def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: - from sqlbuild._native import render_migration_state_create_sql - from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME + from sqlbuild._native import MIGRATION_TABLE_NAME, render_migration_state_create_sql create_sql: str = render_migration_state_create_sql( kind="model", @@ -675,8 +674,7 @@ def render_create_migration_state_table_sql(self, *, database: str | None, schem def render_create_old_name_view_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild._native import render_migration_state_create_sql - from sqlbuild.compiler.migrations.constants import OLD_NAME_VIEW_TABLE_NAME + from sqlbuild._native import OLD_NAME_VIEW_TABLE_NAME, render_migration_state_create_sql create_sql: str = render_migration_state_create_sql( kind="old_name", @@ -695,8 +693,7 @@ def render_create_old_name_view_state_table_sql( def render_create_column_migration_state_table_sql( self, *, database: str | None, schema: str ) -> str: - from sqlbuild._native import render_migration_state_create_sql - from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME + from sqlbuild._native import COLUMN_MIGRATION_TABLE_NAME, render_migration_state_create_sql create_sql: str = render_migration_state_create_sql( kind="column", @@ -733,7 +730,7 @@ def render_prune_source_freshness_history_sql( schema: str, retain_versions: int, ) -> str: - from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME table_name: str | None = self.render_qualified_name( database=database, diff --git a/src/sqlbuild/assets.py b/src/sqlbuild/assets.py index 49143893ab..296ae1f5d2 100644 --- a/src/sqlbuild/assets.py +++ b/src/sqlbuild/assets.py @@ -4,6 +4,7 @@ from collections.abc import Callable, Mapping, Sequence +from sqlbuild._native import SourceColumnEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.python_nodes.models import AssetContext from sqlbuild.python_nodes.main.apply_asset import apply_asset @@ -15,7 +16,6 @@ SqlResourceRef, ) from sqlbuild.python_nodes.types import ColumnLineageRefSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry __all__ = ("AssetContext", "SkipMode", "asset", "get_asset_definition") diff --git a/src/sqlbuild/assets.pyi b/src/sqlbuild/assets.pyi index 5740b14106..ae882c433e 100644 --- a/src/sqlbuild/assets.pyi +++ b/src/sqlbuild/assets.pyi @@ -1,6 +1,7 @@ from collections.abc import Callable, Mapping, Sequence from typing import overload +from sqlbuild._native import SourceColumnEntry from sqlbuild.compiler.python_nodes.types import SkipMode as SkipMode from sqlbuild.executor.python_nodes.models import AssetContext as AssetContext from sqlbuild.python_nodes.models import ( @@ -10,7 +11,6 @@ from sqlbuild.python_nodes.models import ( SqlResourceRef, ) from sqlbuild.python_nodes.types import ColumnLineageRefSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry __all__ = ("AssetContext", "SkipMode", "asset", "get_asset_definition") diff --git a/src/sqlbuild/audits.py b/src/sqlbuild/audits.py index 800380b82a..a686467180 100644 --- a/src/sqlbuild/audits.py +++ b/src/sqlbuild/audits.py @@ -1,5 +1,7 @@ """Public authoring API for generated SQL audit attachments.""" +from sqlbuild._native import MeasurementThresholdBound as MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholds as MeasurementThresholds from sqlbuild.compiler.auditing.main.above import above as above from sqlbuild.compiler.auditing.main.audit_factory import audit_factory as audit_factory from sqlbuild.compiler.auditing.main.below import below as below @@ -7,12 +9,6 @@ get_audit_factory_definition as get_audit_factory_definition, ) from sqlbuild.compiler.auditing.main.outside import outside as outside -from sqlbuild.compiler.auditing.models import ( - MeasurementThresholdBound as MeasurementThresholdBound, -) -from sqlbuild.compiler.auditing.models import ( - MeasurementThresholds as MeasurementThresholds, -) from sqlbuild.compiler.auditing.types import ( AuditSeverity as AuditSeverity, ) diff --git a/src/sqlbuild/cli/__init__.py b/src/sqlbuild/cli/__init__.py index a3ba1f69ba..e69de29bb2 100644 --- a/src/sqlbuild/cli/__init__.py +++ b/src/sqlbuild/cli/__init__.py @@ -1 +0,0 @@ -"""CLI package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/entry/__init__.py b/src/sqlbuild/cli/commands/_helpers/entry/__init__.py index 053344bc45..e69de29bb2 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/__init__.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/__init__.py @@ -1 +0,0 @@ -"""CLI entry package.""" diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py deleted file mode 100644 index 53387605c0..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Direct build source freshness state helpers.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -from sqlbuild._native import ( - CostContext, - DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, - SourceFreshnessRenderers, - write_source_freshness_records, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.executor.build.models import BuildExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.runtime.observability.main.run_scope import run_scope -from sqlbuild.runtime.observability.models import OperationAttributes - - -def append_eligible_direct_source_freshness_records( - *, - plan: PlanOutput, - result: BuildExecutionResult, - adapter: BaseAdapter, - connection_config: dict[str, object], - run_id: str, -) -> None: - """Append observed source freshness only after affected selected models succeed.""" - - source_freshness: DirectSourceFreshnessPlanningResult | None = plan.source_freshness - if source_freshness is None or source_freshness.propagation is None: - return - - selected_model_names: frozenset[str] = frozenset(entry.name for entry in plan.model_entries) - if not selected_model_names: - return - - successful_model_names: frozenset[str] = frozenset( - model_result.model_name - for model_result in result.model_results - if getattr(model_result.status, "value", model_result.status) == ExecutionStatus.SUCCESS - ) - observed_records_by_identity: dict[SourceFreshnessIdentity, SourceFreshnessRecord] = { - record.identity: record for record in source_freshness.observed_records - } - entries_by_name: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} - - records_by_location: dict[tuple[str | None, str], list[SourceFreshnessRecord]] = {} - identity: SourceFreshnessIdentity - affected_model_names: frozenset[str] - for ( - identity, - affected_model_names, - ) in source_freshness.propagation.changed_source_model_names.items(): - affected_selected_model_names: frozenset[str] = affected_model_names & selected_model_names - if not affected_selected_model_names: - continue - if not affected_selected_model_names <= successful_model_names: - continue - record: SourceFreshnessRecord | None = observed_records_by_identity.get(identity) - if record is None: - continue - records_by_location = _collect_record_for_affected_model_schemas( - records_by_location=records_by_location, - record=replace(record, run_id=run_id), - affected_model_names=affected_selected_model_names, - entries_by_name=entries_by_name, - ) - record_count: int = sum(len(records) for records in records_by_location.values()) - if record_count == 0: - return - with run_scope(run_id): - with OperationLifecycle( - operation_kind="freshness", - operation_name="source_freshness_publication", - metadata={"item_count": record_count}, - attributes=OperationAttributes(phase="publish", target_kind="state_batch"), - ): - connection: Any = adapter.connect(connection_config) - try: - _append_records_by_location( - adapter=adapter, - connection=connection, - records_by_location=records_by_location, - ) - finally: - adapter.close(connection) - - -def _collect_record_for_affected_model_schemas( - *, - records_by_location: dict[tuple[str | None, str], list[SourceFreshnessRecord]], - record: SourceFreshnessRecord, - affected_model_names: frozenset[str], - entries_by_name: dict[str, ModelPlanEntry], -) -> dict[tuple[str | None, str], list[SourceFreshnessRecord]]: - recorded_locations: set[tuple[str | None, str]] = set() - model_name: str - for model_name in sorted(affected_model_names): - entry: ModelPlanEntry | None = entries_by_name.get(model_name) - if entry is None or entry.destination.schema is None: - continue - location: tuple[str | None, str] = (entry.destination.database, entry.destination.schema) - if location in recorded_locations: - continue - recorded_locations.add(location) - records_by_location.setdefault(location, []).append(record) - return records_by_location - - -def _append_records_by_location( - *, - adapter: BaseAdapter, - connection: Any, - records_by_location: dict[tuple[str | None, str], list[SourceFreshnessRecord]], -) -> None: - location: tuple[str | None, str] - records: list[SourceFreshnessRecord] - for location, records in records_by_location.items(): - database, schema = location - records_by_source_name: dict[str, list[SourceFreshnessRecord]] = {} - record: SourceFreshnessRecord - for record in records: - records_by_source_name.setdefault(record.source_name, []).append(record) - source_name: str - source_records: list[SourceFreshnessRecord] - for source_name, source_records in records_by_source_name.items(): - with CostContext.resource_scope( - resource_type="source", - resource_name=source_name, - phase="freshness_finalization", - ): - write_source_freshness_records( - connection=connection, - execute=adapter.execute, - database=database, - schema=schema, - records=tuple(source_records), - renderers=SourceFreshnessRenderers( - render_qualified_name=adapter.render_qualified_name, - render_framework_type=adapter.render_framework_type, - render_insert_records_sql=adapter.render_insert_source_freshness_records_sql, - render_create_table_sql=adapter.render_create_source_freshness_table_sql, - render_create_index_sqls=adapter.render_create_source_freshness_index_sqls, - ), - ) diff --git a/src/sqlbuild/cli/commands/classes/__init__.py b/src/sqlbuild/cli/commands/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/__init__.py b/src/sqlbuild/cli/commands/main/__init__.py index 37dfe1d1cb..e69de29bb2 100644 --- a/src/sqlbuild/cli/commands/main/__init__.py +++ b/src/sqlbuild/cli/commands/main/__init__.py @@ -1 +0,0 @@ -"""Main CLI command package.""" diff --git a/src/sqlbuild/cli/commands/main/dbt/__init__.py b/src/sqlbuild/cli/commands/main/dbt/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index a8cc32b3bd..926b51d22c 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -6,14 +6,18 @@ from datetime import UTC, datetime from pathlib import Path -from sqlbuild._native import CliNamespace, LocalFilesystemComputeLogStorage, supports_color +from sqlbuild._native import ( + COMPUTE_LOG_FORMAT_VERSION, + CliNamespace, + LocalFilesystemComputeLogStorage, + supports_color, +) from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( log_compute_capture_failure, ) from sqlbuild.diagnostics.classes.invocation_diagnostic_routing import InvocationDiagnosticRouting from sqlbuild.diagnostics.main.configure import configure_diagnostics from sqlbuild.diagnostics.models import DiagnosticRoutingOptions -from sqlbuild.runtime.compute_logs.constants import COMPUTE_LOG_FORMAT_VERSION from sqlbuild.runtime.compute_logs.main.run_with_compute_log_capture import ( run_with_compute_log_capture, ) diff --git a/src/sqlbuild/cli/commands/main/inspection/__init__.py b/src/sqlbuild/cli/commands/main/inspection/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/project/__init__.py b/src/sqlbuild/cli/commands/main/project/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/state/__init__.py b/src/sqlbuild/cli/commands/main/state/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/workspace/__init__.py b/src/sqlbuild/cli/commands/main/workspace/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_reuse/__init__.py b/src/sqlbuild/cli/compile_reuse/__init__.py index e7ebc10ef2..e69de29bb2 100644 --- a/src/sqlbuild/cli/compile_reuse/__init__.py +++ b/src/sqlbuild/cli/compile_reuse/__init__.py @@ -1 +0,0 @@ -"""Whole-project compile reuse when no compile input changed.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py deleted file mode 100644 index a75a54a0d4..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Interpreter and invocation facts the Python host supplies to native compile reuse.""" - -from __future__ import annotations - -import json -import os -import stat -import sys - -import sqlbuild._native as native -from sqlbuild.cli.compile_reuse.constants import OUTPUT_FILE_DESCRIPTORS, REUSE_FORMAT_VERSION -from sqlbuild.cli.compile_reuse.models import CompileReuseRequest - - -def runtime_identity() -> dict[str, str]: - """Return the interpreter and native build identity that produced a compile.""" - - return { - "format": str(REUSE_FORMAT_VERSION), - "python": sys.version, - "executable": sys.executable, - "platform": sys.platform, - "cache_tag": str(sys.implementation.cache_tag), - "native_build": str(native.BUILD_IDENTITY), - } - - -def invocation_identity( - *, request: CompileReuseRequest, project_dir: str, use_color: bool -) -> bytes: - """Encode every command-line choice and location that can change compile output.""" - - payload: dict[str, object] = { - "project_dir": project_dir, - "requested_project_dir": None if request.project_dir is None else str(request.project_dir), - "cwd": os.getcwd(), - "no_sql_validation": request.no_sql_validation, - "defer_to": request.defer_to, - "selected_target": request.selected_target, - "json_output": request.json_output, - "manifest": request.manifest, - "dag_path": request.dag_path, - "no_color": request.no_color, - "use_color": use_color, - "lineage_mode": request.lineage_mode, - "select": list(request.select), - "exclude": list(request.exclude), - "cli_vars": request.cli_vars, - "sys_path": list(sys.path), - } - return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=repr).encode( - "utf-8", "surrogateescape" - ) - - -def redirected_output_files() -> list[tuple[int, int]]: - """Identify stdout and stderr files, which hold this command's output, not its inputs.""" - - identities: set[tuple[int, int]] = set() - for descriptor in OUTPUT_FILE_DESCRIPTORS: - try: - status: os.stat_result = os.fstat(descriptor) - except OSError: - continue - if stat.S_ISREG(status.st_mode): - identities.add((status.st_dev, status.st_ino)) - return sorted(identities) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py b/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py index 0cc52f976d..79cc5c8b2e 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py @@ -2,31 +2,15 @@ import site import sys -from os import stat_result -from pathlib import Path + +from sqlbuild._native import startup_input_stamps def startup_inputs() -> list[tuple[str, int | None, int | None]]: """Stamp startup directories, path files, interpreter and virtual-environment configuration.""" - directories: set[Path] = { - Path(entry).absolute() - for entry in (*sys.path, *site.getsitepackages(), site.getusersitepackages()) - } - paths: set[Path] = { - *directories, - Path(sys.executable), - Path(sys.prefix) / "pyvenv.cfg", - Path(sys.base_prefix) / "pyvenv.cfg", - } - for directory in directories: - paths.update(directory.glob("*.pth")) - stamps: list[tuple[str, int | None, int | None]] = [] - for path in sorted(paths): - try: - status: stat_result = path.stat() - except FileNotFoundError: - stamps.append((str(path), None, None)) - else: - stamps.append((str(path), status.st_mtime_ns, status.st_size)) - return stamps + return startup_input_stamps( + directories=(*sys.path, *site.getsitepackages(), site.getusersitepackages()), + files=(sys.executable,), + prefixes=(sys.prefix, sys.base_prefix), + ) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py deleted file mode 100644 index c115ceb35f..0000000000 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compile reuse runtime identity constants.""" - -from __future__ import annotations - -REUSE_FORMAT_VERSION: int = 1 -OUTPUT_FILE_DESCRIPTORS: tuple[int, ...] = (1, 2) diff --git a/src/sqlbuild/cli/entry/constants.py b/src/sqlbuild/cli/entry/constants.py index 12e543d6ee..3bff31b705 100644 --- a/src/sqlbuild/cli/entry/constants.py +++ b/src/sqlbuild/cli/entry/constants.py @@ -4,4 +4,3 @@ NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS: str = "10" OUTPUT_STREAM_ENCODING: str = "utf-8" OUTPUT_STREAM_ERRORS: str = "backslashreplace" -LAUNCH_RECORD_OPTION: str = "--sqlbuild-launch-record=" diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py index 69729dc592..b544413b74 100644 --- a/src/sqlbuild/cli/entry/main/launched.py +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -1,13 +1,22 @@ """Python CLI entrypoint the native `sqb` executable hands every invocation it does not serve.""" +import os import sys -from sqlbuild.cli.entry.constants import LAUNCH_RECORD_OPTION +from sqlbuild.cli.entry.constants import ( + NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS, + NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, +) from sqlbuild.cli.entry.main.entry import main as run_cli def main() -> int: - """Run the CLI with the arguments `sqb` received; the CLI reads a leading launch record.""" + """Run the CLI with the `sqb` arguments, setting the allocator purge delay first.""" + + os.environ.setdefault( + NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS + ) + from sqlbuild._native import LAUNCH_RECORD_OPTION arguments: list[str] = sys.argv[1:] offered: bool = bool(arguments) and arguments[0].startswith(LAUNCH_RECORD_OPTION) diff --git a/src/sqlbuild/cli/output/__init__.py b/src/sqlbuild/cli/output/__init__.py deleted file mode 100644 index c17e7f4325..0000000000 --- a/src/sqlbuild/cli/output/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI output capabilities.""" diff --git a/src/sqlbuild/cli/output/constants.py b/src/sqlbuild/cli/output/constants.py deleted file mode 100644 index dd8a0acf62..0000000000 --- a/src/sqlbuild/cli/output/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -"""CLI execution output constants.""" - -INTEGRATION_RESULT_PATH_ENV: str = "SQLBUILD_INTEGRATION_RESULT_PATH" diff --git a/src/sqlbuild/compiler/__init__.py b/src/sqlbuild/compiler/__init__.py index d687f690a0..e69de29bb2 100644 --- a/src/sqlbuild/compiler/__init__.py +++ b/src/sqlbuild/compiler/__init__.py @@ -1 +0,0 @@ -"""Compiler package.""" diff --git a/src/sqlbuild/compiler/analysis_session/__init__.py b/src/sqlbuild/compiler/analysis_session/__init__.py deleted file mode 100644 index e973b2dc08..0000000000 --- a/src/sqlbuild/compiler/analysis_session/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The native model analysis session.""" diff --git a/src/sqlbuild/compiler/auditing/__init__.py b/src/sqlbuild/compiler/auditing/__init__.py index 3e85debfe7..e69de29bb2 100644 --- a/src/sqlbuild/compiler/auditing/__init__.py +++ b/src/sqlbuild/compiler/auditing/__init__.py @@ -1 +0,0 @@ -"""Compiler auditing package.""" diff --git a/src/sqlbuild/compiler/auditing/main/above.py b/src/sqlbuild/compiler/auditing/main/above.py index ede46717e8..544e4516fc 100644 --- a/src/sqlbuild/compiler/auditing/main/above.py +++ b/src/sqlbuild/compiler/auditing/main/above.py @@ -1,6 +1,6 @@ """Build an above-threshold audit bound.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/src/sqlbuild/compiler/auditing/main/below.py b/src/sqlbuild/compiler/auditing/main/below.py index 0e891b0e96..c38ebe6402 100644 --- a/src/sqlbuild/compiler/auditing/main/below.py +++ b/src/sqlbuild/compiler/auditing/main/below.py @@ -1,6 +1,6 @@ """Build a below-threshold audit bound.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/src/sqlbuild/compiler/auditing/main/outside.py b/src/sqlbuild/compiler/auditing/main/outside.py index 3f31c2cfdc..4d94ba1938 100644 --- a/src/sqlbuild/compiler/auditing/main/outside.py +++ b/src/sqlbuild/compiler/auditing/main/outside.py @@ -1,6 +1,6 @@ """Build an outside-range audit bound.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/src/sqlbuild/compiler/auditing/models.py b/src/sqlbuild/compiler/auditing/models.py deleted file mode 100644 index ef187e02d3..0000000000 --- a/src/sqlbuild/compiler/auditing/models.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Auditing identity models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from math import isfinite -from typing import cast - -from sqlbuild._native import MeasurementAuditError -from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator - - -@dataclass(frozen=True, kw_only=True) -class MeasurementThresholdBound: - """One directional threshold bound for a measurement audit.""" - - operator: ThresholdOperator - limit: float | None = None - lower: float | None = None - upper: float | None = None - - def __post_init__(self) -> None: - if self.operator in {ThresholdOperator.BELOW, ThresholdOperator.ABOVE}: - if self.limit is None or self.lower is not None or self.upper is not None: - raise MeasurementAuditError(f"{self.operator.value} threshold requires limit only") - if isinstance(self.limit, bool): - raise MeasurementAuditError("measurement threshold limit must not be boolean") - object.__setattr__(self, "limit", float(self.limit)) - if not isfinite(self.limit): - raise MeasurementAuditError("measurement threshold limit must be finite") - return - - if self.limit is not None or self.lower is None or self.upper is None: - raise MeasurementAuditError("outside threshold requires lower and upper only") - if isinstance(self.lower, bool): - raise MeasurementAuditError("measurement threshold lower must not be boolean") - if isinstance(self.upper, bool): - raise MeasurementAuditError("measurement threshold upper must not be boolean") - object.__setattr__(self, "lower", float(self.lower)) - object.__setattr__(self, "upper", float(self.upper)) - if not isfinite(self.lower): - raise MeasurementAuditError("measurement threshold lower must be finite") - if not isfinite(self.upper): - raise MeasurementAuditError("measurement threshold upper must be finite") - if self.lower > self.upper: - raise MeasurementAuditError( - "outside threshold lower must be less than or equal to upper" - ) - - -@dataclass(frozen=True, kw_only=True) -class MeasurementThresholds: - """Warning and error policy for a measurement audit.""" - - warn: MeasurementThresholdBound | None = None - error: MeasurementThresholdBound | None = None - - def __post_init__(self) -> None: - if self.warn is None and self.error is None: - raise MeasurementAuditError("at least one measurement threshold is required") - if self.warn is None or self.error is None: - return - if self.warn.operator != self.error.operator: - raise MeasurementAuditError( - "mixed measurement threshold operators are unsupported in v1" - ) - if self.warn.operator == ThresholdOperator.BELOW: - if cast(float, self.error.limit) >= cast(float, self.warn.limit): - raise MeasurementAuditError("below error limit must be less than warn limit") - return - if self.warn.operator == ThresholdOperator.ABOVE: - if cast(float, self.error.limit) <= cast(float, self.warn.limit): - raise MeasurementAuditError("above error limit must be greater than warn limit") - return - if cast(float, self.error.lower) >= cast(float, self.warn.lower) or cast( - float, self.error.upper - ) <= cast(float, self.warn.upper): - raise MeasurementAuditError("outside error range must strictly contain warn range") - - -@dataclass(frozen=True, kw_only=True) -class MeasurementContract: - """Columns and sample unit returned by a measurement query.""" - - value_column: str - sample_count_column: str | None = None - sample_unit: str | None = None - - def __post_init__(self) -> None: - if not self.value_column.strip(): - raise MeasurementAuditError("measurement contract value_column must be non-empty") - if self.sample_count_column is not None and not self.sample_count_column.strip(): - raise MeasurementAuditError( - "measurement contract sample_count_column must be non-empty" - ) - if self.sample_unit is not None and not self.sample_unit.strip(): - raise MeasurementAuditError("measurement contract sample_unit must be non-empty") - - -@dataclass(frozen=True) -class AuditIdentity: - """Stable identity values for one planned audit binding.""" - - binding_key: str - audit_name: str - definition_fingerprint: str - execution_fingerprint: str - severity: AuditSeverity - run_scope_phase: str - attachment_kind: str - attached_target_name: str | None = None - attached_column_name: str | None = None - always_run: bool = False - - -@dataclass(frozen=True) -class AuditGateIdentity: - """Aggregate identity for a model's runtime audit gate.""" - - binding_set_hash: str - blocking_set_hash: str - audits: tuple[AuditIdentity, ...] diff --git a/src/sqlbuild/compiler/compile/__init__.py b/src/sqlbuild/compiler/compile/__init__.py index 033199b706..e69de29bb2 100644 --- a/src/sqlbuild/compiler/compile/__init__.py +++ b/src/sqlbuild/compiler/compile/__init__.py @@ -1 +0,0 @@ -"""Compile package.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/__init__.py b/src/sqlbuild/compiler/compile/_helpers/__init__.py index bac4869b96..e69de29bb2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/__init__.py +++ b/src/sqlbuild/compiler/compile/_helpers/__init__.py @@ -1 +0,0 @@ -"""Helper utilities for compile input attachment.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index 778eaaefb4..c550729a37 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -7,10 +7,13 @@ from functools import partial from pathlib import Path -from sqlbuild._native import snippet_help +from sqlbuild._native import ( + DiscoveredProjectInputs, + DiscoveredProvider, + SourceLocation, + snippet_help, +) from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider -from sqlbuild.spec.contracts.models import SourceLocation def python_resource_rows( diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index 48af17b0f7..297577b5ab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -5,13 +5,13 @@ from pathlib import Path from sqlbuild._native import ( + DiscoveredProjectInputs, + TargetConfig, build_compile_target_context, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.spec.contracts.models import TargetConfig def target_context_with_macro_call_store( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py index a6c0b0fcaa..12893298c9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/user_macros.py @@ -14,31 +14,27 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild._native import validate_resource_identity -from sqlbuild.compiler.compile.exceptions import ( - CompileInputError, - MacroDeclarationLookupError, -) -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + ConstantDeclaration, + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DeclarationResolutionContext, + DiscoveredMacroFile, + EnumDeclaration, LoadedMacro, - MacroContext, + SqlValue, + SqlValueKind, StaticMacroExport, StaticMacroFault, StaticMacroInventory, + validate_resource_identity, ) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredMacroFile, - EnumDeclaration, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, +from sqlbuild.compiler.compile.exceptions import ( + CompileInputError, + MacroDeclarationLookupError, ) -from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind +from sqlbuild.compiler.compile.models import MacroContext _OBJECT_ENTRY_LENGTH: int = 2 _MACRO_MODULE_LOAD_LOCK: threading.RLock = threading.RLock() diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py deleted file mode 100644 index 7ab946ed48..0000000000 --- a/src/sqlbuild/compiler/compile/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Stable constants for compile-time helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import SqlTestMode - -MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" - - -MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" - - -DEFAULT_SQL_TEST_MODE: SqlTestMode = SqlTestMode.MODEL diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index d038ba2808..184084b274 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -2,97 +2,7 @@ from __future__ import annotations -from pathlib import Path - -from sqlbuild.spec.contracts.models import SourceLocation - - -class CompileInputError(ValueError): - """Raised when discovered inputs cannot be attached into a compile view.""" - - code: str = "P001" - - def __init__( - self, - message: str, - *, - code: str | None = None, - help: str | None = None, - ) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class MacroArgumentError(CompileInputError): - """Raised when one macro call's arguments do not parse; `offset` is in the expanded SQL.""" - - def __init__( # noqa: PLR0913 - self, - *, - detail: str, - help: str, - macro_name: str, - file_path: Path, - offset: int, - relative_position: tuple[int, int], - location: SourceLocation | None = None, - ) -> None: - line, column = ( - (location.line, location.column) if location is not None else relative_position - ) - scope: str = ( - f"of '@{macro_name}' in '{file_path}'" if location is not None else f"in '{file_path}'" - ) - suffix: str = "" if location is not None else f" of the '@{macro_name}' arguments" - super().__init__( - f"Macro arguments {scope} {detail} at line {line}, column {column}{suffix}", - help=help, - ) - self.detail: str = detail - self.macro_name: str = macro_name - self.file_path: Path = file_path - self.offset: int = offset - self.relative_position: tuple[int, int] = relative_position - self.location: SourceLocation | None = location - - def shifted(self, by: int) -> MacroArgumentError: - """The same error, its offset moved by `by` into the enclosing SQL.""" - - return MacroArgumentError( - detail=self.detail, - help=self.help or "", - macro_name=self.macro_name, - file_path=self.file_path, - offset=self.offset + by, - relative_position=self.relative_position, - ) - - def located(self, location: SourceLocation) -> MacroArgumentError: - """The same error at its authored file location.""" - - return MacroArgumentError( - detail=self.detail, - help=self.help or "", - macro_name=self.macro_name, - file_path=self.file_path, - offset=self.offset, - relative_position=self.relative_position, - location=location, - ) - - -class SqlTestReferenceError(CompileInputError): - """Raised when a SQL-test CTE calls a reference the test query cannot resolve.""" - - code: str = "P013" - - def __init__(self, message: str, *, location: SourceLocation, help: str) -> None: - super().__init__( - f"{location.path.as_posix()}:{location.line}:{location.column}: {message}", help=help - ) - self.location: SourceLocation = location +from sqlbuild._native import CompileInputError class MacroDeclarationLookupError(CompileInputError, KeyError): diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 453c264aba..6bfcbc16d5 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1,791 +1,5 @@ -"""Structured runtime models for project compilation.""" +"""Macro context type that project macros import for their `ctx` annotation.""" -from __future__ import annotations +from sqlbuild._native import MacroContext -import inspect -from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass, field -from functools import cached_property -from pathlib import Path -from typing import Any, overload - -import sqlbuild._native as _native -from sqlbuild._native import ( - CollectionRendering, - ColumnLineageConfidence, - ColumnTransformKind, - CompiledDirectLogicSqlTestPayload, - CompiledLineageColumnFact, - CompiledModelSqlTestPayload, - CompiledSqlExpansion, - CompiledSqlTestResource, - CompileModelConfig, - CompileSqlScenarioCte, - DeclarationExpansionContext, - DeclarationScopeResolver, - DiagnosticPhase, - DiagnosticSeverity, - DiscoveredAuditBlock, - DiscoveredAuditFile, - DiscoveredSchemaFile, - DiscoveredSqlHookFile, - DiscoveredSqlTestBlock, - DynamicColumnContractProof, - LiteralSqlRelation, - ModelSchemaDeclaration, - RelatedLocation, -) -from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, - RelationKeyRow, - SqlTestMode, - TypedSqlValueRenderer, -) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, - EnumDeclaration, - SqlTestParameterDeclaration, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - ScopeIndex, - UsageRecord, - VisibilityRecord, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ( - ScenarioConfig, - SchemaModelEntry, - SettingsConfig, - SourceLocation, - TargetConfig, -) -from sqlbuild.sql_values.models import SqlValue - - -@dataclass(frozen=True) -class CompilerDiagnostic: - """One project diagnostic produced by compile-time checks.""" - - phase: DiagnosticPhase | str - severity: DiagnosticSeverity | str - code: str - message: str - resource_type: CompiledResourceType | str | None = None - resource_name: str | None = None - column_name: str | None = None - path: Path | None = None - line: int | None = None - column: int | None = None - location: SourceLocation | None = None - related_locations: tuple[RelatedLocation, ...] = field(default_factory=tuple) - help: str | None = None - notes: tuple[str, ...] = () - affected_rules: tuple[str, ...] = () - - def __post_init__(self) -> None: - object.__setattr__(self, "phase", DiagnosticPhase(self.phase)) - object.__setattr__(self, "severity", DiagnosticSeverity(self.severity)) - if self.resource_type is not None: - object.__setattr__( - self, - "resource_type", - CompiledResourceType(self.resource_type), - ) - if ( - self.location is None - and self.path is not None - and self.line is not None - and self.column is not None - ): - object.__setattr__( - self, - "location", - SourceLocation(path=self.path, line=self.line, column=self.column), - ) - if self.location is not None: - object.__setattr__(self, "path", self.location.path) - object.__setattr__(self, "line", self.location.line) - object.__setattr__(self, "column", self.location.column) - - @property - def is_error(self) -> bool: - """Return whether this diagnostic should fail the command.""" - - return self.severity == DiagnosticSeverity.ERROR - - -@dataclass(frozen=True) -class InferredColumn: - """One output column inferred from query SQL via sql_analysis parsing.""" - - name: str - type: str | None = None - nullability: InferredNullability = InferredNullability.UNKNOWN - - -@dataclass(frozen=True) -class LoadedMacro: - """One loaded project macro available for compile-time SQL expansion.""" - - name: str - file_path: Path - relative_path: Path - raw_source: str - function: Callable[..., object] - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - - @cached_property - def injects_context(self) -> bool: - """Whether the first positional parameter receives the injected macro context.""" - - parameters: tuple[inspect.Parameter, ...] = tuple( - inspect.signature(self.function).parameters.values() - ) - return bool( - parameters - and parameters[0].kind - in {inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD} - and parameters[0].name == MACRO_CONTEXT_PARAMETER_NAME - ) - - -@dataclass(frozen=True) -class StaticMacroExport: - """Value-free AST inventory for one project-owned macro export.""" - - name: str - relative_path: Path - parameters: tuple[str, ...] - line: int - source_digest: str - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class StaticMacroFault: - """One value-free static macro inventory fault.""" - - relative_path: Path - message: str - - -@dataclass(frozen=True) -class StaticMacroInventory: - """AST macro exports and faults collected without module execution.""" - - exports: tuple[StaticMacroExport, ...] = field(default_factory=tuple) - faults: tuple[StaticMacroFault, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class MacroContext: - """Expose target settings and caller-visible declarations to a Python SQL macro.""" - - adapter_name: str - sql_analysis_enabled: bool - target_name: str | None - vars: dict[str, object] = field(default_factory=dict) - constants: Mapping[str, object] = field(default_factory=dict) - enums: Mapping[str, Mapping[str, str | int]] = field(default_factory=dict) - _value_renderer: TypedSqlValueRenderer | None = field(default=None, repr=False, compare=False) - _collection_rendering: CollectionRendering = field( - default=CollectionRendering.VALUE_LIST, repr=False, compare=False - ) - _constant_declarations: Mapping[str, ConstantDeclaration] = field( - default_factory=dict, repr=False, compare=False - ) - _enforce_explicit_references: bool = field(default=True, repr=False, compare=False) - - def render_constant(self, name: str) -> str: - """Render one visible constant as an adapter-safe SQL value.""" - - if self._value_renderer is None: - raise CompileInputError( - "Constant SQL rendering is available only during a project macro invocation" - ) - _ = self.constants[name] - declaration: ConstantDeclaration | None = self._constant_declarations.get(name) - if declaration is None: - raise CompileInputError( - "Constant SQL rendering requires a declaration resolved for this macro invocation" - ) - from sqlbuild._native import render_constant_declaration - - return render_constant_declaration( - declaration=declaration, - value_renderer=self._value_renderer, - collection_rendering=self._collection_rendering, - ) - - def render_enum_member(self, *, enum_name: str, member_name: str) -> str: - """Render one visible enum member as a SQL scalar literal.""" - - value: str | int = self.enums[enum_name][member_name] - from sqlbuild._native import render_enum_member_value - - return render_enum_member_value(value=value) - - -@dataclass(frozen=True) -class DeclarationResolutionContext: - """Declarations visible during one authored SQL expansion.""" - - enums: dict[str, EnumDeclaration] = field(default_factory=dict) - constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - inaccessible_enums: dict[str, DeclarationRecord] = field(default_factory=dict) - inaccessible_constants: dict[str, DeclarationRecord] = field(default_factory=dict) - enum_visibility: dict[str, tuple[VisibilityRecord, ...]] = field(default_factory=dict) - constant_visibility: dict[str, tuple[VisibilityRecord, ...]] = field(default_factory=dict) - macros: dict[str, LoadedMacro] = field(default_factory=dict) - macro_records: dict[str, DeclarationRecord] = field(default_factory=dict) - macro_visibility: dict[str, tuple[VisibilityRecord, ...]] = field(default_factory=dict) - inaccessible_macros: dict[str, DeclarationRecord] = field(default_factory=dict) - consumer: ResourceIdentity | DeclarationIdentity | None = None - - -@dataclass(frozen=True) -class ModelInputBuildContext: - """Run-constant config and macros for building model compile inputs.""" - - effective_vars: dict[str, object] - effective_settings: SettingsConfig - target_config: TargetConfig | None - effective_target_name: str | None - run_id: str - macro_context: MacroContext - loaded_macros: dict[str, LoadedMacro] - value_renderer: TypedSqlValueRenderer - collection_rendering: CollectionRendering - sql_lexical_syntax: SqlLexicalSyntax - public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) - public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - public_model_schemas: dict[str, ModelSchemaDeclaration] = field(default_factory=dict) - declaration_resolver: DeclarationScopeResolver | None = None - - @property - def declaration_expansion(self) -> DeclarationExpansionContext: - """Build the public declaration expansion context for non-model resources.""" - - return DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=self.public_enums, - constants=self.public_constants, - ), - value_renderer=self.value_renderer, - collection_rendering=self.collection_rendering, - resolver=self.declaration_resolver, - ) - - -@dataclass(frozen=True) -class CompileSqlReference: - """One logical SQL reference discovered from compiled SQL text.""" - - ref_kind: SqlReferenceKind | str - ref_name: str - ref_package: str | None = None - call_argument_count: int | None = None - - def __post_init__(self) -> None: - object.__setattr__(self, "ref_kind", SqlReferenceKind(self.ref_kind)) - - -@dataclass(frozen=True) -class CompiledLineageSourceFact: - """Compact upstream column fact extracted during SQL analysis.""" - - resource_type: CompiledResourceType | str - resource_name: str - column_name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True, eq=False) -class CompactLineageFacts(Sequence[CompiledLineageColumnFact]): - """Indexed native lineage rows with lazy object projection.""" - - string_pool: tuple[str, ...] - rows: tuple[ - tuple[int, int, int, tuple[tuple[int, int, int], ...]], - ..., - ] - resource_name_indexes: dict[int, int] = field(default_factory=dict) - resource_names: dict[int, str] = field(default_factory=dict) - _cache: dict[int, CompiledLineageColumnFact] = field( - default_factory=dict, - init=False, - repr=False, - compare=False, - ) - - def __len__(self) -> int: - return len(self.rows) - - @overload - def __getitem__(self, index: int) -> CompiledLineageColumnFact: ... - - @overload - def __getitem__(self, index: slice) -> Sequence[CompiledLineageColumnFact]: ... - - def __getitem__( - self, index: int | slice - ) -> CompiledLineageColumnFact | Sequence[CompiledLineageColumnFact]: - if isinstance(index, slice): - return tuple(self[item_index] for item_index in range(*index.indices(len(self)))) - normalized_index: int = index if index >= 0 else len(self) + index - if normalized_index < 0 or normalized_index >= len(self): - raise IndexError(index) - cached: CompiledLineageColumnFact | None = self._cache.get(normalized_index) - if cached is not None: - return cached - name_index, transform_code, confidence_code, sources = self.rows[normalized_index] - fact: CompiledLineageColumnFact = CompiledLineageColumnFact( - output_column=self.string_pool[name_index], - upstream_columns=tuple( - CompiledLineageSourceFact( - resource_type=self.string_pool[source[0]], - resource_name=self.resource_name(source[1]), - column_name=self.string_pool[source[2]], - ) - for source in sources - ), - transform_kind=self.transform_kind(transform_code), - confidence=self.confidence(confidence_code), - ) - self._cache[normalized_index] = fact - return fact - - def __iter__(self) -> Iterator[CompiledLineageColumnFact]: - return (self[index] for index in range(len(self))) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Sequence): - return NotImplemented - return tuple(self) == tuple(other) - - def resource_name(self, index: int) -> str: - """Resolve a canonical native relation index to this model's resource name.""" - - name: str | None = self.resource_names.get(index) - if name is not None: - return name - return self.string_pool[self.resource_name_indexes.get(index, index)] - - @staticmethod - def transform_kind(code: int) -> ColumnTransformKind: - return ( - ColumnTransformKind.DIRECT, - ColumnTransformKind.CAST, - ColumnTransformKind.EXPRESSION, - ColumnTransformKind.AGGREGATION, - ColumnTransformKind.STAR, - ColumnTransformKind.CONSTANT, - )[code] - - @staticmethod - def confidence(code: int) -> ColumnLineageConfidence: - return ( - ColumnLineageConfidence.UNKNOWN, - ColumnLineageConfidence.HIGH, - ColumnLineageConfidence.MEDIUM, - )[code] - - -@dataclass(frozen=True) -class CompiledObjectKey: - """Stable logical identity for one compiled resource or external dependency.""" - - resource_type: CompiledResourceType | str - name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "resource_type", CompiledResourceType(self.resource_type)) - - -@dataclass(frozen=True) -class CompileModelInput: - """One discovered model file with its attached schema metadata, if any.""" - - model_file: DiscoveredSqlModelFile - config: CompileModelConfig = field(default_factory=CompileModelConfig) - query_sql: str = "" - macro_source_sql: str = "" - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - schema_entry: SchemaModelEntry | None = None - schema_file: DiscoveredSchemaFile | None = None - sql_validation_enabled: bool = False - rejected_sql_analysis_opt_out: SourceLocation | None = None - enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) - constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - enum_columns: dict[str, EnumDeclaration] = field(default_factory=dict) - macro_deps: tuple[str, ...] = field(default_factory=tuple) - macro_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - - @property - def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: - """Each reference as `(kind, name, package, call argument count)`.""" - - return [ - (str(ref.ref_kind), ref.ref_name, ref.ref_package, ref.call_argument_count) - for ref in self.references - ] - - @property - def relation_key_row(self) -> RelationKeyRow: - """The model's declared key facts, for deriving its unique keys.""" - - return _native.relation_key_row( - "ref", self.model_file.file_path.stem, self.config.values, self.schema_entry - ) - - @property - def attached_schema_name(self) -> str | None: - """The name of the schema entry attached to the model, or `None`.""" - - return self.schema_entry.name if self.schema_entry is not None else None - - sql_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) - lint_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) - """The compile's whole expansion, variables included, which early SQL lint reuses.""" - - -@dataclass(frozen=True) -class CompiledRelationLocation: - """Logical and physical relation location resolved during compile.""" - - database: str | None - schema: str | None - name: str - qualified_name: str | None - logical_schema: str | None = None - logical_database: str | None = None - - -@dataclass(frozen=True) -class CompiledModel: - """Planner-ready compiled model metadata.""" - - key: CompiledObjectKey - deps: tuple[CompiledObjectKey, ...] - name: str - relative_path: Path - query_sql: str - config: CompileModelConfig - destination: CompiledRelationLocation - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - schema_entry: SchemaModelEntry | None = None - inferred_columns: tuple[InferredColumn, ...] | None = None - fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = None - fast_lineage_has_star: bool = False - fast_lineage_star_resolved: bool = False - authored_sql: str = "" - authored_query_sql: str = "" - output_column_locations: dict[str, SourceLocation] = field(default_factory=dict) - extract_implicit_alias_columns: bool = True - macro_deps: tuple[str, ...] = field(default_factory=tuple) - enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) - constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - enum_columns: dict[str, EnumDeclaration] = field(default_factory=dict) - binding_diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - binding_validated: bool = False - dynamic_column_contract: DynamicColumnContractProof | None = None - unchecked_output_columns: frozenset[str] = frozenset() - rejected_sql_analysis_opt_out: SourceLocation | None = None - - @property - def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: - """Each reference as `(kind, name, package, call argument count)`.""" - - return [ - (str(ref.ref_kind), ref.ref_name, ref.ref_package, ref.call_argument_count) - for ref in self.references - ] - - @property - def relation_key_row(self) -> RelationKeyRow: - """The model's declared key facts, for deriving its unique keys.""" - - return _native.relation_key_row("ref", self.name, self.config.values, self.schema_entry) - - @property - def key_row(self) -> tuple[str, str]: - """`(resource type, name)` of the model's key.""" - - return (str(self.key.resource_type), self.key.name) - - @property - def dep_rows(self) -> list[tuple[str, str]]: - """`(resource type, name)` of each dependency.""" - - return [(str(dep.resource_type), dep.name) for dep in self.deps] - - @property - def destination_row(self) -> tuple[str | None, str | None, str]: - """`(database, schema, name)` of the model's destination.""" - - return (self.destination.database, self.destination.schema, self.destination.name) - - @property - def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: - """Each inferred column as `(name, type, nullability)`, or `None`.""" - - if self.inferred_columns is None: - return None - return [ - (column.name, column.type, str(column.nullability)) for column in self.inferred_columns - ] - - @property - def schema_column_rows(self) -> list[tuple[str, str | None]] | None: - """Each declared column of the attached schema as `(name, type)`, or `None`.""" - - if self.schema_entry is None: - return None - return [(column.name, column.type) for column in self.schema_entry.columns] - - @property - def has_dynamic_columns(self) -> bool: - """Whether the attached schema declares dynamic columns.""" - - return self.schema_entry is not None and bool(self.schema_entry.dynamic_columns) - - @property - def output_column_count(self) -> int: - """Inferred columns, else the attached schema's declared columns, else zero.""" - - if self.inferred_columns is not None: - return len(self.inferred_columns) - return len(self.schema_entry.columns) if self.schema_entry is not None else 0 - - -@dataclass(frozen=True) -class CompiledAudit: - """Compiled audit metadata selected by scope dependencies.""" - - key: CompiledObjectKey - scope_deps: tuple[CompiledObjectKey, ...] - name: str - definition_name: str - audit_file: DiscoveredAuditFile - audit_block: DiscoveredAuditBlock - sql_body: str - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode.VIOLATIONS - measurement_contract: MeasurementContract | None = None - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - measure_sql: str | None = None - evidence_sql: str | None = None - evidence_limit: int | None = None - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - attached_target_kind: AttachedAuditTargetKind | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - severity: AuditSeverity | None = None - run_scope: str | None = None - always_run: bool = False - description: str | None = None - - -@dataclass(frozen=True) -class CompiledSqlScenario: - """Compiled SQL-native scenario metadata selected by inferred graph targets.""" - - key: CompiledObjectKey - name: str - scenario_file: DiscoveredSqlScenarioFile - sql_body: str - authored_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - source_fixture_names: tuple[str, ...] = field(default_factory=tuple) - ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - seed_fixture_names: tuple[str, ...] = field(default_factory=tuple) - dbt_ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - target_model_names: tuple[str, ...] = field(default_factory=tuple) - source_path: Path | None = None - ownership_root: Path | None = None - - def __post_init__(self) -> None: - if self.source_path is None: - object.__setattr__(self, "source_path", self.scenario_file.relative_path) - if self.ownership_root is None: - object.__setattr__(self, "ownership_root", self.scenario_file.ownership_root) - if not self.target_model_names: - object.__setattr__( - self, - "target_model_names", - tuple( - dict.fromkeys((*self.expected_model_names, *self.assertion_target_model_names)) - ), - ) - - -@dataclass(frozen=True) -class CompiledProject: - """Planner-ready whole-project compile output.""" - - run_id: str - effective_target_name: str | None - effective_connection: dict[str, object] - effective_vars: dict[str, object] - binding_catalog: Any | None = field(default=None, repr=False, compare=False) - effective_target_database: str | None = None - effective_target_schema: str | None = None - sql_analysis_dialect: str | None = None - sql_lexical_syntax: SqlLexicalSyntax = field(default_factory=SqlLexicalSyntax) - compile_cache_dir: Path | None = None - settings: SettingsConfig = field(default_factory=SettingsConfig) - scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - models: tuple[CompiledModelView, ...] = field(default_factory=tuple) - sources: tuple[CompiledSourceView, ...] = field(default_factory=tuple) - seeds: tuple[CompiledSeedView, ...] = field(default_factory=tuple) - functions: tuple[CompiledFunctionView, ...] = field(default_factory=tuple) - audits: tuple[CompiledAuditView, ...] = field(default_factory=tuple) - sql_tests: tuple[CompiledSqlTest, ...] = field(default_factory=tuple) - sql_scenarios: tuple[CompiledSqlScenario, ...] = field(default_factory=tuple) - loader_functions: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - enforce_explicit_references: bool = True - sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = field(default_factory=tuple) - materialization_files: tuple[DiscoveredMaterializationFile, ...] = field(default_factory=tuple) - public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) - public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - loaded_macros: dict[str, LoadedMacro] = field(default_factory=dict) - diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) - unmatched_literal_sql_relations: tuple[LiteralSqlRelation, ...] = field(default_factory=tuple) - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - scope_index: ScopeIndex = field(default_factory=ScopeIndex) - sql_expansions: dict[Path, CompiledSqlExpansion] = field( - default_factory=dict, compare=False, repr=False - ) - native_session: Any | None = field(default=None, compare=False, repr=False) - native_project: _native.NativeCompiledProject | None = field( - default=None, compare=False, repr=False - ) - - @cached_property - def lineage_graph(self) -> _native.NativeProjectGraph: - """The lineage edges, tags, model folders and names, indexed natively once per project.""" - - rows: list[tuple[tuple[str, str], list[tuple[str, str]], list[str], str | None]] = [ - ( - self._graph_key(model.key), - self._graph_keys(model.deps), - self._graph_tags(model.config.values.get("tags")), - str(model.relative_path.parent), - ) - for model in self.models - ] - rows.extend( - (self._graph_key(source.key), self._graph_keys(source.deps), [], None) - for source in self.sources - ) - rows.extend( - ( - self._graph_key(seed.key), - self._graph_keys(seed.deps), - list(seed.schema_entry.tags), - None, - ) - for seed in self.seeds - ) - rows.extend( - ( - self._graph_key(function.key), - self._graph_keys(function.deps), - list(function.tags), - None, - ) - for function in self.functions - ) - return _native.NativeProjectGraph.from_resources(rows) - - @staticmethod - def _graph_key(key: CompiledObjectKey) -> tuple[str, str]: - return (str(key.resource_type), key.name) - - @classmethod - def _graph_keys(cls, keys: Sequence[CompiledObjectKey]) -> list[tuple[str, str]]: - return [cls._graph_key(key) for key in keys] - - @staticmethod - def _graph_tags(value: object) -> list[str]: - if isinstance(value, list | tuple): - return [str(item) for item in value] - return [] - - -@dataclass(frozen=True) -class CompiledSqlTest: - """Compiled SQL-native unit test metadata selected by expected targets.""" - - key: CompiledObjectKey - scope_deps: tuple[CompiledObjectKey, ...] - name: str - test_file: DiscoveredSqlTestFile - test_block: DiscoveredSqlTestBlock - sql_body: str - mode: SqlTestMode = DEFAULT_SQL_TEST_MODE - payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = field( - default_factory=CompiledModelSqlTestPayload - ) - source_path: Path | None = None - ownership_root: Path | None = None - block_index: int | None = None - explicit_name: str | None = None - parent_name: str | None = None - case_name: str | None = None - case_index: int | None = None - case_fingerprint: str | None = None - parameter_schema: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) - parameter_values: tuple[tuple[str, SqlValue], ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - read_helper_names: tuple[str, ...] = field(default_factory=tuple) - reference_target_model_names: tuple[str, ...] = field(default_factory=tuple) - target_model_names: tuple[str, ...] = field(default_factory=tuple) - tested_resources: tuple[CompiledSqlTestResource, ...] = field(default_factory=tuple) - - def __post_init__(self) -> None: - if self.source_path is None: - object.__setattr__(self, "source_path", self.test_file.relative_path) - if self.ownership_root is None: - object.__setattr__(self, "ownership_root", self.test_file.ownership_root) - if self.block_index is None: - object.__setattr__(self, "block_index", self.test_block.test_index) - if self.explicit_name is None and self.test_block.name is not None: - object.__setattr__(self, "explicit_name", self.test_block.name) - if self.parent_name is None: - object.__setattr__( - self, - "parent_name", - self.test_block.name or self.test_file.relative_path.stem, - ) +__all__ = ("MacroContext",) diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 23f8069960..f1eac540c4 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -5,7 +5,7 @@ from enum import StrEnum from typing import TYPE_CHECKING, Any, Protocol -from sqlbuild.sql_values.models import SqlValue +from sqlbuild._native import SqlValue if TYPE_CHECKING: from collections.abc import Callable, Sequence @@ -14,51 +14,46 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompiledLineageColumnFact, + CompiledObjectKey, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompiledSqlExpansion, + CompiledSqlScenario, + CompiledSqlTest, CompileModelConfig, + CompilerDiagnostic, + CompileSqlReference, + ConstantDeclaration, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredHookFunction, + DiscoveredLoaderFunction, + DiscoveredMaterializationFile, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, DiscoveredSqlHookFile, DynamicColumnContractProof, + EnumDeclaration, FunctionArgument, FunctionReturnColumn, - LiteralSqlRelation, - ) - from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds - from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity - from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, - CompilerDiagnostic, - CompileSqlReference, InferredColumn, + LiteralSqlRelation, LoadedMacro, - ) - from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredHookFunction, - DiscoveredLoaderFunction, - DiscoveredMaterializationFile, - DiscoveredSourceFile, - EnumDeclaration, - ) - from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord - from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - from sqlbuild.spec.contracts.models import ( + MeasurementContract, + MeasurementThresholds, ScenarioConfig, SchemaModelEntry, SchemaSeedEntry, + ScopeIndex, SettingsConfig, SourceEntry, SourceLocation, + SqlLexicalSyntax, + UsageRecord, ) + from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity type RelationKeyRow = tuple[ @@ -458,3 +453,38 @@ def name(self) -> str | None: ... def assembly_row( self, ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: ... + + +class ExternalSqlReferenceResolver(Protocol): + """Resolve first-class SQLBuild references backed by external metadata.""" + + def validate_model_names(self, *, known_model_names: set[str]) -> None: + """Validate SQLBuild model names against external integration resources.""" + + def extend_sql_test_model_names(self, *, known_model_names: set[str]) -> set[str]: + """Return extra model names valid for SQL-native test targets.""" + + def extend_sql_test_source_names(self, *, known_source_names: set[str]) -> set[str]: + """Return extra source names valid for SQL-native test mocks.""" + + def extend_sql_test_seed_names(self, *, known_seed_names: set[str]) -> set[str]: + """Return extra seed names valid for SQL-native test mocks.""" + + def validate_reference( + self, + *, + ref_kind: str, + ref_name: str, + ref_package: str | None, + owner_relative_sql_path: Path, + ) -> None: + """Validate one external SQL reference from a SQLBuild-owned file.""" + + def resolve_reference( + self, + *, + ref_kind: str, + ref_name: str, + ref_package: str | None, + ) -> str | None: + """Return the physical relation for an external reference, if resolvable.""" diff --git a/src/sqlbuild/compiler/contract_adoption/__init__.py b/src/sqlbuild/compiler/contract_adoption/__init__.py deleted file mode 100644 index 4bbe706e8a..0000000000 --- a/src/sqlbuild/compiler/contract_adoption/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Physical contract comparison and repository adoption.""" diff --git a/src/sqlbuild/compiler/contracts/__init__.py b/src/sqlbuild/compiler/contracts/__init__.py deleted file mode 100644 index 9a4ad64d87..0000000000 --- a/src/sqlbuild/compiler/contracts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Model contract validation.""" diff --git a/src/sqlbuild/compiler/discovery/__init__.py b/src/sqlbuild/compiler/discovery/__init__.py index 3634fb2678..e69de29bb2 100644 --- a/src/sqlbuild/compiler/discovery/__init__.py +++ b/src/sqlbuild/compiler/discovery/__init__.py @@ -1 +0,0 @@ -"""Discovery package.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/__init__.py b/src/sqlbuild/compiler/discovery/_helpers/__init__.py index 84c54a2f0c..e69de29bb2 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/__init__.py +++ b/src/sqlbuild/compiler/discovery/_helpers/__init__.py @@ -1 +0,0 @@ -"""Discovery helpers package.""" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py index b961b51995..45f154939a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/command_output_sinks.py @@ -9,17 +9,17 @@ from typing import get_type_hints import sqlbuild._native as _native -from sqlbuild._native import EventExporterDiscoveryError -from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( - load_sink_module, - provider_usages, -) -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredCommandOutputSink, DiscoveredCommandOutputSinkDeclaration, DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, + EventExporterDiscoveryError, +) +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_sink_module, + provider_usages, ) from sqlbuild.runtime.event_exporting.main.get_command_output_sink_definition import ( get_command_output_sink_definition, diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 4134e3df5b..51216b3abe 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -13,16 +13,15 @@ from typing import get_type_hints from sqlbuild._native import ( + PYTHON_INIT_MODULE_STEM, + PYTHON_NODE_ROOT, + DiscoveredProvider, + DiscoveredProviderUsage, EventExporterDiscoveryError, ProviderDiscoveryError, PythonNodeDiscoveryError, project_relative_path, ) -from sqlbuild.compiler.discovery.constants import ( - PYTHON_INIT_MODULE_STEM, - PYTHON_NODE_ROOT, -) -from sqlbuild.compiler.discovery.models import DiscoveredProvider, DiscoveredProviderUsage from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORTER_EVENT_PARAMETER_NAME from sqlbuild.runtime.observability.models import LifecycleEvent diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 46d967b628..9ea32d11dd 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -6,7 +6,7 @@ from typing import Protocol import sqlbuild._native as _native -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredLoaderFunction, diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py deleted file mode 100644 index 4eca4a8a6c..0000000000 --- a/src/sqlbuild/compiler/discovery/constants.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public discovery constants.""" - -from __future__ import annotations - -PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.toml" - -PYTHON_INIT_MODULE_STEM: str = "__init__" -PYTHON_NODE_ROOT: str = "python" - - -SETTINGS_SECTION: str = "settings" -TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" -STAGED_PROMOTION_MODE: str = "staged" - -SQL_TESTS_OWNERSHIP_ROOT: str = "tests/unit" -SQL_SCENARIOS_OWNERSHIP_ROOT: str = "tests/scenarios" diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index 32b378c44a..8a41d0579b 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -5,28 +5,24 @@ import sqlbuild._native as _native from sqlbuild._native import ( - ProjectConfigError, - bind_event_exporter_declarations, - discover_event_exporter_declarations, - discover_provider_classes, -) -from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( - bind_command_output_sink_declarations, - discover_command_output_sink_declarations, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredCommandOutputSinkDeclaration, DiscoveredEventExporter, DiscoveredEventExporterDeclaration, DiscoveredProvider, DiscoveredRuntimeExtensions, -) -from sqlbuild.spec.contracts.models import ( + EventExportSeverity, LifecycleEventSinkFilterConfig, LifecycleEventSinksConfig, + ProjectConfigError, + bind_event_exporter_declarations, + discover_event_exporter_declarations, + discover_provider_classes, +) +from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( + bind_command_output_sink_declarations, + discover_command_output_sink_declarations, ) -from sqlbuild.spec.contracts.types import EventExportSeverity def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtensions: diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py deleted file mode 100644 index 9958fad246..0000000000 --- a/src/sqlbuild/compiler/discovery/models.py +++ /dev/null @@ -1,409 +0,0 @@ -"""Structured models for discovered project inputs.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild._native import ( - CollectionRendering, - DiscoveredAdapterFile, - DiscoveredAuditFile, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredModelSchemaFile, - DiscoveredPythonFunctionFile, - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveredSqlModelFile, - DiscoveredSqlTestBlock, - EnumMember, - NativeDiscoverySession, -) -from sqlbuild.compiler.discovery.constants import ( - SQL_SCENARIOS_OWNERSHIP_ROOT, - SQL_TESTS_OWNERSHIP_ROOT, -) -from sqlbuild.compiler.discovery.types import ( - LoaderConnectionMode, - ProjectProvider, -) -from sqlbuild.compiler.scopes.types import ScopeKind -from sqlbuild.python_nodes.models import AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef -from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS -from sqlbuild.runtime.output_capture.types import CommandOutputStream -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SourceColumnEntry, - SourceEntry, -) -from sqlbuild.spec.contracts.types import EventExportSeverity, SourceWriteStrategy -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind - - -@dataclass(frozen=True) -class SqlHookEntry: - """A model lifecycle hook that executes SQL.""" - - statement: str - name: str | None = None - relative_path: Path | None = None - definition_sql: str | None = None - kwargs: dict[str, object] | None = None - description: str | None = None - reads: tuple[SqlResourceRef, ...] = () - - -@dataclass(frozen=True) -class PythonHookEntry: - """A model lifecycle hook that invokes a discovered Python hook.""" - - name: str - kwargs: dict[str, object] - - -@dataclass(frozen=True) -class EnumDeclaration: - """One validated public or model-local enum declaration.""" - - name: str - members: tuple[EnumMember, ...] - scalar_type: str - relative_path: Path - model_name: str | None = None - - -@dataclass(frozen=True) -class ConstantDeclaration: - """One validated public or model-local constant declaration.""" - - name: str - value: SqlValue - relative_path: Path - model_name: str | None = None - render_as: CollectionRendering | None = None - - @property - def logical_type(self) -> SqlLogicalType: - """Return the normalized logical type.""" - - return self.value.logical_type - - @property - def scalar_type(self) -> str: - """Retain the existing SQL scalar type label for scalar declarations.""" - - names: dict[SqlValueKind, str] = { - SqlValueKind.STRING: "VARCHAR", - SqlValueKind.INTEGER: "INTEGER", - SqlValueKind.BOOLEAN: "BOOLEAN", - SqlValueKind.FLOAT: "FLOAT", - SqlValueKind.DECIMAL: "DECIMAL", - SqlValueKind.NULL: "NULL", - } - return names.get(self.value.kind, self.value.logical_type.display_name.upper()) - - -@dataclass(frozen=True) -class DiscoveredSourceFile: - """A discovered source declaration file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - source_entries: tuple[SourceEntry, ...] - - -@dataclass(frozen=True) -class DiscoveredSqlTestFile: - """A discovered SQL-native test file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - blocks: tuple[DiscoveredSqlTestBlock, ...] - ownership_root: Path = Path(SQL_TESTS_OWNERSHIP_ROOT) - - -@dataclass(frozen=True) -class SqlTestParameterDeclaration: - """One typed parameter declared by a SQL-native test template.""" - - name: str - value_type: SqlValueKind - nullable: bool = False - - -@dataclass(frozen=True) -class DiscoveredSqlScenarioFile: - """A discovered SQL-native scenario file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - header_values: dict[str, object] - sql_body: str - name: str - ownership_root: Path = Path(SQL_SCENARIOS_OWNERSHIP_ROOT) - sql_body_span: tuple[int, int] | None = None - - -@dataclass(frozen=True) -class DiscoveredMacroFile: - """A discovered project macro file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredMaterializationFile: - """A discovered custom materialization Python file.""" - - file_path: Path - relative_path: Path - name: str - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredProviderUsage: - """Framework-owned metadata for provider usage by a Python callable surface.""" - - provider_name: str - parameter_name: str - annotation_class_name: str | None = None - annotation_module: str | None = None - - -@dataclass(frozen=True) -class DiscoveredLoaderFunction: - """A discovered project source loader function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - destination: str | None = None - write_strategy: SourceWriteStrategy | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - contract: str | None = None - description: str | None = None - connection_mode: LoaderConnectionMode = LoaderConnectionMode.SQLBUILD - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredTaskFunction: - """A discovered project task function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - retry: RetryPolicy | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredAssetFunction: - """A discovered project asset function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = None - retry: RetryPolicy | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredCheckFunction: - """A discovered project check function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] - severity: PythonCheckSeverity = PythonCheckSeverity.ERROR - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredHookFunction: - """A discovered project model lifecycle hook function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - description: str | None = None - reads: tuple[SqlResourceRef, ...] = field(default_factory=tuple) - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - scope_kind: ScopeKind = ScopeKind.GLOBAL - ownership_root: Path | None = None - owning_path: Path | None = None - declaration_root: Path | None = None - - -@dataclass(frozen=True) -class DiscoveredAuditFactory: - """A discovered audit factory and its normalized cases.""" - - name: str - function: Callable[..., object] - file_path: Path - relative_path: Path - line: int - cases: tuple[AuditCase, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredEventExporterDeclaration: - """A discovered exporter declaration before provider binding.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - event_kinds: frozenset[str] - min_severity: EventExportSeverity - - -@dataclass(frozen=True) -class DiscoveredEventExporter: - """A discovered canonical lifecycle event exporter function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - event_kinds: frozenset[str] = EVENT_EXPORT_KINDS - min_severity: EventExportSeverity = EventExportSeverity.DEBUG - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredCommandOutputSinkDeclaration: - """A command-output sink declaration before provider binding.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - streams: frozenset[CommandOutputStream] - - -@dataclass(frozen=True) -class DiscoveredCommandOutputSink: - """A discovered command-output sink function.""" - - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - streams: frozenset[CommandOutputStream] - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredProvider: - """A discovered project provider class and validated settings object.""" - - file_path: Path - relative_path: Path - name: str - provider_class: type[ProjectProvider] - settings: ProjectProvider - - -@dataclass(frozen=True) -class DiscoveredPythonNodeFunctions: - """Discovered project Python DAG node functions grouped by kind.""" - - loaders: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) - tasks: tuple[DiscoveredTaskFunction, ...] = field(default_factory=tuple) - assets: tuple[DiscoveredAssetFunction, ...] = field(default_factory=tuple) - checks: tuple[DiscoveredCheckFunction, ...] = field(default_factory=tuple) - audit_factories: tuple[DiscoveredAuditFactory, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DiscoveredProjectInputs: - """All raw project inputs discovered from disk before semantic resolution.""" - - project_config: ProjectConfig - local_config: LocalConfig - project_dir: Path | None = None - model_files: tuple[DiscoveredSqlModelFile, ...] = field(default_factory=tuple) - enum_files: tuple[DiscoveredEnumFile, ...] = field(default_factory=tuple) - constant_files: tuple[DiscoveredConstantFile, ...] = field(default_factory=tuple) - model_schema_files: tuple[DiscoveredModelSchemaFile, ...] = field(default_factory=tuple) - sql_function_files: tuple[DiscoveredSqlFunctionFile, ...] = field(default_factory=tuple) - sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = field(default_factory=tuple) - python_function_files: tuple[DiscoveredPythonFunctionFile, ...] = field(default_factory=tuple) - schema_files: tuple[DiscoveredSchemaFile, ...] = field(default_factory=tuple) - source_files: tuple[DiscoveredSourceFile, ...] = field(default_factory=tuple) - seed_files: tuple[DiscoveredSeedFile, ...] = field(default_factory=tuple) - test_files: tuple[DiscoveredSqlTestFile, ...] = field(default_factory=tuple) - scenario_files: tuple[DiscoveredSqlScenarioFile, ...] = field(default_factory=tuple) - audit_files: tuple[DiscoveredAuditFile, ...] = field(default_factory=tuple) - macro_files: tuple[DiscoveredMacroFile, ...] = field(default_factory=tuple) - materialization_files: tuple[DiscoveredMaterializationFile, ...] = field(default_factory=tuple) - loader_functions: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) - task_functions: tuple[DiscoveredTaskFunction, ...] = field(default_factory=tuple) - asset_functions: tuple[DiscoveredAssetFunction, ...] = field(default_factory=tuple) - check_functions: tuple[DiscoveredCheckFunction, ...] = field(default_factory=tuple) - audit_factories: tuple[DiscoveredAuditFactory, ...] = field(default_factory=tuple) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - event_exporters: tuple[DiscoveredEventExporter, ...] = field(default_factory=tuple) - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) - providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) - adapter_file: DiscoveredAdapterFile | None = None - native_session: NativeDiscoverySession | None = field(default=None, compare=False, repr=False) - """The native session that read these declaration files, for later native stages. - - `None` means no session: readers must use the Python objects. Any copy that changes a - discovered collection must set it to `None` so it never describes other files. - """ - - -@dataclass(frozen=True) -class DiscoveredRuntimeExtensions: - """Command-startup providers, typed sinks, and lifecycle delivery settings.""" - - providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) - event_exporters: tuple[DiscoveredEventExporter, ...] = field(default_factory=tuple) - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) - lifecycle_shutdown_timeout_seconds: int | None = None diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py deleted file mode 100644 index 133eee9d52..0000000000 --- a/src/sqlbuild/compiler/discovery/types.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Discovery domain types.""" - -from __future__ import annotations - -from enum import StrEnum -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from sqlbuild.provider.classes.provider import Provider - -type ProjectProvider = Provider - - -class LoaderConnectionMode(StrEnum): - """How a source loader owns the destination warehouse connection.""" - - SQLBUILD = "sqlbuild" - EXTERNAL = "external" diff --git a/src/sqlbuild/compiler/fingerprints/__init__.py b/src/sqlbuild/compiler/fingerprints/__init__.py index a3e4551e45..e69de29bb2 100644 --- a/src/sqlbuild/compiler/fingerprints/__init__.py +++ b/src/sqlbuild/compiler/fingerprints/__init__.py @@ -1 +0,0 @@ -"""Compiler fingerprints package.""" diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index 586c4174a5..5a2aed808a 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -2,24 +2,5 @@ from __future__ import annotations -FINGERPRINT_TABLE_NAME: str = "_sqlbuild_fingerprints" - -COLUMN_NODE_TYPE: str = "node_type" -COLUMN_NODE_NAME: str = "node_name" -COLUMN_TARGET_DATABASE: str = "target_database" -COLUMN_TARGET_SCHEMA: str = "target_schema" -COLUMN_TARGET_NAME: str = "target_name" -COLUMN_RUN_ID: str = "run_id" -COLUMN_DEFINITION_HASH: str = "definition_hash" -COLUMN_VERSION_HASH: str = "version_hash" -COLUMN_SCHEMA_FINGERPRINT: str = "schema_fingerprint" -COLUMN_DEFINITION_B64: str = "definition_b64" -COLUMN_METADATA_JSON_B64: str = "metadata_json_b64" -COLUMN_TIMESTAMP: str = "ts" - -NODE_TYPE_MODEL: str = "model" -NODE_TYPE_SEED: str = "seed" NODE_TYPE_HOOK: str = "hook" NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" - -AUDIT_GATE_METADATA_KEY: str = "audit_gate" diff --git a/src/sqlbuild/compiler/fingerprints/models.py b/src/sqlbuild/compiler/fingerprints/models.py deleted file mode 100644 index c205f5e659..0000000000 --- a/src/sqlbuild/compiler/fingerprints/models.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Fingerprint domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - - -@dataclass(frozen=True) -class Fingerprint: - """One applied fingerprint record from a successful materialization.""" - - node_type: str - node_name: str - target_database: str | None - target_schema: str | None - target_name: str | None - run_id: str - definition_hash: str - schema_fingerprint: str - definition: str - ts: datetime - metadata_json: str = "{}" - version_hash: str = "" - - -@dataclass(frozen=True) -class FingerprintSet: - """Latest fingerprints per node name for one target schema.""" - - schema: str - fingerprints: dict[str, Fingerprint] - fingerprints_by_identity: dict[tuple[str, str], Fingerprint] | None = None - - def __post_init__(self) -> None: - if self.fingerprints_by_identity is None: - object.__setattr__( - self, - "fingerprints_by_identity", - {(fp.node_type, fp.node_name): fp for fp in self.fingerprints.values()}, - ) diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py index 6db46f9b53..e69de29bb2 100644 --- a/src/sqlbuild/compiler/frontier/__init__.py +++ b/src/sqlbuild/compiler/frontier/__init__.py @@ -1 +0,0 @@ -"""The native compiler frontier seam.""" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py deleted file mode 100644 index 02232e4b6c..0000000000 --- a/src/sqlbuild/compiler/frontier/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Compiler cache namespace and stage capture constants.""" - -from __future__ import annotations - -import itertools - -STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" -COMPILE_REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" -STAGE_CAPTURE_SEQUENCE: itertools.count[int] = itertools.count(1) diff --git a/src/sqlbuild/compiler/graph/__init__.py b/src/sqlbuild/compiler/graph/__init__.py deleted file mode 100644 index c4dd642de4..0000000000 --- a/src/sqlbuild/compiler/graph/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler graph algorithms.""" diff --git a/src/sqlbuild/compiler/lineage/__init__.py b/src/sqlbuild/compiler/lineage/__init__.py index ba0f8bbfd4..e69de29bb2 100644 --- a/src/sqlbuild/compiler/lineage/__init__.py +++ b/src/sqlbuild/compiler/lineage/__init__.py @@ -1 +0,0 @@ -"""Column lineage semantic analysis.""" diff --git a/src/sqlbuild/compiler/macro_bridge/__init__.py b/src/sqlbuild/compiler/macro_bridge/__init__.py index 73c3a31a12..e69de29bb2 100644 --- a/src/sqlbuild/compiler/macro_bridge/__init__.py +++ b/src/sqlbuild/compiler/macro_bridge/__init__.py @@ -1 +0,0 @@ -"""Native macro bridge: call-site scans, the in-compile call memo and splicing.""" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py index a88eb85027..e69de29bb2 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/__init__.py @@ -1 +0,0 @@ -"""Macro call store keys and the environment they are valid in.""" diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py index a22b1804b0..6d6ad99c4e 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_environment.py @@ -13,6 +13,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import STORE_TRACKED_ENVIRONMENT_PREFIXES, STORE_UNTRACKED_ENVIRONMENT_NAMES from sqlbuild.compiler.macro_bridge.constants import ( INSTALLED_DISTRIBUTION_SUFFIX, INSTALLED_PACKAGE_DIRECTORY_NAMES, @@ -22,8 +23,6 @@ MAIN_MODULE_NAME, MISSING_SEARCH_PATH_STAMP, PROJECT_ROOT_SEARCH_PATH_STAMP, - STORE_TRACKED_ENVIRONMENT_PREFIXES, - STORE_UNTRACKED_ENVIRONMENT_NAMES, ) from sqlbuild.compiler.macro_bridge.models import ModuleSources @@ -41,10 +40,7 @@ def store_environment(*, project_dir: Path, fingerprint: str) -> str: [ MACRO_CALL_STORE_ENVIRONMENT_VERSION, _native.BUILD_IDENTITY, - sys.version, - sys.executable, - sys.platform, - str(sys.implementation.cache_tag), + *_native.interpreter_identity(), json.dumps(_search_path_stamps(project_dir=project_dir)), json.dumps(_tracked_environment()), json.dumps(_installed_distribution_records()), diff --git a/src/sqlbuild/compiler/macro_bridge/classes/__init__.py b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py index 2e8ae1fd31..e69de29bb2 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/__init__.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/__init__.py @@ -1 +0,0 @@ -"""Macro bridge runtime classes.""" diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index f70fcd3439..0226aab70f 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -5,11 +5,6 @@ import itertools from contextvars import ContextVar -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, -) - ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) @@ -24,10 +19,3 @@ INSTALLED_RECORD_FILE_NAME: str = "RECORD" INTERPRETER_MODULE_ORIGINS: frozenset[str] = frozenset({"built-in", "frozen"}) MAIN_MODULE_NAME: str = "__main__" -STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") -STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - { - COMPILE_REUSE_DISABLE_ENV_VAR, - STAGE_CAPTURE_DIR_ENV_VAR, - } -) diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py index eaf52e580e..e69de29bb2 100644 --- a/src/sqlbuild/compiler/migrations/__init__.py +++ b/src/sqlbuild/compiler/migrations/__init__.py @@ -1 +0,0 @@ -"""Direct-mode model migration state package.""" diff --git a/src/sqlbuild/compiler/migrations/constants.py b/src/sqlbuild/compiler/migrations/constants.py deleted file mode 100644 index 07187e2a3c..0000000000 --- a/src/sqlbuild/compiler/migrations/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Stable constants for append-only model migration state.""" - -from __future__ import annotations - -MIGRATION_TABLE_NAME: str = "_sqlbuild_migrations" -MIGRATION_WRITE_ATTEMPTS: int = 3 - -COLUMN_MIGRATION_TABLE_NAME: str = "_sqlbuild_column_migrations" - -OLD_NAME_VIEW_TABLE_NAME: str = "_sqlbuild_old_name_views" diff --git a/src/sqlbuild/compiler/migrations/models.py b/src/sqlbuild/compiler/migrations/models.py deleted file mode 100644 index 115abbdb10..0000000000 --- a/src/sqlbuild/compiler/migrations/models.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Model migration state domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -from sqlbuild._native import MigrationDiscovery -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, - OldNameViewDropReason, - OldNameViewEventType, - OldNameViewStatus, -) - - -@dataclass(frozen=True) -class MigrationRelation: - """One physical relation named by a migration event.""" - - database: str | None - schema: str | None - name: str - - @property - def identity(self) -> tuple[str | None, str, str]: - """Return the case-insensitive relation identity.""" - - return ( - None if self.database is None else self.database.lower(), - "" if self.schema is None else self.schema.lower(), - self.name.lower(), - ) - - def matches(self, other: MigrationRelation) -> bool: - """Compare relations, treating an unknown database as a wildcard.""" - - left: tuple[str | None, str, str] = self.identity - right: tuple[str | None, str, str] = other.identity - if left[1:] != right[1:]: - return False - return left[0] is None or right[0] is None or left[0] == right[0] - - -@dataclass(frozen=True) -class MigrationEvent: - """One immutable fact that a origin relation was cloned over a destination.""" - - event_id: str - target_name: str | None - origin_model: str | None - origin: MigrationRelation - destination_model: str - destination: MigrationRelation - origin_version_hash: str - discovery: MigrationDiscovery - decision: MigrationDecision - run_id: str - created_at: datetime - - def mentions(self, relation: MigrationRelation) -> bool: - """Return whether this event names the relation as origin or destination.""" - - return self.origin.matches(relation) or self.destination.matches(relation) - - -@dataclass(frozen=True) -class ColumnMigrationEvent: - """One immutable fact that a column of a model relation was renamed in place.""" - - event_id: str - target_name: str | None - model_name: str - relation: MigrationRelation - origin_column: str - destination_column: str - discovery: MigrationDiscovery - decision: ColumnMigrationDecision - run_id: str - created_at: datetime - - def mentions(self, *, relation: MigrationRelation, columns: frozenset[str]) -> bool: - """Return whether this event renamed one of the columns of the relation.""" - - return self.relation.matches(relation) and bool( - {self.origin_column.lower(), self.destination_column.lower()} & columns - ) - - def renamed(self, *, origin_column: str, destination_column: str) -> bool: - """Return whether this event renamed exactly this origin column to this destination.""" - - return ( - self.origin_column.lower() == origin_column.lower() - and self.destination_column.lower() == destination_column.lower() - ) - - -@dataclass(frozen=True) -class OldNameViewEvent: - """One immutable fact about the old name of one recorded move.""" - - event_id: str - target_name: str | None - event_type: OldNameViewEventType - migration_event_id: str - destination_model: str - old: MigrationRelation - new: MigrationRelation - run_id: str - created_at: datetime - view_retention: str | None = None - archive_name: str | None = None - column_aliases: tuple[tuple[str, str], ...] = () - expires_at: datetime | None = None - drop_reason: OldNameViewDropReason | None = None - grants_copied: tuple[str, ...] | None = None - view_sql: str | None = None - - -@dataclass(frozen=True) -class OldNameViewHistory: - """The recorded move and every old-name fact that references it.""" - - move: MigrationEvent - required: OldNameViewEvent - archived: OldNameViewEvent | None = None - created: OldNameViewEvent | None = None - dropped: OldNameViewEvent | None = None - - def status(self, *, now: datetime) -> OldNameViewStatus: - """Project the current state of this move's old-name steps.""" - - if self.dropped is not None: - return OldNameViewStatus.DROPPED - if self.created is not None: - expires_at: datetime | None = self.created.expires_at - if expires_at is not None and expires_at <= now: - return OldNameViewStatus.EXPIRED - return OldNameViewStatus.LIVE - if self.archived is not None: - return OldNameViewStatus.PENDING_VIEW - return OldNameViewStatus.PENDING_ARCHIVE - - @property - def old(self) -> MigrationRelation: - """Return the relation name the compatibility view occupies.""" - - return self.required.old - - @property - def new(self) -> MigrationRelation: - """Return the relation the compatibility view reads.""" - - return self.required.new diff --git a/src/sqlbuild/compiler/migrations/types.py b/src/sqlbuild/compiler/migrations/types.py deleted file mode 100644 index ff24918059..0000000000 --- a/src/sqlbuild/compiler/migrations/types.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Stable model migration value types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class MigrationDecision(StrEnum): - """Per-run decision for one declared or discovered model migration.""" - - DONE = "done" - MIGRATE = "migrate" - REDO = "redo" - SUPERSEDED_REPLACE = "superseded_replace" - FORCED_REPLACE = "forced_replace" - CONFLICT = "conflict" - ORIGIN_MISSING = "origin_missing" - RENAMED = "renamed" - - @property - def moves_data(self) -> bool: - """Return whether this decision clones the origin over the destination.""" - - return self in _MOVING_DECISIONS - - @property - def blocks_build(self) -> bool: - """Return whether this decision stops a build regardless of compatibility.""" - - return self in _BLOCKING_DECISIONS - - @property - def checks_compatibility(self) -> bool: - """Return whether this decision evaluated the origin against the destination.""" - - return self.moves_data or self == MigrationDecision.CONFLICT - - @property - def label(self) -> str: - """Return the human-readable plan label for this decision.""" - - return self.value.replace("_", " ") - - -_MOVING_DECISIONS: frozenset[MigrationDecision] = frozenset( - { - MigrationDecision.MIGRATE, - MigrationDecision.REDO, - MigrationDecision.SUPERSEDED_REPLACE, - MigrationDecision.FORCED_REPLACE, - } -) -_BLOCKING_DECISIONS: frozenset[MigrationDecision] = frozenset( - {MigrationDecision.CONFLICT, MigrationDecision.ORIGIN_MISSING} -) - - -class MigrationPromotion(StrEnum): - """How a completed migration stage replaces or creates the destination.""" - - SWAP = "swap" - RENAME = "rename" - TRANSACTIONAL_RENAME = "transactional_rename" - - @property - def label(self) -> str: - """Return the human-readable plan label for this promotion.""" - - return self.value.replace("_", " ") - - -class ColumnMigrationDecision(StrEnum): - """Per-run decision for one declared or detected column rename.""" - - RENAME = "rename" - RECORD = "record" - DONE = "done" - CONFLICT = "conflict" - SOURCE_MISSING = "source_missing" - STILL_PRODUCED = "still_produced" - UNSUPPORTED = "unsupported" - - @property - def renames(self) -> bool: - """Return whether this decision renames the warehouse column.""" - - return self == ColumnMigrationDecision.RENAME - - @property - def records_event(self) -> bool: - """Return whether this decision appends a column migration event.""" - - return self in _RECORDING_COLUMN_DECISIONS - - @property - def blocks_build(self) -> bool: - """Return whether this decision stops a build before any execution.""" - - return self in _BLOCKING_COLUMN_DECISIONS - - @property - def label(self) -> str: - """Return the human-readable plan label for this decision.""" - - return _COLUMN_DECISION_LABELS[self] - - -class OldNameViewEventType(StrEnum): - """Immutable facts about what happened at a migrated relation's old name.""" - - REQUIRED = "required" - ORIGIN_ARCHIVED = "origin_archived" - VIEW_CREATED = "view_created" - VIEW_DROPPED = "view_dropped" - - -class OldNameViewDropReason(StrEnum): - """Why a compatibility view at an old name was dropped.""" - - EXPIRED = "expired" - EARLY = "early" - MISSING = "missing" - - -class OldNameViewStatus(StrEnum): - """Projected state of the old-name steps of one recorded move.""" - - PENDING_ARCHIVE = "pending_archive" - PENDING_VIEW = "pending_view" - LIVE = "live" - EXPIRED = "expired" - DROPPED = "dropped" - - -_RECORDING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( - {ColumnMigrationDecision.RENAME, ColumnMigrationDecision.RECORD} -) -_BLOCKING_COLUMN_DECISIONS: frozenset[ColumnMigrationDecision] = frozenset( - { - ColumnMigrationDecision.CONFLICT, - ColumnMigrationDecision.SOURCE_MISSING, - ColumnMigrationDecision.STILL_PRODUCED, - ColumnMigrationDecision.UNSUPPORTED, - } -) -_COLUMN_DECISION_LABELS: dict[ColumnMigrationDecision, str] = { - ColumnMigrationDecision.RENAME: "rename in place", - ColumnMigrationDecision.RECORD: "already renamed, record", - ColumnMigrationDecision.DONE: "done", - ColumnMigrationDecision.CONFLICT: "conflict", - ColumnMigrationDecision.SOURCE_MISSING: "source missing", - ColumnMigrationDecision.STILL_PRODUCED: "source still produced", - ColumnMigrationDecision.UNSUPPORTED: "unsupported", -} diff --git a/src/sqlbuild/compiler/pipeline/__init__.py b/src/sqlbuild/compiler/pipeline/__init__.py index 235d3cea5e..e69de29bb2 100644 --- a/src/sqlbuild/compiler/pipeline/__init__.py +++ b/src/sqlbuild/compiler/pipeline/__init__.py @@ -1 +0,0 @@ -"""Compiler orchestration pipeline package.""" diff --git a/src/sqlbuild/compiler/pipeline/_helpers/__init__.py b/src/sqlbuild/compiler/pipeline/_helpers/__init__.py index e04a1d8f1a..e69de29bb2 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/__init__.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/__init__.py @@ -1 +0,0 @@ -"""Pipeline helpers.""" diff --git a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py index a03e62ffdc..c8225f220e 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py @@ -7,8 +7,7 @@ from types import ModuleType from typing import Any -from sqlbuild.compiler.discovery.models import DiscoveredMaterializationFile -from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild._native import DiscoveredMaterializationFile, PlannerInputError from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult diff --git a/src/sqlbuild/compiler/pipeline/models.py b/src/sqlbuild/compiler/pipeline/models.py deleted file mode 100644 index db16309d72..0000000000 --- a/src/sqlbuild/compiler/pipeline/models.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Compiler pipeline models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from functools import cached_property -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import ( - LineageGraphViews, - PythonPlanEntry, - lineage_graph_views, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import ( - PlanOutput, -) - - -@dataclass(frozen=True) -class ProjectGraph: - """Static compiled project graph held natively; dict views are built only when read.""" - - project: CompiledProjectView - native: _native.NativeProjectGraph = field(compare=False, repr=False) - - @cached_property - def lineage_views(self) -> LineageGraphViews: - """Lineage edges and selector indexes as dicts, converted from the native graph once.""" - - return lineage_graph_views(self.native) - - @property - def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Lineage upstream edges, SQL tests stripped.""" - - return self.lineage_views.upstream_deps - - @property - def downstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Lineage downstream edges sorted by `(resource type, name)`.""" - - return self.lineage_views.downstream_deps - - @property - def tag_index(self) -> dict[str, frozenset[CompiledObjectKey]]: - """Tag to tagged models, seeds and functions.""" - - return self.lineage_views.tag_index - - @property - def path_index(self) -> dict[CompiledObjectKey, str]: - """Model key to its folder below `models/`.""" - - return self.lineage_views.path_index - - @property - def all_keys(self) -> dict[str, CompiledObjectKey]: - """Selector name to key.""" - - return self.lineage_views.all_keys - - -@dataclass(frozen=True) -class CompilePipelineResult: - """Complete output from the compile-and-plan pipeline.""" - - project: CompiledProjectView - plan_output: PlanOutput - custom_materializations: dict[str, Callable[..., Any]] = field(default_factory=dict) - python_node_names: frozenset[str] = field(default_factory=frozenset) - python_plan_entries: tuple[PythonPlanEntry, ...] = field(default_factory=tuple) - compile_seconds: float | None = None - planning_seconds: float | None = None diff --git a/src/sqlbuild/compiler/planner/__init__.py b/src/sqlbuild/compiler/planner/__init__.py index 8d15113566..e69de29bb2 100644 --- a/src/sqlbuild/compiler/planner/__init__.py +++ b/src/sqlbuild/compiler/planner/__init__.py @@ -1 +0,0 @@ -"""Planner package.""" diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py deleted file mode 100644 index cffbab89c0..0000000000 --- a/src/sqlbuild/compiler/planner/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Planner domain constants.""" - -from __future__ import annotations - -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.cursor_algebra.types import BoundSentinel - -SELECTOR_KIND_SEPARATOR: str = ":" -MICROBATCH_START_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.START) -MICROBATCH_END_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.END) -MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" -SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" -SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py deleted file mode 100644 index 18882d134f..0000000000 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Expected planner-stage exception types.""" - -from __future__ import annotations - -from typing import Any - - -class PlannerInputError(ValueError): - """Raised when planner inputs cannot be resolved safely.""" - - code: str = "S000" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class NativeSqlTestPlanningError(PlannerInputError): - """Raised when the native SQL-test planning boundary returns invalid data.""" - - -class FutureCursorSafetyError(PlannerInputError): - """Raised when an effective cursor exceeds the configured future limit.""" - - -class MaximumAutomaticStartError(PlannerInputError): - """Raised when automatic cursor recovery would exceed its configured horizon.""" - - def __init__(self, message: str, *, evidence: Any | None = None) -> None: - super().__init__(message) - self.evidence = evidence diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py deleted file mode 100644 index eedaf65977..0000000000 --- a/src/sqlbuild/compiler/planner/models.py +++ /dev/null @@ -1,1051 +0,0 @@ -"""Planner domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime -from decimal import Decimal, InvalidOperation -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - BackfillResult, - ColumnRenameHint, - CursorWatermarkMode, - DirectSourceFreshnessPlanningResult, - FunctionReturnColumn, - GraphResourceKind, - MigrationCompatibility, - MigrationDiscovery, - OldNameViewAction, - PlanProviderUsage, - PlanWarning, - ScenarioArtifactName, - ScenarioGraphPlan, - SchemaAction, - SchemaChangeKind, - SchemaColumnSource, - SqlTestAssertionStep, -) -from sqlbuild._native import cursor_parse as parse -from sqlbuild.adapter.contract.models import ColumnInfo, RetentionRequest -from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.auditing.models import MeasurementThresholds -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditEvaluationMode, - AuditRunScope, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - FunctionLanguage, -) -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, - MigrationPromotion, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, - RetentionDirection, - RetentionPlanPhase, - ScenarioArtifactKind, -) -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import ( - DateValue, - IntegerValue, - TimestampValue, -) -from sqlbuild.cursor_algebra.models import ( - Duration as Duration, -) -from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import ( - FutureCursorsConfig, - SchemaDynamicColumnFamily, - SeedCsvSettings, - SourceEntry, - StartCursorsConfig, -) -from sqlbuild.spec.contracts.types import ( - FutureCursorAction, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - SourceWriteStrategy, -) -from sqlbuild.sql_values.models import SqlValue - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorOverrides: - """Cursor override values from CLI flags; untyped `start`/`end` await planner typing.""" - - start_ts: CursorScalar | None = None - end_ts: CursorScalar | None = None - start_int: CursorScalar | None = None - end_int: CursorScalar | None = None - start: str | None = None - end: str | None = None - - def __init__( - self, - *, - start_ts: str | CursorScalar | None = None, - end_ts: str | CursorScalar | None = None, - start_int: str | CursorScalar | None = None, - end_int: str | CursorScalar | None = None, - start: str | None = None, - end: str | None = None, - ) -> None: - object.__setattr__(self, "start", start) - object.__setattr__(self, "end", end) - object.__setattr__( - self, - "start_ts", - self._parse_timestamp(field_name="--start-cursor-ts", value=start_ts), - ) - object.__setattr__( - self, "end_ts", self._parse_timestamp(field_name="--end-cursor-ts", value=end_ts) - ) - object.__setattr__( - self, - "start_int", - self._parse_integer(field_name="--start-cursor-int", value=start_int), - ) - object.__setattr__( - self, "end_int", self._parse_integer(field_name="--end-cursor-int", value=end_int) - ) - - @property - def has_untyped(self) -> bool: - """Return whether `--start-cursor` or `--end-cursor` still needs a cursor type.""" - - return self.start is not None or self.end is not None - - def typed_as(self, *, cursor_type: CursorType) -> CursorOverrides: - """Return these overrides with `--start-cursor`/`--end-cursor` read as `cursor_type`.""" - - if not self.has_untyped: - return self - if cursor_type == CursorType.INTEGER: - return CursorOverrides( - start_ts=self.start_ts, - end_ts=self.end_ts, - start_int=self._parse_integer(field_name="--start-cursor", value=self.start) - or self.start_int, - end_int=self._parse_integer(field_name="--end-cursor", value=self.end) - or self.end_int, - ) - return CursorOverrides( - start_ts=self._parse_timestamp(field_name="--start-cursor", value=self.start) - or self.start_ts, - end_ts=self._parse_timestamp(field_name="--end-cursor", value=self.end) or self.end_ts, - start_int=self.start_int, - end_int=self.end_int, - ) - - @staticmethod - def _parse_timestamp( - *, field_name: str, value: str | CursorScalar | None - ) -> CursorScalar | None: - if value is None or not isinstance(value, str): - return value - try: - datetime.fromisoformat(value) - except (ValueError, TypeError) as error: - raise PlannerInputError( - f"{field_name} value '{value}' is not a valid ISO timestamp: {error}" - ) from None - return parse(raw=value, cursor_type="timestamp") - - @staticmethod - def _parse_integer(*, field_name: str, value: str | CursorScalar | None) -> CursorScalar | None: - if value is None or not isinstance(value, str): - return value - try: - decimal_value: Decimal = Decimal(value) - if decimal_value != int(decimal_value): - raise PlannerInputError(f"{field_name} value '{value}' is not a whole number") - parsed: CursorScalar = parse(raw=value, cursor_type="integer") - if isinstance(parsed, IntegerValue): - return parsed - except InvalidOperation: - raise PlannerInputError( - f"{field_name} value '{value}' is not a valid integer" - ) from None - raise PlannerInputError(f"{field_name} value '{value}' is not a valid integer") - - -@dataclass(frozen=True) -class EmptyCursorInputDecision: - """Cursor inputs whose lack of rows leaves no window, and whether other inputs have rows.""" - - input_names: tuple[str, ...] = () - waiting_on_empty_inputs: bool = False - - -@dataclass(frozen=True, kw_only=True, init=False) -class ModelCursorSnapshot: - """Cursor MIN/MAX values gathered from warehouse for one incremental model.""" - - target_max: CursorScalar | None - upstream_mins: tuple[CursorScalar, ...] - upstream_maxes: tuple[CursorScalar, ...] - physical_target_max: CursorScalar | None = field(default=None, compare=False) - target_eligible_max: CursorScalar | None = None - target_relation: str | None = field(default=None, compare=False) - destination_cursor_column: str | None = field(default=None, compare=False) - input_evidence: tuple[CursorInputEvidence, ...] = field(default=(), compare=False) - expected_watermark_count: int = field(default=0, compare=False) - unavailable_watermark_tags: tuple[str, ...] = () - cursor_watermark_mode: CursorWatermarkMode = CursorWatermarkMode.ALL - upstream_terminal_starts: tuple[CursorScalar, ...] = () - upstream_terminal_ends: tuple[CursorScalar, ...] = () - upstream_end_inputs: tuple[tuple[CursorScalar | None, CursorScalar | None], ...] = () - upstream_availability_ends: tuple[CursorScalar, ...] = () - empty_input_names: tuple[str, ...] = field(default=(), compare=False) - unreadable_input_names: tuple[str, ...] = field(default=(), compare=False) - - def __init__(self, **values: Any) -> None: - defaults: dict[str, object] = { - "physical_target_max": None, - "target_eligible_max": None, - "target_relation": None, - "destination_cursor_column": None, - "input_evidence": (), - "expected_watermark_count": 0, - "unavailable_watermark_tags": (), - "cursor_watermark_mode": CursorWatermarkMode.ALL, - "upstream_terminal_starts": (), - "upstream_terminal_ends": (), - "upstream_end_inputs": (), - "upstream_availability_ends": (), - "empty_input_names": (), - "unreadable_input_names": (), - } - defaults.update(values) - field_name: str - for field_name in self.__dataclass_fields__: - object.__setattr__(self, field_name, defaults[field_name]) - self.__post_init__() - - def __post_init__(self) -> None: - for field_name in ("target_max", "physical_target_max", "target_eligible_max"): - object.__setattr__( - self, field_name, CursorBounds._parse_optional(raw=getattr(self, field_name)) - ) - for field_name in ( - "upstream_mins", - "upstream_maxes", - "upstream_terminal_starts", - "upstream_terminal_ends", - "upstream_availability_ends", - ): - object.__setattr__( - self, - field_name, - tuple( - CursorBounds._parse_inferred(raw=value) for value in getattr(self, field_name) - ), - ) - object.__setattr__( - self, - "upstream_end_inputs", - tuple( - ( - CursorBounds._parse_optional(raw=physical), - CursorBounds._parse_optional(raw=terminal), - ) - for physical, terminal in self.upstream_end_inputs - ), - ) - object.__setattr__( - self, "cursor_watermark_mode", CursorWatermarkMode(self.cursor_watermark_mode) - ) - - @property - def watermarks_available(self) -> bool: - """Return whether every required physical watermark value was available.""" - - return not self.unavailable_watermark_tags - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorBounds: - """Effective cursor start and end values for one incremental model.""" - - start: Bound - end: Bound - future_safety: FutureCursorSafetyEvidence | None = None - maximum_start_safety: MaximumStartSafetyEvidence | None = None - - def __init__( - self, - *, - start: Bound | str, - end: Bound | str, - future_safety: FutureCursorSafetyEvidence | None = None, - maximum_start_safety: MaximumStartSafetyEvidence | None = None, - ) -> None: - object.__setattr__(self, "start", CursorBounds._parse_bound(raw=start)) - object.__setattr__(self, "end", CursorBounds._parse_bound(raw=end)) - object.__setattr__(self, "future_safety", future_safety) - object.__setattr__(self, "maximum_start_safety", maximum_start_safety) - - @staticmethod - def _parse_inferred(*, raw: object) -> CursorScalar: - if isinstance(raw, TimestampValue | DateValue | IntegerValue): - return raw - try: - return parse(raw=str(raw), cursor_type="integer") - except CursorAlgebraError: - return parse(raw=str(raw), cursor_type="timestamp") - - @staticmethod - def _parse_optional(*, raw: object | None) -> CursorScalar | None: - return None if raw is None else CursorBounds._parse_inferred(raw=raw) - - @staticmethod - def _parse_bound(*, raw: object) -> Bound: - if isinstance(raw, BoundSentinel): - return raw - if raw == BoundSentinel.START.value: - return BoundSentinel.START - if raw == BoundSentinel.END.value: - return BoundSentinel.END - return CursorBounds._parse_inferred(raw=raw) - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorInputEvidence: - """Observed bounds for one physical cursor input.""" - - relation: str - cursor_column: str - minimum: CursorScalar | None - maximum: CursorScalar - - def __init__( - self, - *, - relation: str, - cursor_column: str, - minimum: CursorScalar | str | None, - maximum: CursorScalar | str, - ) -> None: - object.__setattr__(self, "relation", relation) - object.__setattr__(self, "cursor_column", cursor_column) - object.__setattr__(self, "minimum", minimum) - object.__setattr__(self, "maximum", maximum) - self.__post_init__() - - def __post_init__(self) -> None: - object.__setattr__(self, "minimum", CursorBounds._parse_optional(raw=self.minimum)) - object.__setattr__(self, "maximum", CursorBounds._parse_inferred(raw=self.maximum)) - - -@dataclass(frozen=True, kw_only=True, init=False) -class FutureCursorSafetyEvidence: - """Structured evidence for one applied future-cursor cap.""" - - action: FutureCursorAction - max_distance: str - invocation_time: CursorScalar - discovered_start: CursorScalar - discovered_end: CursorScalar - applied_start: CursorScalar - applied_end: CursorScalar - maximum_allowed_start: CursorScalar - maximum_allowed_end: CursorScalar - future_start_detected: bool - future_end_detected: bool - determining_relation: str | None - determining_cursor_column: str | None - inputs: tuple[CursorInputEvidence, ...] = () - - def __init__(self, **values: Any) -> None: - values.setdefault("inputs", ()) - field_name: str - for field_name in self.__dataclass_fields__: - object.__setattr__(self, field_name, values[field_name]) - self.__post_init__() - - def __post_init__(self) -> None: - for field_name in ( - "invocation_time", - "discovered_start", - "discovered_end", - "applied_start", - "applied_end", - "maximum_allowed_start", - "maximum_allowed_end", - ): - object.__setattr__( - self, field_name, CursorBounds._parse_inferred(raw=getattr(self, field_name)) - ) - - -@dataclass(frozen=True, kw_only=True, init=False) -class MaximumStartSafetyEvidence: - """Structured evidence for an automatic-start eligibility decision.""" - - action: FutureCursorAction - max_ahead: str - invocation_time: CursorScalar - physical_target_max: CursorScalar - highest_eligible_target_max: CursorScalar | None - effective_start: CursorScalar - maximum_allowed_start: CursorScalar - target_relation: str - cursor_column: str - - def __init__( - self, - *, - action: FutureCursorAction, - max_ahead: str, - invocation_time: CursorScalar | str, - physical_target_max: CursorScalar | str, - highest_eligible_target_max: CursorScalar | str | None, - effective_start: CursorScalar | str, - maximum_allowed_start: CursorScalar | str, - target_relation: str, - cursor_column: str, - ) -> None: - for field_name, value in ( - ("action", action), - ("max_ahead", max_ahead), - ("invocation_time", invocation_time), - ("physical_target_max", physical_target_max), - ("highest_eligible_target_max", highest_eligible_target_max), - ("effective_start", effective_start), - ("maximum_allowed_start", maximum_allowed_start), - ("target_relation", target_relation), - ("cursor_column", cursor_column), - ): - object.__setattr__(self, field_name, value) - self.__post_init__() - - def __post_init__(self) -> None: - for field_name in ( - "invocation_time", - "physical_target_max", - "effective_start", - "maximum_allowed_start", - ): - object.__setattr__( - self, field_name, CursorBounds._parse_inferred(raw=getattr(self, field_name)) - ) - object.__setattr__( - self, - "highest_eligible_target_max", - CursorBounds._parse_optional(raw=self.highest_eligible_target_max), - ) - - -@dataclass(frozen=True, kw_only=True, init=False) -class CursorInputRelation: - """One cursor-bearing input relation for runtime range discovery.""" - - relation: str - cursor_column: str - cursor_grain: CursorGrain | None = None - is_model_backed: bool = False - is_runtime_produced: bool = False - terminal_cursor_start: CursorScalar | None = None - terminal_cursor_end: CursorScalar | None = None - input_name: str | None = None - input_kind: GraphResourceKind | None = None - - def __init__( - self, - *, - relation: str, - cursor_column: str, - cursor_grain: CursorGrain | str | None = None, - is_model_backed: bool = False, - is_runtime_produced: bool = False, - terminal_cursor_start: CursorScalar | str | None = None, - terminal_cursor_end: CursorScalar | str | None = None, - input_name: str | None = None, - input_kind: GraphResourceKind | None = None, - ) -> None: - object.__setattr__(self, "relation", relation) - object.__setattr__(self, "cursor_column", cursor_column) - object.__setattr__(self, "cursor_grain", cursor_grain) - object.__setattr__(self, "is_model_backed", is_model_backed) - object.__setattr__(self, "is_runtime_produced", is_runtime_produced) - object.__setattr__(self, "terminal_cursor_start", terminal_cursor_start) - object.__setattr__(self, "terminal_cursor_end", terminal_cursor_end) - object.__setattr__(self, "input_name", input_name) - object.__setattr__(self, "input_kind", input_kind) - self.__post_init__() - - def __post_init__(self) -> None: - object.__setattr__( - self, - "cursor_grain", - CursorGrain(self.cursor_grain) if self.cursor_grain is not None else None, - ) - object.__setattr__( - self, - "terminal_cursor_start", - CursorBounds._parse_optional(raw=self.terminal_cursor_start), - ) - object.__setattr__( - self, "terminal_cursor_end", CursorBounds._parse_optional(raw=self.terminal_cursor_end) - ) - - @property - def is_runtime_owned(self) -> bool: - """Return whether bounds must be discovered after scheduled upstream execution.""" - - return self.is_runtime_produced - - -@dataclass(frozen=True) -class MaximumStartPolicyInputs: - """Effective policy and materialization safety inputs for automatic starts.""" - - config: StartCursorsConfig | None = None - invocation_time: datetime | None = None - incremental_strategy: str | None = None - incremental_mode: str | None = None - - -@dataclass(frozen=True) -class SchemaFinding: - """One detected schema difference between expected and warehouse columns.""" - - kind: SchemaChangeKind - column_name: str - source: SchemaColumnSource - expected_type: str | None = None - actual_type: str | None = None - - -@dataclass(frozen=True) -class RetentionPlanEntry: - """Warehouse retention work independent of model identity actions.""" - - request: RetentionRequest - model_names: tuple[str, ...] - actual_days: int | None - effective_days: int | None - source: str - direction: RetentionDirection - phase: RetentionPlanPhase - statements: tuple[str, ...] = field(default_factory=tuple) - irreversible_warning: str | None = None - decrease_policy: str = "deny" - - @property - def decreases(self) -> bool: - """Whether this entry lowers any live retention value.""" - - return self.direction in (RetentionDirection.DECREASE, RetentionDirection.MIXED) - - -@dataclass(frozen=True) -class TableTypePlanEntry: - """Snowflake table-type work independent of model identity actions.""" - - model_name: str - destination: CompiledRelationLocation - copy_name: str - desired_type: str - actual_type: str | None - source: str - downgrade: bool - downgrade_policy: str - irreversible_warning: str | None = None - - -@dataclass(frozen=True) -class ModelMigrationPlanEntry: - """One declared or discovered model migration and its per-run decision.""" - - model_name: str - discovery: MigrationDiscovery - decision: MigrationDecision - compatibility: MigrationCompatibility - origin_model: str | None - origin: CompiledRelationLocation - destination: CompiledRelationLocation - target_name: str | None - origin_version_hash: str = "" - origin_is_transient: bool = False - stage_is_transient: bool | None = None - transfer: MigrationTransfer | None = None - transfer_fallback: MigrationTransfer | None = None - promotion: MigrationPromotion | None = None - completed_at: datetime | None = None - compatibility_findings: tuple[str, ...] = () - message: str | None = None - origin_tracked: bool = False - missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW - - @property - def origin_missing(self) -> bool: - """Return whether the declared origin was not found in this target.""" - - return self.decision == MigrationDecision.ORIGIN_MISSING - - @property - def origin_hidden(self) -> bool: - """Return whether the missing origin has build history in this target.""" - - return self.origin_missing and self.origin_tracked - - @property - def storage_transition(self) -> str | None: - """Return the Snowflake table-type transition label, when both types are known.""" - - if self.stage_is_transient is None: - return None - labels: tuple[str, str] = ("permanent", "transient") - return f"{labels[self.origin_is_transient]} -> {labels[self.stage_is_transient]}" - - @property - def target_label(self) -> str: - """Return the target name shown in migration text, naming the unnamed target 'default'.""" - - return self.target_name or "default" - - @property - def blocks_build(self) -> bool: - """Return whether this migration must stop a build before any execution.""" - - if self.origin_missing: - return ( - self.origin_tracked - or self.missing_origin_policy == MissingMigrationOriginPolicy.DENY - ) - return self.decision.blocks_build or ( - self.decision.moves_data and self.compatibility == MigrationCompatibility.INCOMPATIBLE - ) - - -@dataclass(frozen=True) -class OldNameViewPlanEntry: - """What one build does at the old name of one migrated model.""" - - model_name: str - origin: CompiledRelationLocation - destination: CompiledRelationLocation - action: OldNameViewAction - target_name: str | None - retention: str | None = None - expires_at: datetime | None = None - column_aliases: tuple[tuple[str, str], ...] = () - stores_history: bool = False - migration_event_id: str | None = None - records_requirement: bool = False - reason: str | None = None - grants_copied: int | None = None - grants_supported: bool = False - archived: bool = False - - @property - def runs_steps(self) -> bool: - """Return whether this build archives the old relation or creates the view.""" - - return self.action in (OldNameViewAction.ARCHIVE_AND_VIEW, OldNameViewAction.VIEW_ONLY) - - -@dataclass(frozen=True) -class OldNameView: - """One compatibility view recorded at an old name and not recorded as dropped.""" - - destination_model: str - old: CompiledRelationLocation - new: CompiledRelationLocation - expires_at: datetime | None - column_aliases: tuple[tuple[str, str], ...] - migration_event_id: str - target_name: str | None - name_reused_by: str | None = None - - -@dataclass(frozen=True) -class ColumnMigrationPlanEntry: - """One declared or detected in-place column rename and its per-run decision.""" - - model_name: str - destination: CompiledRelationLocation - origin_column: str - destination_column: str - discovery: MigrationDiscovery - decision: ColumnMigrationDecision - target_name: str | None - completed_at: datetime | None = None - message: str | None = None - missing_origin_policy: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW - - @property - def origin_missing(self) -> bool: - """Return whether the declared origin column was not found in this target.""" - - return self.decision == ColumnMigrationDecision.SOURCE_MISSING - - @property - def blocks_build(self) -> bool: - """Return whether this column migration must stop a build before any execution.""" - - if self.origin_missing: - return self.missing_origin_policy == MissingMigrationOriginPolicy.DENY - return self.decision.blocks_build - - -@dataclass(frozen=True) -class ModelPlanEntry: - """Per-model execution plan entry with action, reason, and resolved artifacts.""" - - key: CompiledObjectKey - name: str - relative_path: Path - materialization_type: MaterializationType - action: PlanAction - reason: PlanReason - destination: CompiledRelationLocation - fingerprint_query_sql: str - resolved_sql: str - logical_ddl: str - migration_fingerprint: str | None = None - incremental_strategy: str | None = None - incremental_mode: str | None = None - microbatch_strategy: str | None = None - cursor_watermark_mode: str | None = None - cursor_column: str | None = None - cursor_type: str | None = None - cursor_grain: str | None = None - cursor_start: str | None = None - cursor_end: str | None = None - lookback: str | None = None - cursor_bounds: CursorBounds | None = None - empty_cursor_inputs: tuple[str, ...] = () - waiting_on_empty_inputs: bool = False - cursor_input_relations: tuple[CursorInputRelation, ...] = field(default_factory=tuple) - batch_size: str | None = None - batch_concurrency: int = 1 - unaccounted_partition_policy: str | None = None - microbatch_range: CursorBounds | None = None - microbatch_limit: int | None = None - declared_microbatch_limit_action: MicrobatchLimitAction | None = None - microbatch_safety_limit: int | None = None - microbatch_safety_limit_action: MicrobatchLimitAction | None = None - microbatch_limit_count: int | None = None - microbatch_limit_action: MicrobatchLimitAction | None = None - microbatch_limit_warning: str | None = None - start_cursor_override: str | None = None - end_cursor_override: str | None = None - future_cursor_config: FutureCursorsConfig | None = None - start_cursor_config: StartCursorsConfig | None = None - invocation_time: datetime | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - merge_exclude_columns: tuple[str, ...] = field(default_factory=tuple) - snapshot_strategy: str | None = None - updated_at_column: str | None = None - check_columns: tuple[str, ...] = field(default_factory=tuple) - observed_at_column: str | None = None - historical_input: str | None = None - valid_from_column: str | None = None - valid_to_column: str | None = None - initial_valid_from: str | None = None - invalidate_hard_deletes: bool = False - snapshot_full_refresh: str | None = None - snapshot_schema_change: str | None = None - table_type: str = "transient" - on_schema_change: OnSchemaChange | None = None - type_enforcement: bool = False - declared_columns: tuple[ColumnInfo, ...] = field(default_factory=tuple) - contract_enforced: bool = False - contract_columns: tuple[ColumnInfo, ...] = field(default_factory=tuple) - contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = field(default_factory=tuple) - pre_hooks: object = None - post_hooks: object = None - previous_query_sql: str | None = None - fingerprint_metadata_json: str | None = None - previous_metadata_json: str | None = None - fingerprint_version_hash: str | None = None - previous_version_hash: str | None = None - query_changed: bool = False - config_changed: bool = False - schema_actions: tuple[SchemaAction, ...] = field(default_factory=tuple) - schema_findings: tuple[SchemaFinding, ...] = field(default_factory=tuple) - backfill: BackfillResult = field( - default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) - ) - changed_functions: tuple[str, ...] = field(default_factory=tuple) - custom_materialization_name: str | None = None - custom_config: dict[str, object] = field(default_factory=dict) - custom_placeholders: dict[str, str] = field(default_factory=dict) - column_rename_hints: tuple[ColumnRenameHint, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class CloneSourcePlanEntry: - """One managed source relation selected for target cloning.""" - - key: CompiledObjectKey - name: str - destination: CompiledRelationLocation - - -@dataclass(frozen=True) -class SeedPlanEntry: - """Per-seed execution plan entry.""" - - key: CompiledObjectKey - name: str - destination: CompiledRelationLocation - file_path: Path - columns: tuple[ColumnInfo, ...] - csv_settings: SeedCsvSettings - fingerprint_definition: str = "" - fingerprint_version_hash: str = "" - fingerprint_metadata_json: str = "{}" - action: PlanAction = PlanAction.LOAD_SEED - reason: PlanReason = PlanReason.FIRST_RUN - - -@dataclass(frozen=True) -class SourceLoadPlanEntry: - """Per-managed-source loader execution plan entry.""" - - key: CompiledObjectKey - name: str - loader: str - destination: str - resource_kind: ExecutionResourceKind = ExecutionResourceKind.SOURCE - write_strategy: SourceWriteStrategy | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - is_reload: bool = False - integration_kind: str | None = None - - -@dataclass(frozen=True) -class FunctionPlanEntry: - """Per-SQL-function execution plan entry.""" - - key: CompiledObjectKey - name: str - relative_path: Path - destination: CompiledRelationLocation - arguments: tuple[object, ...] - returns: str - body_sql: str - fingerprint_query_sql: str - fingerprint_destination: CompiledRelationLocation - return_columns: tuple[FunctionReturnColumn, ...] = field(default_factory=tuple) - language: FunctionLanguage = FunctionLanguage.SQL - source_file_path: Path | None = None - runtime_version: str | None = None - entry_point: str | None = None - packages: tuple[str, ...] = field(default_factory=tuple) - previous_query_sql: str | None = None - reason: PlanReason = PlanReason.NO_CHANGE - - -@dataclass(frozen=True) -class AuditPlanEntry: - """Per-audit execution plan entry with resolved SQL and scheduling metadata.""" - - key: CompiledObjectKey - name: str - definition_name: str - resolved_sql: str - unresolved_sql: str - attachment_kind: AuditAttachmentKind - severity: AuditSeverity - requested_run_scope: AuditRunScope - effective_run_scope: AuditRunScope - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode.VIOLATIONS - value_column: str | None = None - sample_count_column: str | None = None - sample_unit: str | None = None - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - evidence_resolved_sql: str | None = None - evidence_unresolved_sql: str | None = None - evidence_limit: int | None = None - scope_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - attached_target_kind: AttachedAuditTargetKind | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - always_run: bool = False - description: str | None = None - reads_outside_target_lineage: bool = False - - -@dataclass(frozen=True) -class ChainStep: - """One step in a chained test execution sequence.""" - - model_name: str - resolved_sql: str - expected_cte_sql: str | None = None - lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) - comparison_body_sql: str | None = None - expected_columns: tuple[str, ...] | None = None - expected_lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SqlTestPlanEntry: - """Per-test execution plan entry with chained resolution.""" - - key: CompiledObjectKey - name: str - source_path: Path | None = None - block_index: int | None = None - parent_name: str | None = None - case_name: str | None = None - case_index: int | None = None - case_fingerprint: str | None = None - parameter_schema: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) - parameter_values: tuple[tuple[str, SqlValue], ...] = field(default_factory=tuple) - mock_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_source_names: tuple[str, ...] = field(default_factory=tuple) - mock_seed_names: tuple[str, ...] = field(default_factory=tuple) - mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_table_function_names: tuple[str, ...] = field(default_factory=tuple) - chain: tuple[ChainStep, ...] = field(default_factory=tuple) - assertions: tuple[SqlTestAssertionStep, ...] = field(default_factory=tuple) - scope_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - function_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - sql_analysis_enabled: bool = True - - -@dataclass(frozen=True) -class ScenarioRelationMap: - """Resolved scenario hash prefix and scenario-owned relation names.""" - - scenario_name: str - hash_prefix: str - artifacts: tuple[ScenarioArtifactName, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScenarioRelationPlan: - """Resolved scenario relation locations for model/source/seed ref resolution.""" - - scenario_name: str - relation_map: ScenarioRelationMap - model_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - seed_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - project_source_map: dict[str, SourceEntry] = field(default_factory=dict) - source_map: dict[str, SourceEntry] = field(default_factory=dict) - source_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - ref_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - seed_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - dbt_ref_fixture_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - - -@dataclass(frozen=True) -class ScenarioFixturePlan: - """Self-contained fixture SQL planned for scenario materialization.""" - - kind: ScenarioArtifactKind - logical_name: str - destination: CompiledRelationLocation - sql: str - - -@dataclass(frozen=True) -class ScenarioExpectedExpectationPlan: - """Expected-output comparison inputs for one scenario target model.""" - - model_name: str - actual_destination: CompiledRelationLocation - expected_sql: str - - -@dataclass(frozen=True) -class ScenarioAssertionExpectationPlan: - """Zero-row assertion SQL for one scenario assertion CTE.""" - - name: str - sql: str - - -@dataclass(frozen=True) -class ScenarioExecutionPlan: - """Dry-run execution plan for a SQL scenario graph slice.""" - - key: CompiledObjectKey - name: str - graph_plan: ScenarioGraphPlan - relation_plan: ScenarioRelationPlan - fixture_plans: tuple[ScenarioFixturePlan, ...] = field(default_factory=tuple) - seed_entries: tuple[SeedPlanEntry, ...] = field(default_factory=tuple) - function_entries: tuple[FunctionPlanEntry, ...] = field(default_factory=tuple) - model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - expected_expectations: tuple[ScenarioExpectedExpectationPlan, ...] = field( - default_factory=tuple - ) - assertion_expectations: tuple[ScenarioAssertionExpectationPlan, ...] = field( - default_factory=tuple - ) - - -@dataclass(frozen=True) -class PlanOutput: - """Complete execution plan produced by the planner.""" - - execution_order: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) - seed_entries: tuple[SeedPlanEntry, ...] = field(default_factory=tuple) - source_load_entries: tuple[SourceLoadPlanEntry, ...] = field(default_factory=tuple) - function_entries: tuple[FunctionPlanEntry, ...] = field(default_factory=tuple) - audit_entries: tuple[AuditPlanEntry, ...] = field(default_factory=tuple) - test_entries: tuple[SqlTestPlanEntry, ...] = field(default_factory=tuple) - available_test_case_names: tuple[str, ...] = field(default_factory=tuple) - selected_keys: frozenset[CompiledObjectKey] = field(default_factory=frozenset) - warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) - retention_entries: tuple[RetentionPlanEntry, ...] = field(default_factory=tuple) - table_type_entries: tuple[TableTypePlanEntry, ...] = field(default_factory=tuple) - migration_entries: tuple[ModelMigrationPlanEntry, ...] = field(default_factory=tuple) - column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = field(default_factory=tuple) - old_name_view_entries: tuple[OldNameViewPlanEntry, ...] = field(default_factory=tuple) - old_name_views: tuple[OldNameView, ...] = field(default_factory=tuple) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( - default_factory=dict - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = field( - default_factory=dict - ) - model_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - seed_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - function_locations: dict[str, CompiledRelationLocation] = field(default_factory=dict) - source_map: dict[str, SourceEntry] = field(default_factory=dict) - source_read_map: dict[str, SourceEntry] = field(default_factory=dict) - python_source_read_map: dict[str, SourceEntry] = field(default_factory=dict) - hook_functions: tuple[DiscoveredHookFunction, ...] = field(default_factory=tuple) - enforce_explicit_references: bool = True - provider_usages: tuple[PlanProviderUsage, ...] = field(default_factory=tuple) - source_freshness: DirectSourceFreshnessPlanningResult | None = None - python_identity_fingerprints: dict[tuple[str, str], Fingerprint] = field(default_factory=dict) - metadata: dict[str, object] = field(default_factory=dict) - cursor_overrides: CursorOverrides | None = None - - @property - def python_source_entries(self) -> dict[str, SourceEntry]: - """Return the source entries Python reads resolve, as a selected SQL read sees them.""" - - return self.python_source_read_map or self.source_read_map or self.source_map diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py deleted file mode 100644 index 75c869829c..0000000000 --- a/src/sqlbuild/compiler/planner/types.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Planner domain types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class BackfillAction(StrEnum): - FULL = "full" - BOUNDED = "bounded" - FORWARD_ONLY = "forward" - - -class OnSchemaChange(StrEnum): - IGNORE = "ignore" - FAIL = "fail" - APPEND_NEW_COLUMNS = "append_new_columns" - SYNC_ALL_COLUMNS = "sync_all_columns" - - -class ContractPolicy(StrEnum): - NONE = "none" - ENFORCED = "enforced" - - -class RetentionDirection(StrEnum): - INCREASE = "increase" - DECREASE = "decrease" - MIXED = "mixed" - MATCH = "match" - APPLY_AFTER_CREATE = "apply_after_create" - - -class RetentionPlanPhase(StrEnum): - PRE = "pre" - POST = "post" - AFTER_CREATE = "after_create" - NONE = "none" - - -class PlanAction(StrEnum): - CREATE_VIEW = "create_view" - CREATE_TABLE = "create_table" - INCREMENTAL_APPEND = "incremental_append" - INCREMENTAL_DELETE_INSERT = "incremental_delete_insert" - INCREMENTAL_MERGE = "incremental_merge" - SNAPSHOT = "snapshot" - LOAD_SEED = "load_seed" - SKIP = "skip" - CUSTOM = "custom" - - -class PlanReason(StrEnum): - FIRST_RUN = "first_run" - RENAMED = "renamed" - FULL_REFRESH = "full_refresh" - QUERY_CHANGED = "query_changed" - FUNCTION_CHANGED = "function_changed" - CONFIG_CHANGED = "config_changed" - SCHEMA_CHANGED = "schema_changed" - NORMAL_INCREMENTAL = "normal_incremental" - NO_CHANGE = "no_change" - DISABLED = "disabled" - SOURCE_FRESHNESS_ERROR = "source_freshness_error" - EXTERNAL_UPSTREAM_FAILED = "external_upstream_failed" - - -class IncrementalStrategy(StrEnum): - APPEND = "append" - DELETE_INSERT = "delete_insert" - MERGE = "merge" - - -class MicrobatchStrategy(StrEnum): - """How an ordinary microbatch range is selected.""" - - ROLLING_WINDOW = "rolling_window" - WATERMARK = "watermark" - - -class IncrementalMode(StrEnum): - FULL = "full" - MICROBATCH = "microbatch" - - -class SnapshotStrategy(StrEnum): - TIMESTAMP = "timestamp" - CHECK = "check" - - -class HistoricalInput(StrEnum): - SNAPSHOT = "snapshot" - CHANGES = "changes" - - -class InitialValidFrom(StrEnum): - UPDATED_AT = "updated_at" - OBSERVED_AT = "observed_at" - EXECUTION_TIME = "execution_time" - - -class SnapshotSchemaChangePolicy(StrEnum): - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - APPEND_NEW_COLUMNS = "append_new_columns" - - -class CursorType(StrEnum): - TIMESTAMP = "timestamp" - INTEGER = "integer" - - -class CursorGrain(StrEnum): - SECOND = "second" - MINUTE = "minute" - HOUR = "hour" - DAY = "day" - MONTH = "month" - YEAR = "year" - - -class MaterializationType(StrEnum): - VIEW = "view" - TABLE = "table" - INCREMENTAL = "incremental" - SNAPSHOT = "snapshot" - SEED = "seed" - CUSTOM = "custom" - - @classmethod - def is_table_backed(cls, *, materialized: object | None) -> bool: - """Return whether a materialization owns a physical managed table.""" - - return materialized in {cls.TABLE, cls.INCREMENTAL, cls.SNAPSHOT} - - -class ScenarioArtifactKind(StrEnum): - SOURCE = "source" - REF = "ref" - SEED = "seed" - DBT_REF = "dbt_ref" - MODEL = "model" diff --git a/src/sqlbuild/compiler/python_nodes/__init__.py b/src/sqlbuild/compiler/python_nodes/__init__.py index 8835368d5d..e69de29bb2 100644 --- a/src/sqlbuild/compiler/python_nodes/__init__.py +++ b/src/sqlbuild/compiler/python_nodes/__init__.py @@ -1 +0,0 @@ -"""Internal Python-node compiler models and helpers.""" diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py b/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py index 69c51bd3d1..e69de29bb2 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal Python-node helper functions.""" diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py index 24edafabb6..fbc7f1b172 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py @@ -8,11 +8,12 @@ import json import textwrap from collections.abc import Callable, Iterable, Mapping +from dataclasses import asdict from pathlib import Path from types import ModuleType from typing import Any, cast -from sqlbuild.compiler.python_nodes.models import PythonIdentityDependency, PythonNodeIdentity +from sqlbuild._native import PythonIdentityDependency, PythonNodeIdentity _IGNORED_PATH_PARTS: frozenset[str] = frozenset( { @@ -66,13 +67,13 @@ def build_python_identity( "source_text": source, } metadata_payload: dict[str, object] = { - "dependencies": tuple(vars(dependency) for dependency in dependencies) + "dependencies": tuple(asdict(dependency) for dependency in dependencies) } version_hash: str = _hash_text( _stable_json( { "definition_hash": definition_hash, - "dependencies": tuple(vars(dependency) for dependency in dependencies), + "dependencies": tuple(asdict(dependency) for dependency in dependencies), } ) ) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py index 03f3fd1e3c..3e03021cf0 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/user_inventory.py @@ -5,8 +5,8 @@ from collections.abc import Callable from pathlib import Path +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition diff --git a/src/sqlbuild/compiler/python_nodes/main/__init__.py b/src/sqlbuild/compiler/python_nodes/main/__init__.py index 65271aefda..e69de29bb2 100644 --- a/src/sqlbuild/compiler/python_nodes/main/__init__.py +++ b/src/sqlbuild/compiler/python_nodes/main/__init__.py @@ -1 +0,0 @@ -"""Public entrypoints for compiler Python-node helpers.""" diff --git a/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py index 7621223123..6ce05786b1 100644 --- a/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py +++ b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py @@ -4,10 +4,9 @@ from pathlib import Path -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction +from sqlbuild._native import DiscoveredHookFunction, PythonNodeIdentity from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity def build_hook_identities( diff --git a/src/sqlbuild/compiler/python_nodes/main/identity.py b/src/sqlbuild/compiler/python_nodes/main/identity.py index 7e1296d2d7..8018b394dd 100644 --- a/src/sqlbuild/compiler/python_nodes/main/identity.py +++ b/src/sqlbuild/compiler/python_nodes/main/identity.py @@ -5,8 +5,8 @@ from collections.abc import Callable, Mapping from pathlib import Path +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.python_nodes._helpers.identity import build_python_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity def build_python_node_identity( diff --git a/src/sqlbuild/compiler/python_nodes/models.py b/src/sqlbuild/compiler/python_nodes/models.py deleted file mode 100644 index 601582f2dd..0000000000 --- a/src/sqlbuild/compiler/python_nodes/models.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Internal Python-node discovery models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild._native import ( - DiscoveredPythonAssetMetadata, - DiscoveredPythonCheckMetadata, - DiscoveredPythonLoaderMetadata, - DiscoveredPythonTaskMetadata, - PythonNodeDependencyEdge, -) -from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef - - -@dataclass(frozen=True) -class DiscoveredPythonNode: - """Shared internal view of a discovered Python DAG node.""" - - kind: PythonNodeKind - file_path: Path - relative_path: Path - name: str - function: Callable[..., object] - depends_on: tuple[Callable[..., object] | SqlResourceRef, ...] = field(default_factory=tuple) - sql_deps: tuple[SqlResourceRef, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - group: str | None = None - description: str | None = None - meta: dict[str, object] | None = None - provider_usages: tuple[DiscoveredProviderUsage, ...] = field(default_factory=tuple) - loader: DiscoveredPythonLoaderMetadata | None = None - task: DiscoveredPythonTaskMetadata | None = None - asset: DiscoveredPythonAssetMetadata | None = None - check: DiscoveredPythonCheckMetadata | None = None - identity: PythonNodeIdentity | None = None - - -@dataclass(frozen=True) -class PythonNodeGraph: - """Internal graph inventory for discovered executable Python nodes.""" - - nodes: tuple[DiscoveredPythonNode, ...] - dependency_edges: tuple[PythonNodeDependencyEdge, ...] - upstream_deps: dict[str, tuple[str, ...]] - downstream_deps: dict[str, tuple[str, ...]] - tag_index: dict[str, frozenset[str]] - path_index: dict[str, str] - nodes_by_name: dict[str, DiscoveredPythonNode] - nodes_by_typed_selector: dict[str, DiscoveredPythonNode] - - def selected_sql_refs(self, *, selected_names: frozenset[str]) -> frozenset[SqlResourceRef]: - """Return SQL references required by a selected Python-node closure.""" - - refs: set[SqlResourceRef] = set() - pending: list[str] = list(selected_names) - visited: set[str] = set() - while pending: - node_name: str = pending.pop() - if node_name in visited: - continue - visited.add(node_name) - node: DiscoveredPythonNode | None = self.nodes_by_name.get(node_name) - if node is not None: - refs.update(node.sql_deps) - pending.extend(self.upstream_deps.get(node_name, ())) - return frozenset(refs) - - -@dataclass(frozen=True) -class PythonSqlRunLifecyclePlan: - """Lifecycle-aware run classification for SQL resources and Python nodes.""" - - ingress_python_node_names: frozenset[str] - ingress_loader_names: frozenset[str] - read_side_python_node_names: frozenset[str] - - -@dataclass(frozen=True) -class PythonIdentityDependency: - """One first-party Python source object included in a Python node identity.""" - - kind: str - module: str - qualname: str - source_path: str - source_hash: str - source_text: str - - -@dataclass(frozen=True) -class PythonNodeIdentity: - """Read-only identity metadata for a SQLBuild Python node.""" - - node_type: str - node_name: str - source_path: str - source_hash: str - definition_hash: str - version_hash: str - dependencies: tuple[PythonIdentityDependency, ...] = field(default_factory=tuple) - definition_json: str = "{}" - metadata_json: str = "{}" diff --git a/src/sqlbuild/compiler/refactoring/__init__.py b/src/sqlbuild/compiler/refactoring/__init__.py deleted file mode 100644 index c2d3b86586..0000000000 --- a/src/sqlbuild/compiler/refactoring/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler-driven project refactorings: model renames, moves, and column renames.""" diff --git a/src/sqlbuild/compiler/references/__init__.py b/src/sqlbuild/compiler/references/__init__.py index 86fdc55261..e69de29bb2 100644 --- a/src/sqlbuild/compiler/references/__init__.py +++ b/src/sqlbuild/compiler/references/__init__.py @@ -1 +0,0 @@ -"""Compiler-owned SQL reference behavior.""" diff --git a/src/sqlbuild/compiler/references/types.py b/src/sqlbuild/compiler/references/types.py deleted file mode 100644 index 468a5b7108..0000000000 --- a/src/sqlbuild/compiler/references/types.py +++ /dev/null @@ -1,76 +0,0 @@ -"""SQL reference type-layer declarations.""" - -from __future__ import annotations - -from enum import StrEnum -from pathlib import Path -from typing import Literal, Protocol - - -class SqlReferenceKind(StrEnum): - REF = "ref" - SEED = "seed" - SOURCE = "source" - DBT_REF = "dbt_ref" - UDF = "udf" - TABLE_FUNCTION = "table_fn" - - @property - def function_name(self) -> str: - return f"__{self.value}" - - @property - def fixture_cte_prefix(self) -> str: - return f"{self.function_name}__" - - def example_call(self, *args: str, quote: Literal["'", '"'] = "'") -> str: - quoted_args: str = ", ".join( - f"{quote}{arg.replace(quote, quote + quote)}{quote}" for arg in args - ) - return f"{self.function_name}({quoted_args})" - - def placeholder_call(self, placeholder: str = "") -> str: - return f"{self.function_name}({placeholder})" - - -class ExternalSqlReferenceResolver(Protocol): - """Resolve first-class SQLBuild references backed by external metadata.""" - - def validate_model_names(self, *, known_model_names: set[str]) -> None: - """Validate SQLBuild model names against external integration resources.""" - - def extend_sql_test_model_names(self, *, known_model_names: set[str]) -> set[str]: - """Return extra model names valid for SQL-native test targets.""" - - def extend_sql_test_source_names(self, *, known_source_names: set[str]) -> set[str]: - """Return extra source names valid for SQL-native test mocks.""" - - def extend_sql_test_seed_names(self, *, known_seed_names: set[str]) -> set[str]: - """Return extra seed names valid for SQL-native test mocks.""" - - def validate_reference( - self, - *, - ref_kind: str, - ref_name: str, - ref_package: str | None, - owner_relative_sql_path: Path, - ) -> None: - """Validate one external SQL reference from a SQLBuild-owned file.""" - - def resolve_reference( - self, - *, - ref_kind: str, - ref_name: str, - ref_package: str | None, - ) -> str | None: - """Return the physical relation for an external reference, if resolvable.""" - - -class HardCodedRelationOwnerKind(StrEnum): - """The kind of Python code that names a project relation in SQL.""" - - NODE = "node" - HOOK = "hook" - LOADER = "loader" diff --git a/src/sqlbuild/compiler/resource_names/__init__.py b/src/sqlbuild/compiler/resource_names/__init__.py deleted file mode 100644 index 2dc62df346..0000000000 --- a/src/sqlbuild/compiler/resource_names/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiler-owned resource name contracts.""" diff --git a/src/sqlbuild/compiler/scopes/__init__.py b/src/sqlbuild/compiler/scopes/__init__.py deleted file mode 100644 index f49da181bf..0000000000 --- a/src/sqlbuild/compiler/scopes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Canonical compiler-owned declaration scope domain.""" diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py deleted file mode 100644 index ca713943b6..0000000000 --- a/src/sqlbuild/compiler/scopes/models.py +++ /dev/null @@ -1,435 +0,0 @@ -"""Immutable models for the canonical compiler scope index.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass, field -from types import MappingProxyType - -from sqlbuild.compiler.scopes.types import ( - CompletenessSection, - DeclarationKind, - DiagnosticSeverity, - GrantKind, - InaccessibleReason, - OwnershipRootKind, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, - UsageKind, - VisibilityReason, -) - - -@dataclass(frozen=True, order=True) -class ResourceIdentity: - """Stable kind-qualified identity of one authored resource.""" - - kind: ResourceKind - name: str - - -@dataclass(frozen=True, order=True) -class DeclarationIdentity: - """Stable declaration identity, including a private declaration owner.""" - - kind: DeclarationKind - name: str - owner: ResourceIdentity | None = None - - -@dataclass(frozen=True, order=True) -class OwnershipRoot: - """Normalized project-relative directory that owns resource scope.""" - - path: str - kind: OwnershipRootKind = OwnershipRootKind.RESOURCE - resource_kind: ResourceKind | None = None - - -@dataclass(frozen=True) -class ResourceRecord: - """One resource participating in lexical declaration scope.""" - - identity: ResourceIdentity - path: str - ownership_root: OwnershipRoot - - -@dataclass(frozen=True) -class MacroMetadata: - """Safe macro metadata required by scope and dependency inspection.""" - - parameters: tuple[str, ...] = field(default_factory=tuple) - dependencies: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - source_digest: str = "" - - -@dataclass(frozen=True) -class EnumMemberMetadata: - """One value-free authored enum member identity.""" - - name: str - - -@dataclass(frozen=True) -class EnumMetadata: - """Safe enum metadata containing member names and their shared scalar type.""" - - members: tuple[EnumMemberMetadata, ...] - scalar_type: str - - -@dataclass(frozen=True) -class ConstantMetadata: - """Value-free normalized metadata for one typed constant.""" - - logical_type: str - collection_kind: str | None = None - item_count: int | None = None - nullable: bool = False - render_as: str | None = None - - -@dataclass(frozen=True) -class DeclarationRecord: - """One declaration definition and its compiler-owned scope metadata.""" - - identity: DeclarationIdentity - path: str - line: int - column: int - scope: ScopeKind - ownership_root: OwnershipRoot - owning_path: str | None = None - macro: MacroMetadata | None = None - enum: EnumMetadata | None = None - constant: ConstantMetadata | None = None - - -@dataclass(frozen=True) -class UsageRecord: - """A runtime or declaration dependency edge.""" - - consumer: ResourceIdentity | DeclarationIdentity - declaration: DeclarationIdentity - kind: UsageKind = UsageKind.RUNTIME - through: ResourceIdentity | DeclarationIdentity | None = None - enum_member: str | None = None - - -@dataclass(frozen=True) -class GrantRecord: - """A compiler relationship granting a public declaration to a resource.""" - - resource: ResourceIdentity - declaration: DeclarationIdentity - through: ResourceIdentity | DeclarationIdentity - kind: GrantKind = GrantKind.EXPECTED_MODEL - - -@dataclass(frozen=True) -class VisibilityRecord: - """A resolved positive visibility fact with stable provenance.""" - - resource: ResourceIdentity - declaration: DeclarationIdentity - reason: VisibilityReason - through: ResourceIdentity | DeclarationIdentity | None = None - - -@dataclass(frozen=True) -class InaccessibleRecord: - """A resolved negative visibility fact with stable provenance.""" - - resource: ResourceIdentity - declaration: DeclarationIdentity - reason: InaccessibleReason - - -@dataclass(frozen=True) -class ScopeDiagnostic: - """Stable diagnostic retained even when an index is partial.""" - - code: ScopeDiagnosticCode - message: str - severity: DiagnosticSeverity = DiagnosticSeverity.ERROR - path: str | None = None - line: int | None = None - column: int | None = None - declaration: DeclarationIdentity | None = None - resource: ResourceIdentity | None = None - - -@dataclass(frozen=True) -class ScopeCompleteness: - """Independent completeness facts for partially invalid projects.""" - - discovery: bool = True - static_visibility: bool = True - runtime_usage: bool = True - relationships: bool = True - placement: bool = True - promotion_impact: bool = True - - @property - def complete(self) -> bool: - """Return whether every index section is complete.""" - - return all(self.as_mapping().values()) - - def as_mapping(self) -> Mapping[CompletenessSection, bool]: - """Return immutable section completeness keyed by stable names.""" - - return MappingProxyType( - { - CompletenessSection.DISCOVERY: self.discovery, - CompletenessSection.STATIC_VISIBILITY: self.static_visibility, - CompletenessSection.RUNTIME_USAGE: self.runtime_usage, - CompletenessSection.RELATIONSHIPS: self.relationships, - CompletenessSection.PLACEMENT: self.placement, - CompletenessSection.PROMOTION_IMPACT: self.promotion_impact, - } - ) - - -@dataclass(frozen=True) -class ScopeIndex: - """Canonical deterministic facts used by every future scope consumer.""" - - ownership_roots: tuple[OwnershipRoot, ...] = field(default_factory=tuple) - resources: tuple[ResourceRecord, ...] = field(default_factory=tuple) - declarations: tuple[DeclarationRecord, ...] = field(default_factory=tuple) - usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - grants: tuple[GrantRecord, ...] = field(default_factory=tuple) - visibility: tuple[VisibilityRecord, ...] = field(default_factory=tuple) - inaccessible: tuple[InaccessibleRecord, ...] = field(default_factory=tuple) - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - completeness: ScopeCompleteness = field(default_factory=ScopeCompleteness) - - -@dataclass(frozen=True) -class DeclarationVisibilityIndex: - """Declaration positions grouped by the scope facts that make them visible.""" - - identities: tuple[DeclarationIdentity, ...] - global_positions: tuple[int, ...] - private_positions: Mapping[ResourceIdentity, tuple[int, ...]] - local_positions: Mapping[str, tuple[int, ...]] - inherited_positions: Mapping[str, tuple[int, ...]] - - -@dataclass(frozen=True) -class ScopeLookup: - """Immutable indexes over one canonical ``ScopeIndex``.""" - - index: ScopeIndex - resources: Mapping[ResourceIdentity, tuple[ResourceRecord, ...]] - resources_by_path: Mapping[str, tuple[ResourceRecord, ...]] - declarations: Mapping[DeclarationIdentity, tuple[DeclarationRecord, ...]] - usages_by_consumer: Mapping[ResourceIdentity | DeclarationIdentity, tuple[UsageRecord, ...]] - usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] - grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] - visibility_index: DeclarationVisibilityIndex - native: object | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True) -class ScopeTargetQuery: - """Non-throwing qualified-identity or path lookup result.""" - - value: str | ResourceIdentity | DeclarationIdentity - matches: tuple[ResourceRecord, ...] = field(default_factory=tuple) - declaration_matches: tuple[DeclarationRecord, ...] = field(default_factory=tuple) - - @property - def unknown(self) -> bool: - """Return whether the target is absent rather than merely inaccessible.""" - - return not self.matches and not self.declaration_matches - - -@dataclass(frozen=True) -class VisibilityResolution: - """Complete static visibility classification for one queried target.""" - - target: ScopeTargetQuery - visible: tuple[VisibilityRecord, ...] = field(default_factory=tuple) - inaccessible: tuple[InaccessibleRecord, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DeclarationVisibility: - """Visible facts and inaccessible declaration identities for one queried target.""" - - target: ScopeTargetQuery - visible: tuple[VisibilityRecord, ...] = field(default_factory=tuple) - inaccessible: tuple[DeclarationIdentity, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScopeReportFilters: - """Orthogonal declaration filters applied in their documented fixed order.""" - - sections: tuple[str, ...] = ("available", "used", "relationship_scope") - include_nearby: bool = False - defined_under: str | None = None - kinds: tuple[DeclarationKind, ...] = field(default_factory=tuple) - glob: str | None = None - used_only: bool = False - dependency_depth: int = 0 - sort: str = "identity" - cursor: str | None = None - page_size: int = 100 - nearby_depth: int = 1 - globals: str = "summary" - - -@dataclass(frozen=True) -class SourceLocation: - """Safe project-relative source location.""" - - path: str - line: int - column: int - - -@dataclass(frozen=True) -class VisibilityProvenance: - """Why a declaration is visible or inaccessible.""" - - reason: str - through: str | None = None - - -@dataclass(frozen=True) -class DeclarationReport: - """Value-free declaration projection used by every report section.""" - - identity: str - kind: str - name: str - owner: str | None - definition: SourceLocation - scope: str - owning_path: str | None - visibility: VisibilityProvenance | None = None - inaccessible_reason: str | None = None - metadata: tuple[tuple[str, object], ...] = field(default_factory=tuple) - consumers: tuple[str, ...] = field(default_factory=tuple) - dependencies: tuple[str, ...] = field(default_factory=tuple) - grants: tuple[str, ...] = field(default_factory=tuple) - required_scope: str | None = None - required_path: str | None = None - promotion_impact: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScopeChainEntry: - """One exact or inherited lexical scope in deterministic path order.""" - - kind: str - path: str - declaration_count: int - - -@dataclass(frozen=True) -class ScopeSection: - """Pagination and completeness metadata for one report section.""" - - name: str - total: int - returned: int - collapsed: bool = False - collapsed_count: int = 0 - truncated: bool = False - complete: bool = True - next_cursor: str | None = None - cursor: str | None = None - page_size: int = 100 - - -@dataclass(frozen=True) -class ScopeResourceReport: - """Resolved existing resource or prospective authored path.""" - - target: str - identity: str | None - path: str | None - prospective: bool = False - directory: bool = False - duplicate_count: int = 0 - - -@dataclass(frozen=True) -class DeclarationExplanation: - """Detailed explanation for one qualified declaration.""" - - declaration: DeclarationReport | None - complete: bool - - -@dataclass(frozen=True) -class MovePreview: - """Pure visibility delta for moving one existing resource.""" - - resource: str - destination: str - new_ownership_root: str | None - retained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - gained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - lost: tuple[DeclarationReport, ...] = field(default_factory=tuple) - invalidated_usages: tuple[str, ...] = field(default_factory=tuple) - private_retained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - relationship_retained: tuple[DeclarationReport, ...] = field(default_factory=tuple) - complete: bool = True - - -@dataclass(frozen=True) -class ScopeFolder: - """One declaration-definition folder with exact recursive counts.""" - - path: str - name: str - descendant_count: int - child_count: int - used_count: int - kind_counts: tuple[tuple[str, int], ...] - - -@dataclass(frozen=True) -class ScopeBrowseResult: - """Folder-first browse result.""" - - folder: str - folders: tuple[ScopeFolder, ...] - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - complete: bool = True - - -@dataclass(frozen=True) -class ScopeListResult: - """Recursive declaration list beneath one definition folder.""" - - folder: str - declarations: tuple[DeclarationReport, ...] - section: ScopeSection - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScopeReport: - """Canonical payload shared by all non-rendering scope consumers.""" - - resource: ScopeResourceReport - scope_chain: tuple[ScopeChainEntry, ...] = field(default_factory=tuple) - available: tuple[DeclarationReport, ...] = field(default_factory=tuple) - used: tuple[DeclarationReport, ...] = field(default_factory=tuple) - relationship_scope: tuple[DeclarationReport, ...] = field(default_factory=tuple) - nearby_unavailable: tuple[DeclarationReport, ...] = field(default_factory=tuple) - filters: ScopeReportFilters = field(default_factory=ScopeReportFilters) - sections: tuple[ScopeSection, ...] = field(default_factory=tuple) - explanation: DeclarationExplanation | None = None - move_preview: MovePreview | None = None - diagnostics: tuple[ScopeDiagnostic, ...] = field(default_factory=tuple) - complete: bool = True diff --git a/src/sqlbuild/compiler/scopes/types.py b/src/sqlbuild/compiler/scopes/types.py deleted file mode 100644 index c9dfe6bcd3..0000000000 --- a/src/sqlbuild/compiler/scopes/types.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Enums used by the compiler-owned scope domain.""" - -from __future__ import annotations - -from enum import StrEnum - - -class ResourceKind(StrEnum): - """Kinds of authored resources represented in a scope index.""" - - MODEL = "model" - TEST = "test" - SCENARIO = "scenario" - FUNCTION = "function" - SOURCE = "source" - SEED = "seed" - - -class DeclarationKind(StrEnum): - """Independent public declaration namespaces.""" - - MACRO = "macro" - ENUM = "enum" - CONSTANT = "constant" - AUDIT = "audit" - SINGULAR_AUDIT = "singular_audit" - SCHEMA = "schema" - SQL_HOOK = "sql_hook" - PYTHON_HOOK = "python_hook" - - -class ScopeKind(StrEnum): - """Compiler-enforced declaration visibility tiers.""" - - GLOBAL = "global" - INHERITED = "inherited" - LOCAL = "local" - PRIVATE = "private" - - -class OwnershipRootKind(StrEnum): - """How an ownership root participates in authored resource scope.""" - - RESOURCE = "resource" - GLOBAL = "global" - - -class UsageKind(StrEnum): - """Why one indexed identity consumes a declaration.""" - - RUNTIME = "runtime" - GENERATED = "generated" - DECLARATION_DEPENDENCY = "declaration_dependency" - - -class GrantKind(StrEnum): - """Compiler relationships that grant additional public vocabulary.""" - - EXPECTED_MODEL = "expected_model" - TESTED_MACRO = "tested_macro" - - -class VisibilityReason(StrEnum): - """Positive explanations emitted by the canonical resolver.""" - - GLOBAL = "global" - INHERITED_ANCESTOR = "inherited_ancestor" - LOCAL_OWNER = "local_owner" - PRIVATE_OWNER = "private_owner" - EXPECTED_MODEL = "expected_model" - TESTED_MACRO = "tested_macro" - - -class InaccessibleReason(StrEnum): - """Stable explanations for declarations outside effective scope.""" - - LOCAL_BOUNDARY = "local_boundary" - SIBLING_SCOPE = "sibling_scope" - DESCENDANT_SCOPE = "descendant_scope" - UNRELATED_SCOPE = "unrelated_scope" - PRIVATE_OWNER = "private_owner" - UNSUPPORTED_RESOURCE_KIND = "unsupported_resource_kind" - - -class DiagnosticSeverity(StrEnum): - """Severity of a stable scope diagnostic.""" - - ERROR = "error" - WARNING = "warning" - - -class ScopeDiagnosticCode(StrEnum): - """Stable diagnostic identities shared by future scope consumers.""" - - UNKNOWN_TARGET = "S001" - UNQUALIFIED_TARGET = "S002" - DUPLICATE_DECLARATION = "S003" - INVALID_DECLARATION_NAME = "S004" - RESERVED_DECLARATION_NAME = "S005" - INACCESSIBLE_DECLARATION = "S006" - LOCAL_NEEDED_BY_DESCENDANT = "S007" - OVER_BROAD_INHERITED = "S008" - REQUIRES_GLOBAL_PLACEMENT = "S009" - UNUSED_DECLARATION = "S010" - INVALID_PROSPECTIVE_PATH = "S011" - NESTED_DECLARATION_ROOT = "S012" - INCOMPLETE_USAGE = "S013" - INVALID_CURSOR = "S014" - MACRO_MODULE_IMPORT = "S015" - INVALID_MACRO_DEPENDENCY = "S016" - MACRO_DEPENDENCY_CYCLE = "S017" - DUPLICATE_RESOURCE = "S018" - RESOURCE_PARSE_ERROR = "S019" - DECLARATION_PARSE_ERROR = "S020" - MACRO_PARSE_ERROR = "S021" - RELATIONSHIP_PARSE_ERROR = "S022" - CONFIG_PARSE_ERROR = "S023" - OVER_BROAD_GLOBAL = "S024" - - -class CompletenessSection(StrEnum): - """Independently tracked scope-index analysis sections.""" - - DISCOVERY = "discovery" - STATIC_VISIBILITY = "static_visibility" - RUNTIME_USAGE = "runtime_usage" - RELATIONSHIPS = "relationships" - PLACEMENT = "placement" - PROMOTION_IMPACT = "promotion_impact" diff --git a/src/sqlbuild/compiler/source_freshness/__init__.py b/src/sqlbuild/compiler/source_freshness/__init__.py deleted file mode 100644 index f8e4d9e230..0000000000 --- a/src/sqlbuild/compiler/source_freshness/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Direct source freshness state package.""" diff --git a/src/sqlbuild/compiler/source_freshness/constants.py b/src/sqlbuild/compiler/source_freshness/constants.py deleted file mode 100644 index 0cd633a952..0000000000 --- a/src/sqlbuild/compiler/source_freshness/constants.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Stable constants for direct source freshness storage.""" - -from __future__ import annotations - -from sqlbuild.sql_values.types import StateSqlValueType - -SOURCE_FRESHNESS_TABLE_NAME: str = "_sqlbuild_source_freshness" - -COLUMN_SOURCE_NAME: str = "source_name" -COLUMN_TARGET_DATABASE: str = "target_database" -COLUMN_TARGET_SCHEMA: str = "target_schema" -COLUMN_TARGET_NAME: str = "target_name" -COLUMN_RUN_ID: str = "run_id" -COLUMN_STRATEGY: str = "strategy" -COLUMN_VALUE_KIND: str = "value_kind" -COLUMN_DATA_VERSION: str = "data_version" -COLUMN_DATA_VERSION_HASH: str = "data_version_hash" -COLUMN_OBSERVED_AT: str = "observed_at" - -SOURCE_FRESHNESS_COLUMNS: tuple[str, ...] = ( - COLUMN_SOURCE_NAME, - COLUMN_TARGET_DATABASE, - COLUMN_TARGET_SCHEMA, - COLUMN_TARGET_NAME, - COLUMN_RUN_ID, - COLUMN_STRATEGY, - COLUMN_VALUE_KIND, - COLUMN_DATA_VERSION, - COLUMN_DATA_VERSION_HASH, - COLUMN_OBSERVED_AT, -) - -SOURCE_FRESHNESS_COLUMN_TYPES: dict[str, StateSqlValueType] = { - COLUMN_SOURCE_NAME: StateSqlValueType.STRING, - COLUMN_TARGET_DATABASE: StateSqlValueType.STRING, - COLUMN_TARGET_SCHEMA: StateSqlValueType.STRING, - COLUMN_TARGET_NAME: StateSqlValueType.STRING, - COLUMN_RUN_ID: StateSqlValueType.STRING, - COLUMN_STRATEGY: StateSqlValueType.STRING, - COLUMN_VALUE_KIND: StateSqlValueType.STRING, - COLUMN_DATA_VERSION: StateSqlValueType.STRING, - COLUMN_DATA_VERSION_HASH: StateSqlValueType.STRING, - COLUMN_OBSERVED_AT: StateSqlValueType.TEXT_TIMESTAMP, -} diff --git a/src/sqlbuild/compiler/source_freshness/models.py b/src/sqlbuild/compiler/source_freshness/models.py deleted file mode 100644 index 8b20d3d929..0000000000 --- a/src/sqlbuild/compiler/source_freshness/models.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Direct source freshness state models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime - -from sqlbuild._native import ( - SourceFreshnessIdentity, - require_aware_utc_datetime, -) - - -@dataclass(frozen=True) -class SourceFreshnessRecord: - """One append-only direct observation whose observed time is always aware UTC.""" - - source_name: str - target_database: str | None - target_schema: str | None - target_name: str | None - run_id: str - strategy: str - value_kind: str - data_version: str | None - data_version_hash: str - observed_at: datetime - - def __post_init__(self) -> None: - object.__setattr__( - self, - "observed_at", - require_aware_utc_datetime(value=self.observed_at, field_name="observed_at"), - ) - - @property - def identity(self) -> SourceFreshnessIdentity: - return SourceFreshnessIdentity( - source_name=self.source_name, - target_database=self.target_database, - target_schema=self.target_schema, - target_name=self.target_name, - ) diff --git a/src/sqlbuild/compiler/sql_analysis/__init__.py b/src/sqlbuild/compiler/sql_analysis/__init__.py index 95940b4c9d..e69de29bb2 100644 --- a/src/sqlbuild/compiler/sql_analysis/__init__.py +++ b/src/sqlbuild/compiler/sql_analysis/__init__.py @@ -1 +0,0 @@ -"""Compiler-owned SQL analysis behavior.""" diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py b/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py index edf7bcd1ac..e69de29bb2 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal SQL analysis helpers.""" diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index c7e0b21087..d68a9dc000 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -2,12 +2,6 @@ from __future__ import annotations -SQL_ESCAPABLE_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -SQL_ESCAPE_CHARACTER: str = "\\" -SQL_TRIPLE_QUOTE_LENGTH: int = 3 -SQL_LINE_COMMENT_PREFIX: str = "--" -SUPPORTED_SQL_LINE_COMMENT_PREFIXES: frozenset[str] = frozenset({"--", "//", "#"}) -SQL_BLOCK_COMMENT_OPEN: str = "/*" POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX: int = 1 diff --git a/src/sqlbuild/compiler/sql_analysis/main/__init__.py b/src/sqlbuild/compiler/sql_analysis/main/__init__.py index d33bc998ad..e69de29bb2 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/__init__.py +++ b/src/sqlbuild/compiler/sql_analysis/main/__init__.py @@ -1 +0,0 @@ -"""Public SQL analysis entries.""" diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py deleted file mode 100644 index 47f779ab2b..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Stable SQLBuild-owned schema validation models.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import UnsupportedLexicalSyntaxError -from sqlbuild.compiler.sql_analysis.constants import ( - SQL_BLOCK_COMMENT_OPEN, - SQL_ESCAPABLE_QUOTE_CHARACTERS, - SQL_ESCAPE_CHARACTER, - SQL_LINE_COMMENT_PREFIX, - SQL_TRIPLE_QUOTE_LENGTH, - SUPPORTED_SQL_LINE_COMMENT_PREFIXES, -) - - -@dataclass(frozen=True) -class SqlLexicalSyntax: - """Dialect rules for where SQL quoted text and comments end.""" - - backslash_escape_quotes: frozenset[str] = frozenset() - escape_string_prefix: bool = False - raw_string_prefix: bool = False - triple_quoted_strings: bool = False - nested_block_comments: bool = False - line_comment_prefixes: frozenset[str] = frozenset({SQL_LINE_COMMENT_PREFIX}) - - def __post_init__(self) -> None: - unsupported: frozenset[str] = ( - self.line_comment_prefixes - SUPPORTED_SQL_LINE_COMMENT_PREFIXES - ) - if unsupported: - raise UnsupportedLexicalSyntaxError( - f"SQL line comment prefixes {sorted(unsupported)} are not supported; SQLBuild " - f"scans only {sorted(SUPPORTED_SQL_LINE_COMMENT_PREFIXES)}" - ) - - @property - def cache_key(self) -> str: - """Return a process-stable text key for these lexical rules.""" - - return "|".join( - ( - ",".join(sorted(self.backslash_escape_quotes)), - str(self.escape_string_prefix), - str(self.raw_string_prefix), - str(self.triple_quoted_strings), - str(self.nested_block_comments), - ",".join(sorted(self.line_comment_prefixes)), - ) - ) - - @property - def native_mapping(self) -> dict[str, object]: - """Return these lexical rules as the mapping native scanners read.""" - - return { - "backslash_escape_quotes": sorted(self.backslash_escape_quotes), - "escape_string_prefix": self.escape_string_prefix, - "raw_string_prefix": self.raw_string_prefix, - "triple_quoted_strings": self.triple_quoted_strings, - "nested_block_comments": self.nested_block_comments, - "line_comment_prefixes": sorted(self.line_comment_prefixes), - } - - def reads_differently_from_generic(self, sql: str) -> bool: - """Return whether these rules can read the SQL differently from generic SQL.""" - - if SQL_ESCAPE_CHARACTER in sql and ( - self.backslash_escape_quotes or self.escape_string_prefix - ): - return True - if self.triple_quoted_strings and any( - quote * SQL_TRIPLE_QUOTE_LENGTH in sql for quote in SQL_ESCAPABLE_QUOTE_CHARACTERS - ): - return True - if self.nested_block_comments and sql.count(SQL_BLOCK_COMMENT_OPEN) > 1: - return True - return any( - prefix in sql - for prefix in self.line_comment_prefixes - if prefix != SQL_LINE_COMMENT_PREFIX - ) diff --git a/src/sqlbuild/compiler/sql_test_glue/__init__.py b/src/sqlbuild/compiler/sql_test_glue/__init__.py deleted file mode 100644 index e265190973..0000000000 --- a/src/sqlbuild/compiler/sql_test_glue/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native SQL test planning glue.""" diff --git a/src/sqlbuild/compute_logs.py b/src/sqlbuild/compute_logs.py index f9581b03b2..bac1d9b9cf 100644 --- a/src/sqlbuild/compute_logs.py +++ b/src/sqlbuild/compute_logs.py @@ -2,11 +2,11 @@ from pathlib import Path -from sqlbuild._native import LocalFilesystemComputeLogStorage -from sqlbuild.runtime.compute_logs.constants import ( +from sqlbuild._native import ( COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, + LocalFilesystemComputeLogStorage, ) from sqlbuild.runtime.compute_logs.exceptions import ( CaptureAlreadyExistsError, diff --git a/src/sqlbuild/cost/constants.py b/src/sqlbuild/cost/constants.py deleted file mode 100644 index 43664c4c4a..0000000000 --- a/src/sqlbuild/cost/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Cost observability defaults.""" - -from decimal import Decimal - -DEFAULT_USD_PER_CREDIT: Decimal = Decimal("3.00") -SQLBUILD_QUERY_TAG_APP: str = "sqlbuild" diff --git a/src/sqlbuild/cursor_algebra/__init__.py b/src/sqlbuild/cursor_algebra/__init__.py index 0228f3f65d..e69de29bb2 100644 --- a/src/sqlbuild/cursor_algebra/__init__.py +++ b/src/sqlbuild/cursor_algebra/__init__.py @@ -1 +0,0 @@ -"""Naive timestamps are presumed UTC; aware timestamps are normalized to UTC at ingestion; dates have no timezone semantics.""" # noqa: E501 diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py deleted file mode 100644 index 4c93c60836..0000000000 --- a/src/sqlbuild/cursor_algebra/constants.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Canonical cursor grain metadata.""" - -from sqlbuild.compiler.planner.types import CursorGrain - -GRAIN_ORDER: dict[CursorGrain, int] = { - CursorGrain.SECOND: 0, - CursorGrain.MINUTE: 1, - CursorGrain.HOUR: 2, - CursorGrain.DAY: 3, - CursorGrain.MONTH: 4, - CursorGrain.YEAR: 5, -} - -DURATION_YEAR_UNIT: str = "y" -DURATION_MONTH_UNIT: str = "mo" -DURATION_DAY_UNIT: str = "d" -DURATION_HOUR_UNIT: str = "h" -DURATION_MINUTE_UNIT: str = "m" -DURATION_SECOND_UNIT: str = "s" diff --git a/src/sqlbuild/cursor_algebra/models.py b/src/sqlbuild/cursor_algebra/models.py deleted file mode 100644 index 02ec9e9332..0000000000 --- a/src/sqlbuild/cursor_algebra/models.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Immutable cursor algebra values.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import date, datetime -from typing import ClassVar - -from sqlbuild._native import ( - parse_cursor_duration, - shift_by_cursor_duration, - validate_cursor_interval, -) -from sqlbuild.compiler.planner.types import CursorGrain -from sqlbuild.cursor_algebra.constants import ( - DURATION_DAY_UNIT, - DURATION_HOUR_UNIT, - DURATION_MINUTE_UNIT, - DURATION_MONTH_UNIT, - DURATION_SECOND_UNIT, - DURATION_YEAR_UNIT, -) - - -@dataclass(frozen=True, kw_only=True) -class TimestampValue: - """Timestamp cursor value; aware datetimes are normalized to UTC at ingestion.""" - - value: datetime - source_text: str | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True, kw_only=True) -class DateValue: - """Date cursor value which must remain date-formatted when rendered.""" - - value: date - - -@dataclass(frozen=True, kw_only=True) -class IntegerValue: - """Integral cursor value.""" - - value: int - - -@dataclass(frozen=True, kw_only=True) -class AlignedInterval: - """Validated half-open cursor interval.""" - - start: TimestampValue | DateValue | IntegerValue - end: TimestampValue | DateValue | IntegerValue - grain: CursorGrain | None - - def __post_init__(self) -> None: - validate_cursor_interval(start=self.start, end=self.end, grain=self.grain) - - -@dataclass(frozen=True) -class Duration: - """A calendar-aware duration (years/months plus fixed time units).""" - - _MONTHS_PER_YEAR: ClassVar[int] = 12 - _SECONDS_PER_DAY: ClassVar[int] = 86_400 - _SECONDS_PER_HOUR: ClassVar[int] = 3_600 - _SECONDS_PER_MINUTE: ClassVar[int] = 60 - - years: int = 0 - months: int = 0 - days: int = 0 - hours: int = 0 - minutes: int = 0 - seconds: int = 0 - - @classmethod - def parse(cls, value: str) -> Duration | None: - """Parse a string like '1mo', '1y6mo', '30d', or '6h' into a duration.""" - - parts: tuple[int, int, int, int, int, int] | None = parse_cursor_duration(value) - if parts is None: - return None - years, months, days, hours, minutes, seconds = parts - return cls( - years=years, months=months, days=days, hours=hours, minutes=minutes, seconds=seconds - ) - - @property - def is_zero(self) -> bool: - """Return whether the duration is empty.""" - - return self.total_months == 0 and self.fixed_seconds == 0 - - @property - def units(self) -> frozenset[str]: - """Return the unit suffixes with non-zero values.""" - - values_by_unit: tuple[tuple[str, int], ...] = ( - (DURATION_YEAR_UNIT, self.years), - (DURATION_MONTH_UNIT, self.months), - (DURATION_DAY_UNIT, self.days), - (DURATION_HOUR_UNIT, self.hours), - (DURATION_MINUTE_UNIT, self.minutes), - (DURATION_SECOND_UNIT, self.seconds), - ) - return frozenset(unit for unit, amount in values_by_unit if amount != 0) - - def is_single_unit_in(self, allowed_units: frozenset[str]) -> bool: - """Return whether exactly one used unit belongs to the allowed vocabulary.""" - - return len(self.units) == 1 and self.units <= allowed_units - - @property - def has_calendar_component(self) -> bool: - """Return whether the duration includes variable-length years or months.""" - - return self.total_months != 0 - - @property - def fixed_seconds(self) -> int: - """Return the fixed-length portion (days and below) as whole seconds.""" - - return ( - self.days * self._SECONDS_PER_DAY - + self.hours * self._SECONDS_PER_HOUR - + self.minutes * self._SECONDS_PER_MINUTE - + self.seconds - ) - - @property - def total_months(self) -> int: - """Return the whole-month portion (years folded into months).""" - - return self.years * self._MONTHS_PER_YEAR + self.months - - def add_to(self, moment: datetime) -> datetime: - """Return the moment advanced by this duration.""" - - return shift_by_cursor_duration(moment=moment, duration=self, direction=1) - - def subtract_from(self, moment: datetime) -> datetime: - """Return the moment moved back by this duration.""" - - return shift_by_cursor_duration(moment=moment, duration=self, direction=-1) diff --git a/src/sqlbuild/cursor_algebra/types.py b/src/sqlbuild/cursor_algebra/types.py index 6d86a5f165..4fb5c4e2f9 100644 --- a/src/sqlbuild/cursor_algebra/types.py +++ b/src/sqlbuild/cursor_algebra/types.py @@ -2,7 +2,7 @@ from enum import StrEnum -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue +from sqlbuild._native import DateValue, IntegerValue, TimestampValue type CursorScalar = TimestampValue | DateValue | IntegerValue diff --git a/src/sqlbuild/diagnostics/__init__.py b/src/sqlbuild/diagnostics/__init__.py index 6f268ee63e..e69de29bb2 100644 --- a/src/sqlbuild/diagnostics/__init__.py +++ b/src/sqlbuild/diagnostics/__init__.py @@ -1 +0,0 @@ -"""Diagnostics logging support.""" diff --git a/src/sqlbuild/diagnostics/_helpers/logging.py b/src/sqlbuild/diagnostics/_helpers/logging.py index a6ae2cab03..b9f33d612c 100644 --- a/src/sqlbuild/diagnostics/_helpers/logging.py +++ b/src/sqlbuild/diagnostics/_helpers/logging.py @@ -2,13 +2,12 @@ from __future__ import annotations -import hashlib import logging from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar, Token -from sqlbuild.diagnostics.constants import SQL_TEXT_FIELD +from sqlbuild._native import debug_event_record_fields, sql_log_record from sqlbuild.runtime.observability.main.current_execution_identity import ( current_execution_identity, ) @@ -17,7 +16,6 @@ ) from sqlbuild.runtime.observability.models import ExecutionIdentity -_LOGGER_ROOT_NAME: str = "sqlbuild" _EMPTY_CONTEXT: dict[str, object] = {} _DIAGNOSTICS_CONTEXT: ContextVar[dict[str, object] | None] = ContextVar( "sqlbuild_diagnostics_context", default=None @@ -27,11 +25,11 @@ ) -def _current_identity_context() -> dict[str, object]: +def _current_identity_fields() -> dict[str, str | None] | None: identity: ExecutionIdentity | None = current_execution_identity() if identity is None: - return {} - return {f"sqlbuild_{key}": value for key, value in execution_identity_to_dict(identity).items()} + return None + return execution_identity_to_dict(identity) def log_sql( @@ -44,28 +42,15 @@ def log_sql( ) -> None: """Log SQL passed to an adapter or connection.""" - contextual_fields: dict[str, object] = { - key: value - for key, value in (_DIAGNOSTICS_CONTEXT.get() or _EMPTY_CONTEXT).items() - if key != SQL_TEXT_FIELD - } - sql_fields: dict[str, object] = { - "sqlbuild_sql_action": action, - "sqlbuild_sql_digest": f"sha256:{hashlib.sha256(sql.encode('utf-8')).hexdigest()}", - } - if intent is not None: - sql_fields["sqlbuild_sql_intent"] = intent - if query_id is not None: - sql_fields["sqlbuild_sql_query_id"] = query_id - record_fields: dict[str, object] = { - **contextual_fields, - **sql_fields, - **_current_identity_context(), - } - logger.debug( - f"{action} SQL", - extra=record_fields, + message, record_fields = sql_log_record( + context=_DIAGNOSTICS_CONTEXT.get(), + identity=_current_identity_fields(), + sql=sql, + action=action, + intent=intent, + query_id=query_id, ) + logger.debug(message, extra=record_fields) sql_sink: Callable[..., None] | None = _SQL_DIAGNOSTIC_SINK.get() if sql_sink is not None: sql_sink(logger=logger, sql=sql, fields=record_fields) @@ -89,11 +74,11 @@ def log_debug_event(*, logger: logging.Logger, message: str, **context: object) logger.debug( message, - extra={ - **(_DIAGNOSTICS_CONTEXT.get() or _EMPTY_CONTEXT), - **context, - **_current_identity_context(), - }, + extra=debug_event_record_fields( + context=_DIAGNOSTICS_CONTEXT.get(), + event_fields=context, + identity=_current_identity_fields(), + ), ) diff --git a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py b/src/sqlbuild/diagnostics/_helpers/logging_formatters.py deleted file mode 100644 index 5705251f17..0000000000 --- a/src/sqlbuild/diagnostics/_helpers/logging_formatters.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Diagnostics logging formatters.""" - -from __future__ import annotations - -import logging - -from sqlbuild._native import CliStyle - -_LOGGER_ROOT_NAME: str = "sqlbuild" -_INLINE_SQL_TRANSACTION_STATEMENTS: tuple[str, ...] = ("BEGIN", "COMMIT", "ROLLBACK") -_SQL_EVENT_FIELD: str = "sqlbuild_sql" - - -def _short_logger_name(name: str) -> str: - prefix: str = f"{_LOGGER_ROOT_NAME}." - if name.startswith(prefix): - return name.removeprefix(prefix) - return name - - -def _format_console_header(*, record: logging.LogRecord, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - level_tag: str = f"[{record.levelname.lower()}]" - context_message: str | None = _format_context_message(record=record, use_color=use_color) - if context_message is None: - scope: str = _short_logger_name(record.name) - colored_scope: str = style.object_name(scope) - context_message = f"{colored_scope} {_normalize_console_message(record.getMessage())}" - if not use_color: - return f"{level_tag} {context_message}" - return f"{style.log_label(level_tag)} {context_message}" - - -def _normalize_console_message(message: str) -> str: - return message.replace(" SQL", " sql") - - -def _render_sql_inline(sql: str) -> bool: - normalized: str = sql.strip().upper() - return normalized in _INLINE_SQL_TRANSACTION_STATEMENTS - - -def _format_context_message(*, record: logging.LogRecord, use_color: bool) -> str | None: - subject: object = getattr(record, "sqlbuild_subject", None) - if not isinstance(subject, str): - return None - parts: list[str] = [subject] - name: object = getattr(record, "sqlbuild_name", None) - if isinstance(name, str): - parts.append(_colorable_name(name=name, use_color=use_color)) - event: object = getattr(record, "sqlbuild_event", None) - if isinstance(event, str): - parts.append(event) - parts.extend(_format_context_tokens(record)) - message: str = _normalize_console_message(record.getMessage()) - if message: - parts.append(message) - return " ".join(parts) - - -def _format_context_tokens(record: logging.LogRecord) -> list[str]: - token_specs: tuple[tuple[str, str], ...] = ( - ("sqlbuild_phase", "phase"), - ("sqlbuild_kind", "kind"), - ("sqlbuild_action_name", "action"), - ("sqlbuild_audit_name", "audit"), - ("sqlbuild_column_name", "column"), - ("sqlbuild_window", "window"), - ("sqlbuild_status", "status"), - ) - rendered: list[str] = [] - field_name: str - label: str - for field_name, label in token_specs: - value: object = getattr(record, field_name, None) - if isinstance(value, str): - rendered.append(f"{label}={value}") - duration_ms: object = getattr(record, "sqlbuild_duration_ms", None) - if isinstance(duration_ms, int): - rendered.append(f"duration={duration_ms / 1000:.2f}s") - return rendered - - -def _colorable_name(*, name: str, use_color: bool) -> str: - return CliStyle(use_color=use_color).object_name(name) - - -def _get_record_sql(record: logging.LogRecord) -> str | None: - value: object = getattr(record, _SQL_EVENT_FIELD, None) - if isinstance(value, str): - return value - return None - - -def _append_exception(*, record: logging.LogRecord, rendered: str) -> str: - if record.exc_info is None: - return rendered - exception_text: str = logging.Formatter().formatException(record.exc_info) - return f"{rendered}\n{exception_text}" diff --git a/src/sqlbuild/diagnostics/_helpers/redaction.py b/src/sqlbuild/diagnostics/_helpers/redaction.py deleted file mode 100644 index 169d967780..0000000000 --- a/src/sqlbuild/diagnostics/_helpers/redaction.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Redacted structured fields of one logging record.""" - -from __future__ import annotations - -import logging -from typing import Final - -from sqlbuild._native import redact_diagnostic_value -from sqlbuild.diagnostics.constants import FORMATTER_RECORD_FIELDS, SQL_TEXT_FIELD - -_STANDARD_RECORD_FIELDS: Final[frozenset[str]] = frozenset(logging.makeLogRecord({}).__dict__) - - -def safe_extra_fields(record: logging.LogRecord) -> dict[str, object]: - """Collect redacted non-standard fields without evaluating arbitrary formatters.""" - - return { - key: redact_diagnostic_value(name=key, value=value) - for key, value in record.__dict__.items() - if key not in _STANDARD_RECORD_FIELDS - and key not in FORMATTER_RECORD_FIELDS - and key != SQL_TEXT_FIELD - } diff --git a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py index d7a4a12f56..f3fd0216b1 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py +++ b/src/sqlbuild/diagnostics/classes/diagnostic_record_redactor.py @@ -4,8 +4,11 @@ import logging -from sqlbuild._native import redact_diagnostic_text, redact_diagnostic_value -from sqlbuild.diagnostics._helpers.redaction import safe_extra_fields +from sqlbuild._native import ( + redact_diagnostic_text, + redact_diagnostic_value, + safe_diagnostic_extra_fields, +) class DiagnosticRecordRedactor: @@ -27,4 +30,4 @@ def value(*, name: str, value: object) -> object: def extras(record: logging.LogRecord) -> dict[str, object]: """Collect redacted non-standard record fields.""" - return safe_extra_fields(record) + return safe_diagnostic_extra_fields(record) diff --git a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py index 2c1a556162..741ebd95a3 100644 --- a/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py +++ b/src/sqlbuild/diagnostics/classes/diagnostics_console_formatter.py @@ -4,15 +4,7 @@ import logging -from sqlbuild._native import CliStyle -from sqlbuild.diagnostics._helpers.logging_formatters import ( - _append_exception, - _format_console_header, - _get_record_sql, - _render_sql_inline, -) - -_SQL_SEPARATOR: str = "-" * 80 +from sqlbuild._native import format_diagnostics_console_record class DiagnosticsConsoleFormatter(logging.Formatter): @@ -23,15 +15,7 @@ def __init__(self, *, use_color: bool = False) -> None: self._use_color: bool = use_color def format(self, record: logging.LogRecord) -> str: - header: str = _format_console_header(record=record, use_color=self._use_color) - sql: str | None = _get_record_sql(record) - if sql is None: - return _append_exception(record=record, rendered=header) - if _render_sql_inline(sql): - return _append_exception(record=record, rendered=f"{header} {sql.strip()}") - style: CliStyle = CliStyle(use_color=self._use_color) - separator: str = style.muted(_SQL_SEPARATOR) - return _append_exception( - record=record, - rendered=f"{header}\n{separator}\n{sql.rstrip()}\n{separator}", - ) + rendered: str = format_diagnostics_console_record(record, use_color=self._use_color) + if record.exc_info is None: + return rendered + return f"{rendered}\n{logging.Formatter().formatException(record.exc_info)}" diff --git a/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py b/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py index ba42f73323..a0e24e3c17 100644 --- a/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py +++ b/src/sqlbuild/diagnostics/classes/invocation_diagnostic_routing.py @@ -7,15 +7,12 @@ from contextvars import Token from types import TracebackType +from sqlbuild._native import SQL_PRIVATE_RECORD_FIELD, SQL_TEXT_FIELD from sqlbuild.diagnostics._helpers.logging import set_sql_diagnostic_sink from sqlbuild.diagnostics.classes.diagnostics_console_formatter import DiagnosticsConsoleFormatter from sqlbuild.diagnostics.classes.dynamic_stderr_handler import DynamicStderrHandler from sqlbuild.diagnostics.classes.log_record_route_filter import LogRecordRouteFilter -from sqlbuild.diagnostics.constants import ( - SQL_PRIVATE_RECORD_FIELD, - SQL_PRIVATE_RECORD_MARKER, - SQL_TEXT_FIELD, -) +from sqlbuild.diagnostics.constants import SQL_PRIVATE_RECORD_MARKER from sqlbuild.diagnostics.models import DiagnosticRoutingOptions from sqlbuild.runtime.compute_logs.classes.diagnostic_log_handler import ( ComputeDiagnosticLogHandler, diff --git a/src/sqlbuild/diagnostics/classes/log_record_route_filter.py b/src/sqlbuild/diagnostics/classes/log_record_route_filter.py index 08c08fd697..b25ce3030b 100644 --- a/src/sqlbuild/diagnostics/classes/log_record_route_filter.py +++ b/src/sqlbuild/diagnostics/classes/log_record_route_filter.py @@ -4,14 +4,7 @@ import logging -from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor -from sqlbuild.diagnostics.constants import ( - LOGGER_PREFIX, - LOGGER_ROOT_NAME, - REDACTED_SQL_LOG_MESSAGE, - SQL_DIGEST_FIELD, - SQL_TEXT_FIELD, -) +from sqlbuild._native import route_diagnostic_log_record class LogRecordRouteFilter(logging.Filter): @@ -32,40 +25,12 @@ def __init__( def filter(self, record: logging.LogRecord) -> logging.LogRecord | bool: """Return a safe copy when the record belongs at this destination.""" - internal: bool = record.name == LOGGER_ROOT_NAME or record.name.startswith(LOGGER_PREFIX) - if self._internal_only and not internal: + values: dict[str, object] | None = route_diagnostic_log_record( + record=record, + internal_only=self._internal_only, + allow_internal=self._allow_internal, + include_sql_text=self._include_sql_text, + ) + if values is None: return False - if internal and not self._allow_internal: - return False - if not internal and record.levelno < logging.INFO: - return False - if ( - self._include_sql_text - and SQL_DIGEST_FIELD in record.__dict__ - and SQL_TEXT_FIELD not in record.__dict__ - ): - return False - copied_values: dict[str, object] = dict(record.__dict__) - try: - copied_values["msg"] = DiagnosticRecordRedactor.text(record.getMessage()) - except Exception: - copied_values["msg"] = "" - copied_values["args"] = () - if SQL_DIGEST_FIELD in copied_values: - copied_values["msg"] = REDACTED_SQL_LOG_MESSAGE - if record.exc_info is not None: - try: - exception_text: str = logging.Formatter().formatException(record.exc_info) - copied_values["msg"] = ( - f"{copied_values['msg']}\n{DiagnosticRecordRedactor.text(exception_text)}" - ) - except Exception: - pass - copied_values["exc_info"] = None - copied_values["exc_text"] = None - for key, value in tuple(copied_values.items()): - if key.startswith("sqlbuild_") and key != SQL_TEXT_FIELD: - copied_values[key] = DiagnosticRecordRedactor.value(name=key, value=value) - if not self._include_sql_text: - copied_values.pop(SQL_TEXT_FIELD, None) - return logging.makeLogRecord(copied_values) + return logging.makeLogRecord(values) diff --git a/src/sqlbuild/diagnostics/constants.py b/src/sqlbuild/diagnostics/constants.py index a0c5ab610c..86d63524ae 100644 --- a/src/sqlbuild/diagnostics/constants.py +++ b/src/sqlbuild/diagnostics/constants.py @@ -1,10 +1,3 @@ """Diagnostic logging policy constants.""" -LOGGER_ROOT_NAME: str = "sqlbuild" -LOGGER_PREFIX: str = "sqlbuild." -SQL_DIGEST_FIELD: str = "sqlbuild_sql_digest" -SQL_TEXT_FIELD: str = "sqlbuild_sql" -SQL_PRIVATE_RECORD_FIELD: str = "sqlbuild_private_sql_record" SQL_PRIVATE_RECORD_MARKER: object = object() -REDACTED_SQL_LOG_MESSAGE: str = "SQL diagnostic omitted" -FORMATTER_RECORD_FIELDS: frozenset[str] = frozenset(("message", "asctime")) diff --git a/src/sqlbuild/errors/__init__.py b/src/sqlbuild/errors/__init__.py index b7979eec1d..e69de29bb2 100644 --- a/src/sqlbuild/errors/__init__.py +++ b/src/sqlbuild/errors/__init__.py @@ -1 +0,0 @@ -"""Cross-domain error contracts.""" diff --git a/src/sqlbuild/errors/contracts/__init__.py b/src/sqlbuild/errors/contracts/__init__.py index 406e900146..e69de29bb2 100644 --- a/src/sqlbuild/errors/contracts/__init__.py +++ b/src/sqlbuild/errors/contracts/__init__.py @@ -1 +0,0 @@ -"""Cross-domain expected-error contracts.""" diff --git a/src/sqlbuild/execution_history.py b/src/sqlbuild/execution_history.py index a3674fdda6..f95f4f08cb 100644 --- a/src/sqlbuild/execution_history.py +++ b/src/sqlbuild/execution_history.py @@ -2,12 +2,12 @@ from collections.abc import Iterable +from sqlbuild._native import DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT from sqlbuild._native import append_and_project as _append_and_project from sqlbuild._native import canonical_event_content as _canonical_event_content from sqlbuild._native import canonical_event_id as _canonical_event_id from sqlbuild._native import project_execution_runs as _project_execution_runs from sqlbuild._native import validate_history_page_limit as _validate_history_page_limit -from sqlbuild.runtime.execution_history.constants import DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT from sqlbuild.runtime.execution_history.exceptions import ( ExecutionHistoryStorageError, IntegrityConflictError, diff --git a/src/sqlbuild/executor/audit_results/_helpers/sql.py b/src/sqlbuild/executor/audit_results/_helpers/sql.py index 986206bad2..cd79175624 100644 --- a/src/sqlbuild/executor/audit_results/_helpers/sql.py +++ b/src/sqlbuild/executor/audit_results/_helpers/sql.py @@ -4,11 +4,8 @@ from collections.abc import Callable, Sequence -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal, render_state_table_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.audit_results.constants import ( AUDIT_RESULT_COLUMN_TYPES, diff --git a/src/sqlbuild/executor/audit_results/constants.py b/src/sqlbuild/executor/audit_results/constants.py index eec3a1ae12..551379eaaf 100644 --- a/src/sqlbuild/executor/audit_results/constants.py +++ b/src/sqlbuild/executor/audit_results/constants.py @@ -1,6 +1,6 @@ """Audit result warehouse storage constants.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType AUDIT_RESULTS_TABLE_NAME: str = "_sqlbuild_audit_results" AUDIT_RESULT_SCHEMA_VERSION: int = 1 diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 6b16e9c4a3..edfe1f4fd4 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -10,7 +10,10 @@ from typing import Any, cast from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, CostContext, + SourceEntry, assert_no_unresolved_sql_markers, evaluate_measurement, render_audit_sql, @@ -21,8 +24,6 @@ AuditOutcome, AuditRunScope, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, @@ -32,7 +33,6 @@ from sqlbuild.executor.auditing.models import AuditExecutionContext, AuditExecutionResult from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.runtime.observability.models import OperationAttributes -from sqlbuild.spec.contracts.models import SourceEntry def execute_audit_impl( # noqa: PLR0913 diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index dbe7f1d4ec..5e1ca99d0f 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -13,12 +13,16 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import build_audit_gate_identity +from sqlbuild._native import ( + AuditIdentity, + AuditPlanEntry, + CompiledRelationLocation, + MeasurementThresholdBound, + PlanOutput, + build_audit_gate_identity, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.models import AuditIdentity, MeasurementThresholdBound from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.audit_results.constants import AUDIT_RESULT_SCHEMA_VERSION from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.main._write import write_audit_result_records diff --git a/src/sqlbuild/executor/auditing/main/_execute.py b/src/sqlbuild/executor/auditing/main/_execute.py index 715addf1dd..716360750f 100644 --- a/src/sqlbuild/executor/auditing/main/_execute.py +++ b/src/sqlbuild/executor/auditing/main/_execute.py @@ -4,13 +4,11 @@ from typing import Any +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.spec.contracts.models import SourceEntry def execute_audit( diff --git a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py index 3339ec1915..b77e8a9093 100644 --- a/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py +++ b/src/sqlbuild/executor/auditing/main/_execute_reporting_errors.py @@ -4,13 +4,11 @@ from typing import Any +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl, failed_audit_result from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.spec.contracts.models import SourceEntry def execute_audit_reporting_errors( diff --git a/src/sqlbuild/executor/auditing/main/_project_results.py b/src/sqlbuild/executor/auditing/main/_project_results.py index 21eacefa87..93cfe5cbd3 100644 --- a/src/sqlbuild/executor/auditing/main/_project_results.py +++ b/src/sqlbuild/executor/auditing/main/_project_results.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.auditing._helpers.result_projection import project_audit_result_batch_impl from sqlbuild.executor.auditing.models import AuditExecutionResult, AuditResultProjection diff --git a/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py b/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py index af77daa140..9ac2c247fd 100644 --- a/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py +++ b/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py @@ -5,7 +5,7 @@ from collections.abc import Iterator from contextlib import contextmanager -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput from sqlbuild.executor.auditing._helpers.result_projection import ( audit_result_publication_scope_impl, ) diff --git a/src/sqlbuild/executor/auditing/models.py b/src/sqlbuild/executor/auditing/models.py index 01fffea0c8..27c26b1b5a 100644 --- a/src/sqlbuild/executor/auditing/models.py +++ b/src/sqlbuild/executor/auditing/models.py @@ -6,8 +6,8 @@ from dataclasses import dataclass from typing import Any +from sqlbuild._native import CompiledRelationLocation, MeasurementThresholds, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.auditing.models import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -15,9 +15,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.compile.types import AttachedAuditTargetKind -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/build/_helpers/blocking.py b/src/sqlbuild/executor/build/_helpers/blocking.py index 9006edcf40..7a2aceb1eb 100644 --- a/src/sqlbuild/executor/build/_helpers/blocking.py +++ b/src/sqlbuild/executor/build/_helpers/blocking.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild._native import transitive_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, transitive_closure def downstream_blocked_keys( diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index 3169b537d9..b49ea7f544 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -4,11 +4,14 @@ from typing import Any -from sqlbuild._native import audit_resource_id +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + SourceEntry, + audit_resource_id, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) @@ -19,7 +22,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_end_audits( diff --git a/src/sqlbuild/executor/build/_helpers/external_source_loads.py b/src/sqlbuild/executor/build/_helpers/external_source_loads.py index c622624d7a..25840ab945 100644 --- a/src/sqlbuild/executor/build/_helpers/external_source_loads.py +++ b/src/sqlbuild/executor/build/_helpers/external_source_loads.py @@ -4,11 +4,15 @@ from collections.abc import Callable +from sqlbuild._native import ( + CompiledObjectKey, + DiscoveredLoaderFunction, + LoaderConnectionMode, + PlanOutput, + SourceEntry, + SourceLoadPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import ( @@ -22,7 +26,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_external_source_loads_before_connections( diff --git a/src/sqlbuild/executor/build/_helpers/indexes.py b/src/sqlbuild/executor/build/_helpers/indexes.py index d1978b36a3..04df581d06 100644 --- a/src/sqlbuild/executor/build/_helpers/indexes.py +++ b/src/sqlbuild/executor/build/_helpers/indexes.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import AuditPlanEntry, PlanOutput from sqlbuild.compiler.auditing.types import AuditAttachmentKind -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildIndexes diff --git a/src/sqlbuild/executor/build/_helpers/retention.py b/src/sqlbuild/executor/build/_helpers/retention.py index b5a4d06249..cff1e935bd 100644 --- a/src/sqlbuild/executor/build/_helpers/retention.py +++ b/src/sqlbuild/executor/build/_helpers/retention.py @@ -4,6 +4,17 @@ from typing import Any +from sqlbuild._native import ( + IncrementalMode, + ModelPlanEntry, + PlanAction, + PlanOutput, + RetentionDirection, + RetentionPlanEntry, + RetentionPlanPhase, + TableType, + TableTypePlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( RelationInfo, @@ -15,18 +26,6 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - PlanAction, - RetentionDirection, - RetentionPlanPhase, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, @@ -35,7 +34,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.types import TableType def materialization_recreates_relation(entry: ModelPlanEntry) -> bool: diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 1e41b5847a..fcf165264f 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -5,21 +5,21 @@ from collections.abc import Callable, Mapping from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + CompiledObjectKey, + IncrementalMode, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + SnapshotsConfig, assert_no_unresolved_sql_markers, replaces_incremental_relation, resolve_cursor_intrinsics, ) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, @@ -47,7 +47,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SnapshotsConfig type _BuildWorkerResult = ( ModelExecutionResult diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index 17da4ee61c..2548cd8384 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,12 +4,10 @@ from typing import Any -from sqlbuild._native import audit_resource_id +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) diff --git a/src/sqlbuild/executor/build/_helpers/source_node.py b/src/sqlbuild/executor/build/_helpers/source_node.py index 028b2af56b..84ae9ffec6 100644 --- a/src/sqlbuild/executor/build/_helpers/source_node.py +++ b/src/sqlbuild/executor/build/_helpers/source_node.py @@ -7,12 +7,16 @@ from collections.abc import Callable from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import ( + CompiledObjectKey, + CostContext, + DiscoveredLoaderFunction, + PlanOutput, + SourceEntry, + SourceLoadPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import LoadExecutionResult, LoadRuntimeParams @@ -20,7 +24,6 @@ from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry def execute_build_source_node( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 6284493d5c..2d8fad6bcb 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -15,29 +15,30 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CostContext, sql_test_resource_id -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CostContext, + DiscoveredLoaderFunction, FunctionPlanEntry, + IncrementalMode, + IncrementalStrategy, ModelPlanEntry, + PlanAction, PlanOutput, SeedPlanEntry, + SnapshotsConfig, + SourceEntry, SqlTestPlanEntry, TableTypePlanEntry, + sql_test_resource_id, ) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - IncrementalStrategy, - PlanAction, -) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event @@ -124,7 +125,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SnapshotsConfig, SourceEntry _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/build/constants.py b/src/sqlbuild/executor/build/constants.py index 5a0902b57f..f7109ec129 100644 --- a/src/sqlbuild/executor/build/constants.py +++ b/src/sqlbuild/executor/build/constants.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner.types import PlanAction +from sqlbuild._native import PlanAction INCREMENTAL_ACTIONS: frozenset[PlanAction] = frozenset( { diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 27b9768f1b..d67d05dcec 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -5,13 +5,11 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import aggregate_build_result +from sqlbuild._native import PlanOutput, RetentionPlanPhase, aggregate_build_result from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.relation_metadata_cache_scope import ( relation_metadata_cache_scope, ) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.executor.auditing.main._project_results import ( AuditResultProjection, project_audit_result_batch, diff --git a/src/sqlbuild/executor/build/main/_external_source_loads.py b/src/sqlbuild/executor/build/main/_external_source_loads.py index eb81adbc5e..4c8ce832c3 100644 --- a/src/sqlbuild/executor/build/main/_external_source_loads.py +++ b/src/sqlbuild/executor/build/main/_external_source_loads.py @@ -2,10 +2,8 @@ from __future__ import annotations +from sqlbuild._native import CompiledObjectKey, DiscoveredLoaderFunction, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections as _run_external_source_loads_before_connections, ) diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 15bd6e6c8b..6ef91accb1 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -7,18 +7,20 @@ from datetime import datetime from pathlib import Path -from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + DiscoveredLoaderFunction, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry, + SnapshotsConfig, SourceLoadPlanEntry, SqlTestPlanEntry, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent, RelationInfo +from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.types import BeforeModelMaterializeCallback, BuildStatus @@ -33,7 +35,6 @@ from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, NodeStartCallback -from sqlbuild.spec.contracts.models import SnapshotsConfig @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/build/types.py b/src/sqlbuild/executor/build/types.py index d5701202f5..b8075be76a 100644 --- a/src/sqlbuild/executor/build/types.py +++ b/src/sqlbuild/executor/build/types.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: - from sqlbuild.compiler.planner.models import ModelPlanEntry + from sqlbuild._native import ModelPlanEntry from sqlbuild.executor.scheduling.types import ExecutionStatus as ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind as ExecutionResourceKind diff --git a/src/sqlbuild/executor/clone/_helpers/execution_items.py b/src/sqlbuild/executor/clone/_helpers/execution_items.py index f4ab512551..487b5e3694 100644 --- a/src/sqlbuild/executor/clone/_helpers/execution_items.py +++ b/src/sqlbuild/executor/clone/_helpers/execution_items.py @@ -5,17 +5,17 @@ import time from dataclasses import replace -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationLookup, RetentionRequest -from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CloneSourcePlanEntry, + CompiledObjectKey, FunctionPlanEntry, + MaterializationType, ModelPlanEntry, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import RelationLookup, RetentionRequest +from sqlbuild.adapter.contract.types import RetentionScope from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.clone._helpers.operations import clone_relation, recreate_view from sqlbuild.executor.clone._helpers.retention import apply_clone_retention diff --git a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py index 347dbba958..5c2abd241e 100644 --- a/src/sqlbuild/executor/clone/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/clone/_helpers/fingerprinting.py @@ -8,18 +8,21 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import read_latest_fingerprints, write_fingerprints -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.fingerprints.constants import ( +from sqlbuild._native import ( FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL, NODE_TYPE_SEED, + Fingerprint, + FingerprintSet, + MaterializationType, + ModelPlanEntry, + SeedPlanEntry, + read_latest_fingerprints, + write_fingerprints, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationLookup +from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.executor.clone.models import CloneExecutionResult from sqlbuild.executor.clone.types import ( CloneAction, diff --git a/src/sqlbuild/executor/clone/_helpers/inspection.py b/src/sqlbuild/executor/clone/_helpers/inspection.py index 51ba6aebcf..69aae99c1a 100644 --- a/src/sqlbuild/executor/clone/_helpers/inspection.py +++ b/src/sqlbuild/executor/clone/_helpers/inspection.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.planner.models import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.clone.models import CloneExecutionInput from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/executor/clone/_helpers/lifecycle.py b/src/sqlbuild/executor/clone/_helpers/lifecycle.py index 8cb75027d1..acbc6f5c87 100644 --- a/src/sqlbuild/executor/clone/_helpers/lifecycle.py +++ b/src/sqlbuild/executor/clone/_helpers/lifecycle.py @@ -4,15 +4,15 @@ from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CloneSourcePlanEntry, FunctionPlanEntry, ModelPlanEntry, + RetentionPlanPhase, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import RetentionPlanPhase +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.clone._helpers.retention import apply_clone_namespace_retention_phase from sqlbuild.executor.clone.models import ( CloneExecutionInput, diff --git a/src/sqlbuild/executor/clone/_helpers/operations.py b/src/sqlbuild/executor/clone/_helpers/operations.py index 0c88295635..c9fa054a7d 100644 --- a/src/sqlbuild/executor/clone/_helpers/operations.py +++ b/src/sqlbuild/executor/clone/_helpers/operations.py @@ -4,12 +4,12 @@ from typing import Any +from sqlbuild._native import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import CloneSourcePlanEntry, ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names from sqlbuild.executor.clone.main._recreate_view_operation import recreate_view_by_names from sqlbuild.executor.clone.models import CloneItemResult diff --git a/src/sqlbuild/executor/clone/_helpers/retention.py b/src/sqlbuild/executor/clone/_helpers/retention.py index 0740cf885c..6324e91208 100644 --- a/src/sqlbuild/executor/clone/_helpers/retention.py +++ b/src/sqlbuild/executor/clone/_helpers/retention.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import ResolvedTimeTravelRetention, RetentionPlanPhase from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -12,7 +13,6 @@ ) from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionScope from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, @@ -21,7 +21,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention def build_clone_retention_requests( diff --git a/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py b/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py index 778d01aabc..1f5dbbc6e2 100644 --- a/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py +++ b/src/sqlbuild/executor/clone/main/_build_destination_transient_models.py @@ -1,7 +1,7 @@ """Collect destination models whose resolved table type is transient.""" +from sqlbuild._native import TableType from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.spec.contracts.types import TableType def build_destination_transient_models(*, project: CompiledProjectView) -> frozenset[str]: diff --git a/src/sqlbuild/executor/clone/main/_fingerprinting.py b/src/sqlbuild/executor/clone/main/_fingerprinting.py index 8de777f8f9..1c7acb0be2 100644 --- a/src/sqlbuild/executor/clone/main/_fingerprinting.py +++ b/src/sqlbuild/executor/clone/main/_fingerprinting.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ModelPlanEntry, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry from sqlbuild.executor.clone._helpers.fingerprinting import copy_clone_fingerprints as _copy from sqlbuild.executor.clone.models import CloneExecutionResult from sqlbuild.executor.clone.types import CloneFingerprintProgressCallback diff --git a/src/sqlbuild/executor/clone/main/execute.py b/src/sqlbuild/executor/clone/main/execute.py index f796b50209..c660fad021 100644 --- a/src/sqlbuild/executor/clone/main/execute.py +++ b/src/sqlbuild/executor/clone/main/execute.py @@ -2,14 +2,14 @@ from __future__ import annotations -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CloneSourcePlanEntry, + CompiledObjectKey, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry, ) +from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.executor.clone._helpers.canonical_progress import clone_item_enrichment from sqlbuild.executor.clone._helpers.execution_items import ( execute_clone_destination_item, diff --git a/src/sqlbuild/executor/clone/models.py b/src/sqlbuild/executor/clone/models.py index 2dc71e87b7..1ab31e3159 100644 --- a/src/sqlbuild/executor/clone/models.py +++ b/src/sqlbuild/executor/clone/models.py @@ -5,15 +5,16 @@ from dataclasses import dataclass, field from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import LifeCycleEvent, RetentionRequest -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CloneSourcePlanEntry, + CompiledObjectKey, + CompiledRelationLocation, FunctionPlanEntry, ModelPlanEntry, SeedPlanEntry, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import LifeCycleEvent, RetentionRequest from sqlbuild.executor.clone.types import CloneAction, CloneItemCallback, CloneStatus diff --git a/src/sqlbuild/executor/custom/models.py b/src/sqlbuild/executor/custom/models.py index 8b6111a2ed..993b5caf43 100644 --- a/src/sqlbuild/executor/custom/models.py +++ b/src/sqlbuild/executor/custom/models.py @@ -7,10 +7,10 @@ from dataclasses import dataclass, field from typing import Any +from sqlbuild._native import SchemaFinding from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.planner.models import SchemaFinding from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.main._execute_sql import execute_sql_with_recording from sqlbuild.executor.custom.main._qualify_relation import qualify_custom_relation diff --git a/src/sqlbuild/executor/diff/_helpers/bounds.py b/src/sqlbuild/executor/diff/_helpers/bounds.py index 50fd17346b..426af6a8af 100644 --- a/src/sqlbuild/executor/diff/_helpers/bounds.py +++ b/src/sqlbuild/executor/diff/_helpers/bounds.py @@ -5,10 +5,9 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import get_config_str +from sqlbuild._native import Duration, get_config_str from sqlbuild.adapter.contract.models import CursorValue from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.compiler.planner.models import Duration from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py index 1d0196fc8e..c22d58f774 100644 --- a/src/sqlbuild/executor/diff/_helpers/query_artifacts.py +++ b/src/sqlbuild/executor/diff/_helpers/query_artifacts.py @@ -4,20 +4,24 @@ import hashlib import json -import re from datetime import UTC, datetime from typing import Any -from sqlbuild._native import compute_query_hash, read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + Fingerprint, + FingerprintSet, + build_query_diff_artifact_name, + compute_query_hash, + parse_query_diff_artifact_name, + read_latest_fingerprints, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.fingerprints.constants import ( - FINGERPRINT_TABLE_NAME, - NODE_TYPE_QUERY_DIFF_ARTIFACT, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet +from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff.models import ( QueryDiffArtifact, @@ -25,10 +29,6 @@ QueryDiffArtifactInspection, ) -_ARTIFACT_PATTERN: re.Pattern[str] = re.compile( - r"^__sqlbuild_query_diff_(?P\d{8}T\d{6}Z_[0-9a-f]{12})_(?Pleft|right)$", - flags=re.IGNORECASE, -) _ARTIFACT_VERSION: int = 1 @@ -42,7 +42,7 @@ def build_query_diff_artifact( ) -> QueryDiffArtifact: """Build one collision-resistant artifact identity in the active target schema.""" - name: str = f"__sqlbuild_query_diff_{run_id}_{side}" + name: str = build_query_diff_artifact_name(run_id=run_id, side=side) relation: str | None = adapter.render_qualified_name( database=database, schema=schema, @@ -61,12 +61,6 @@ def build_query_diff_artifact( ) -def is_query_diff_artifact_name(name: str) -> bool: - """Return whether a relation uses the strict reserved query-diff name format.""" - - return _ARTIFACT_PATTERN.fullmatch(name) is not None - - def materialize_query_diff_artifact( *, adapter: BaseAdapter, @@ -196,7 +190,9 @@ def inspect_query_diff_artifacts( schemas=(schema,), ) candidates: tuple[RelationInfo, ...] = tuple( - relation for relation in relations if _ARTIFACT_PATTERN.fullmatch(relation.name) + relation + for relation in relations + if parse_query_diff_artifact_name(relation.name) is not None ) if not candidates: return QueryDiffArtifactInspection() @@ -227,13 +223,13 @@ def inspect_query_diff_artifacts( untracked: list[RelationInfo] = [] for relation in candidates: fingerprint: Fingerprint | None = owned.get(relation.name.lower()) - match: re.Match[str] | None = _ARTIFACT_PATTERN.fullmatch(relation.name) - if match is None: + parts: tuple[str, str] | None = parse_query_diff_artifact_name(relation.name) + if parts is None: continue artifact: QueryDiffArtifact = build_query_diff_artifact( adapter=adapter, - run_id=match.group("run_id"), - side=match.group("side").lower(), + run_id=parts[0], + side=parts[1], database=database, schema=schema, ) diff --git a/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py b/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py index 73f32405da..0ef13a9526 100644 --- a/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py +++ b/src/sqlbuild/executor/diff/classes/query_artifact_lifecycle.py @@ -5,13 +5,13 @@ from datetime import datetime from typing import Any +from sqlbuild._native import parse_query_diff_artifact_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.diff._helpers.query_artifacts import ( build_query_diff_artifact, cleanup_expired_query_diff_artifacts, cleanup_query_diff_artifact, inspect_query_diff_artifacts, - is_query_diff_artifact_name, materialize_query_diff_artifact, ) from sqlbuild.executor.diff.models import ( @@ -102,4 +102,4 @@ def inspect( @staticmethod def is_artifact_name(name: str) -> bool: - return is_query_diff_artifact_name(name) + return parse_query_diff_artifact_name(name) is not None diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index f00bba06d6..ed4d2c305f 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -7,6 +7,8 @@ from typing import Any from sqlbuild._native import ( + Fingerprint, + FunctionPlanEntry, compute_function_definition_hash, error_code, error_help, @@ -18,8 +20,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionDefinition from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import FunctionPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.constants import ( diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 965584d612..1eb83014a3 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -5,9 +5,8 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import OldNameViewDropReason, OldNameViewHistory from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.compiler.migrations.models import OldNameViewHistory -from sqlbuild.compiler.migrations.types import OldNameViewDropReason @dataclass(frozen=True, eq=False) diff --git a/src/sqlbuild/executor/janitor_events/_helpers/sql.py b/src/sqlbuild/executor/janitor_events/_helpers/sql.py index 5d85cae77f..1de4e10a49 100644 --- a/src/sqlbuild/executor/janitor_events/_helpers/sql.py +++ b/src/sqlbuild/executor/janitor_events/_helpers/sql.py @@ -4,11 +4,8 @@ from collections.abc import Callable -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal, render_state_table_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.janitor_events.constants import ( COLUMN_ARCHIVE_NAME, diff --git a/src/sqlbuild/executor/janitor_events/constants.py b/src/sqlbuild/executor/janitor_events/constants.py index 721f93d887..9e30530b97 100644 --- a/src/sqlbuild/executor/janitor_events/constants.py +++ b/src/sqlbuild/executor/janitor_events/constants.py @@ -1,6 +1,6 @@ """Janitor audit event warehouse storage constants.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType JANITOR_EVENTS_TABLE_NAME: str = "_sqlbuild_janitor_events" JANITOR_EVENT_SCHEMA_VERSION: int = 1 diff --git a/src/sqlbuild/executor/load/_helpers/cursors.py b/src/sqlbuild/executor/load/_helpers/cursors.py index 4f91b530d7..dfe099793b 100644 --- a/src/sqlbuild/executor/load/_helpers/cursors.py +++ b/src/sqlbuild/executor/load/_helpers/cursors.py @@ -6,12 +6,12 @@ from decimal import Decimal from typing import Any +from sqlbuild._native import DateValue, IntegerValue, TimestampValue from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError diff --git a/src/sqlbuild/executor/load/_helpers/dag_runtime.py b/src/sqlbuild/executor/load/_helpers/dag_runtime.py index d4e93068f9..0695bf42ec 100644 --- a/src/sqlbuild/executor/load/_helpers/dag_runtime.py +++ b/src/sqlbuild/executor/load/_helpers/dag_runtime.py @@ -7,6 +7,7 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.types import SkipMode @@ -39,7 +40,6 @@ current_execution_identity, ) from sqlbuild.runtime.observability.models import ExecutionIdentity -from sqlbuild.spec.contracts.models import SourceEntry def build_load_dag_state( diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 3cca33cf04..955a267d89 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -4,13 +4,12 @@ from collections.abc import Callable -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry def build_load_execution_indexes( diff --git a/src/sqlbuild/executor/load/_helpers/external.py b/src/sqlbuild/executor/load/_helpers/external.py index 2436ffb75d..b0f4773988 100644 --- a/src/sqlbuild/executor/load/_helpers/external.py +++ b/src/sqlbuild/executor/load/_helpers/external.py @@ -6,9 +6,9 @@ import time from collections.abc import Callable +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import ( LoaderContext, @@ -26,7 +26,6 @@ ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry def execute_external_source_load( diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index 21db821933..c07f0fd419 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -7,10 +7,14 @@ from collections.abc import Callable, Mapping from typing import Any +from sqlbuild._native import ( + DiscoveredLoaderFunction, + HardCodedRelationOwnerKind, + SourceEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.execution import ( is_untargeted_self_managed_intermediate, @@ -35,8 +39,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy _SUPPORTED_WRITE_STRATEGIES: frozenset[SourceWriteStrategy] = frozenset( { diff --git a/src/sqlbuild/executor/load/_helpers/relation_refs.py b/src/sqlbuild/executor/load/_helpers/relation_refs.py index 2b2a97a894..64cf6af316 100644 --- a/src/sqlbuild/executor/load/_helpers/relation_refs.py +++ b/src/sqlbuild/executor/load/_helpers/relation_refs.py @@ -5,13 +5,13 @@ from collections.abc import Callable, Mapping from typing import Any +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) from sqlbuild.executor.load.models import LoaderRelationRef -from sqlbuild.spec.contracts.models import SourceEntry def build_loader_relation_refs( diff --git a/src/sqlbuild/executor/load/_helpers/rows.py b/src/sqlbuild/executor/load/_helpers/rows.py index 9df83270e5..83f1bcab53 100644 --- a/src/sqlbuild/executor/load/_helpers/rows.py +++ b/src/sqlbuild/executor/load/_helpers/rows.py @@ -6,12 +6,12 @@ from datetime import date, datetime from decimal import Decimal +from sqlbuild._native import SourceColumnEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import LoaderLogicalType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoaderRowsSchema -from sqlbuild.spec.contracts.models import SourceColumnEntry def iter_loader_row_batches( diff --git a/src/sqlbuild/executor/load/_helpers/scheduling.py b/src/sqlbuild/executor/load/_helpers/scheduling.py index fde0ced4ed..bedf7263e1 100644 --- a/src/sqlbuild/executor/load/_helpers/scheduling.py +++ b/src/sqlbuild/executor/load/_helpers/scheduling.py @@ -10,9 +10,8 @@ from functools import partial from typing import Any +from sqlbuild._native import DiscoveredLoaderFunction, LoaderConnectionMode, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.dag_runtime import ( @@ -38,7 +37,6 @@ from sqlbuild.executor.load.types import LoadProgressCallback from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -from sqlbuild.spec.contracts.models import SourceEntry def validate_external_loaders_are_preconnect_runnable( diff --git a/src/sqlbuild/executor/load/_helpers/schema.py b/src/sqlbuild/executor/load/_helpers/schema.py index aa21d5cd7a..9b7c93205b 100644 --- a/src/sqlbuild/executor/load/_helpers/schema.py +++ b/src/sqlbuild/executor/load/_helpers/schema.py @@ -4,13 +4,12 @@ from typing import Any +from sqlbuild._native import ContractPolicy, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.spec.contracts.models import SourceEntry def validate_and_evolve_existing_target( diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 237ae5b80b..351b1a2db2 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -4,14 +4,13 @@ from typing import Any -from sqlbuild._native import fit_auxiliary_relation_name +from sqlbuild._native import ContractPolicy, SourceEntry, fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import LoaderLogicalType from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.executor.load._helpers.rows import ( build_rows_sql, iter_loader_row_batches, @@ -19,7 +18,6 @@ ) from sqlbuild.executor.load.models import LoaderRowsSchema from sqlbuild.executor.run.constants import STAGING_RELATION_SUFFIX -from sqlbuild.spec.contracts.models import SourceEntry def write_loader_rows_to_staging( diff --git a/src/sqlbuild/executor/load/main/_build_execution_indexes.py b/src/sqlbuild/executor/load/main/_build_execution_indexes.py index 174c422334..4bb37bf351 100644 --- a/src/sqlbuild/executor/load/main/_build_execution_indexes.py +++ b/src/sqlbuild/executor/load/main/_build_execution_indexes.py @@ -2,12 +2,11 @@ from __future__ import annotations -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.executor.load._helpers.execution import ( build_load_execution_indexes as _build_load_execution_indexes, ) from sqlbuild.executor.load.models import LoadExecutionIndexes -from sqlbuild.spec.contracts.models import SourceEntry def build_load_execution_indexes( diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index 4db2a54708..f4ccaa0360 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -7,13 +7,17 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import ( + CostContext, + DiscoveredLoaderFunction, + LoaderConnectionMode, + SourceEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.cursors import ( @@ -46,8 +50,6 @@ from sqlbuild.provider.main.runtime import invoke_with_providers from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy def execute_source_load( diff --git a/src/sqlbuild/executor/load/main/_resource_kind.py b/src/sqlbuild/executor/load/main/_resource_kind.py index f02a8d9364..d7ce0fcb9e 100644 --- a/src/sqlbuild/executor/load/main/_resource_kind.py +++ b/src/sqlbuild/executor/load/main/_resource_kind.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import SourceEntry from sqlbuild.executor.load._helpers.execution import load_resource_kind as _load_resource_kind from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry def load_resource_kind(source: SourceEntry) -> ExecutionResourceKind: diff --git a/src/sqlbuild/executor/load/main/_skipped_result.py b/src/sqlbuild/executor/load/main/_skipped_result.py index f7a3ff7221..e85eb356c9 100644 --- a/src/sqlbuild/executor/load/main/_skipped_result.py +++ b/src/sqlbuild/executor/load/main/_skipped_result.py @@ -2,10 +2,10 @@ from __future__ import annotations +from sqlbuild._native import SourceEntry from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load._helpers.execution import skipped_load_result as _skipped_load_result from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.spec.contracts.models import SourceEntry def skipped_load_result( diff --git a/src/sqlbuild/executor/load/main/run.py b/src/sqlbuild/executor/load/main/run.py index 45333c282e..676e1ed458 100644 --- a/src/sqlbuild/executor/load/main/run.py +++ b/src/sqlbuild/executor/load/main/run.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.executor.load._helpers.execution import build_load_execution_indexes from sqlbuild.executor.load._helpers.scheduling import ( open_load_connections, @@ -20,7 +20,6 @@ LoadExecutionResult, LoadRuntimeParams, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_load_pipeline( diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 7bf948b434..39097300b7 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -9,6 +9,7 @@ from pathlib import Path from typing import Any +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent @@ -16,7 +17,6 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR @@ -29,7 +29,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback, ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/load/types.py b/src/sqlbuild/executor/load/types.py index 7fe3787ea8..231e9958ad 100644 --- a/src/sqlbuild/executor/load/types.py +++ b/src/sqlbuild/executor/load/types.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: - from sqlbuild.spec.contracts.models import SourceEntry + from sqlbuild._native import SourceEntry class LoadProgressCallback(Protocol): diff --git a/src/sqlbuild/executor/migrations/_helpers/columns.py b/src/sqlbuild/executor/migrations/_helpers/columns.py index 8c62013c05..612f484223 100644 --- a/src/sqlbuild/executor/migrations/_helpers/columns.py +++ b/src/sqlbuild/executor/migrations/_helpers/columns.py @@ -5,14 +5,18 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_column_migration_event_id, write_migration_state_event +from sqlbuild._native import ( + MIGRATION_WRITE_ATTEMPTS, + ColumnMigrationDecision, + ColumnMigrationEvent, + ColumnMigrationPlanEntry, + MigrationRelation, + deterministic_column_migration_event_id, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision -from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/naming.py b/src/sqlbuild/executor/migrations/_helpers/naming.py index 03d29b933c..67b53bc9df 100644 --- a/src/sqlbuild/executor/migrations/_helpers/naming.py +++ b/src/sqlbuild/executor/migrations/_helpers/naming.py @@ -6,13 +6,12 @@ from datetime import datetime, timedelta from typing import Any -from sqlbuild._native import build_janitor_archive_name +from sqlbuild._native import ModelMigrationPlanEntry, build_janitor_archive_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations.constants import ( MIGRATION_NAME_ATTEMPTS, diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py index b05e84d7d1..4f71d0f5b3 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_facts.py @@ -6,16 +6,17 @@ from typing import Any from sqlbuild._native import ( + MIGRATION_WRITE_ATTEMPTS, + MigrationRelation, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewPlanEntry, deterministic_old_name_view_event_id, read_migration_state_events, stored_old_name_view_columns, write_migration_state_event, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewEventType -from sqlbuild.compiler.planner.models import OldNameViewPlanEntry from sqlbuild.executor.migrations._helpers.relation_for_location import ( migration_relation_for_location, ) diff --git a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py index c83c502598..5d11472a84 100644 --- a/src/sqlbuild/executor/migrations/_helpers/old_name_views.py +++ b/src/sqlbuild/executor/migrations/_helpers/old_name_views.py @@ -9,6 +9,16 @@ from typing import Any from sqlbuild._native import ( + COLUMN_MIGRATION_TABLE_NAME, + ColumnMigrationEvent, + CompiledRelationLocation, + Duration, + MigrationRelation, + OldNameView, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewPlanEntry, + PlanOutput, build_janitor_archive_name, deterministic_migration_event_id, read_recorded_column_renames, @@ -22,16 +32,6 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.constants import COLUMN_MIGRATION_TABLE_NAME -from sqlbuild.compiler.migrations.models import ( - ColumnMigrationEvent, - MigrationRelation, - OldNameViewEvent, -) -from sqlbuild.compiler.migrations.types import OldNameViewEventType -from sqlbuild.compiler.planner.models import OldNameView, OldNameViewPlanEntry, PlanOutput -from sqlbuild.cursor_algebra.models import Duration from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, diff --git a/src/sqlbuild/executor/migrations/_helpers/promotion.py b/src/sqlbuild/executor/migrations/_helpers/promotion.py index 9e3dac3889..5c30332937 100644 --- a/src/sqlbuild/executor/migrations/_helpers/promotion.py +++ b/src/sqlbuild/executor/migrations/_helpers/promotion.py @@ -5,13 +5,20 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import deterministic_migration_event_id, write_migration_state_event +from sqlbuild._native import ( + MIGRATION_WRITE_ATTEMPTS, + MigrationDecision, + MigrationEvent, + MigrationRelation, + ModelMigrationPlanEntry, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewPlanEntry, + deterministic_migration_event_id, + write_migration_state_event, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.migrations.constants import MIGRATION_WRITE_ATTEMPTS -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation, OldNameViewEvent -from sqlbuild.compiler.migrations.types import MigrationDecision, OldNameViewEventType -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry, OldNameViewPlanEntry from sqlbuild.executor.migrations._helpers.old_name_facts import ( old_name_fact, old_name_state_table_sql, diff --git a/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py index b298a89606..7b0ffd93c5 100644 --- a/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py +++ b/src/sqlbuild/executor/migrations/_helpers/relation_for_location.py @@ -2,8 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.models import MigrationRelation +from sqlbuild._native import CompiledRelationLocation, MigrationRelation def migration_relation_for_location(location: CompiledRelationLocation) -> MigrationRelation: diff --git a/src/sqlbuild/executor/migrations/_helpers/staging.py b/src/sqlbuild/executor/migrations/_helpers/staging.py index b954e24ad0..5ed55d54af 100644 --- a/src/sqlbuild/executor/migrations/_helpers/staging.py +++ b/src/sqlbuild/executor/migrations/_helpers/staging.py @@ -5,10 +5,10 @@ import logging from typing import Any +from sqlbuild._native import ModelMigrationPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations.models import MigrationArtifactNames diff --git a/src/sqlbuild/executor/migrations/main/_apply.py b/src/sqlbuild/executor/migrations/main/_apply.py index 7c4fe28546..c58a46598d 100644 --- a/src/sqlbuild/executor/migrations/main/_apply.py +++ b/src/sqlbuild/executor/migrations/main/_apply.py @@ -7,15 +7,15 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.migrations.models import MigrationEvent -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( + MigrationEvent, ModelMigrationPlanEntry, OldNameViewPlanEntry, PlanOutput, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.naming import resolve_artifact_names from sqlbuild.executor.migrations._helpers.promotion import ( diff --git a/src/sqlbuild/executor/migrations/main/_apply_columns.py b/src/sqlbuild/executor/migrations/main/_apply_columns.py index 4229d79fc9..a9d6254b88 100644 --- a/src/sqlbuild/executor/migrations/main/_apply_columns.py +++ b/src/sqlbuild/executor/migrations/main/_apply_columns.py @@ -6,8 +6,8 @@ from collections.abc import Callable from typing import Any +from sqlbuild._native import ColumnMigrationPlanEntry, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry, PlanOutput from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames diff --git a/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py b/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py index 820a3897f6..55379a6de6 100644 --- a/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py +++ b/src/sqlbuild/executor/migrations/main/_bound_old_name_view_guard.py @@ -6,9 +6,8 @@ from functools import partial from typing import Any +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.migrations._helpers.old_name_views import views_reading from sqlbuild.executor.migrations.classes.bound_old_name_views import BoundOldNameViews from sqlbuild.executor.migrations.models import OldNameViewSource diff --git a/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py b/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py index c16178ce37..191dae4901 100644 --- a/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py +++ b/src/sqlbuild/executor/migrations/main/_compatibility_view_sql.py @@ -4,9 +4,8 @@ from typing import Any +from sqlbuild._native import CompiledRelationLocation, MigrationRelation, OldNameViewHistory from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.models import MigrationRelation, OldNameViewHistory from sqlbuild.executor.migrations._helpers.old_name_views import render_old_name_view_select from sqlbuild.executor.migrations.models import OldNameViewSource diff --git a/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py b/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py index 3457c71a0b..00eaf0ef35 100644 --- a/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py +++ b/src/sqlbuild/executor/migrations/main/_complete_old_name_views.py @@ -4,9 +4,8 @@ from typing import Any +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.migrations._helpers.old_name_views import ( rebind_old_name_views_atomically, run_planned_old_name_steps, diff --git a/src/sqlbuild/executor/migrations/models.py b/src/sqlbuild/executor/migrations/models.py index 23e6b87f43..ac6a9e44f1 100644 --- a/src/sqlbuild/executor/migrations/models.py +++ b/src/sqlbuild/executor/migrations/models.py @@ -4,8 +4,8 @@ from dataclasses import dataclass +from sqlbuild._native import CompiledRelationLocation from sqlbuild.adapter.contract.models import RelationGrant -from sqlbuild.compiler.compile.models import CompiledRelationLocation @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/node_results/_helpers/sql.py b/src/sqlbuild/executor/node_results/_helpers/sql.py index d8669e4ee9..003996c925 100644 --- a/src/sqlbuild/executor/node_results/_helpers/sql.py +++ b/src/sqlbuild/executor/node_results/_helpers/sql.py @@ -4,11 +4,8 @@ from collections.abc import Callable -from sqlbuild._native import render_state_sql_literal +from sqlbuild._native import render_state_sql_literal, render_state_table_create_sql from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapter.state_sql.main.render_state_table_create_sql import ( - render_state_table_create_sql, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.node_results.constants import ( COLUMN_ERROR_MESSAGE, diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index 9408696776..c57129c44c 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -1,6 +1,6 @@ """Runtime node result storage constants.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType NODE_RESULTS_TABLE_NAME: str = "_sqlbuild_node_results" NODE_RESULT_MATERIALIZED_TRUE_VALUE: str = "true" diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index 299d04eb72..e2b7e4e095 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -6,6 +6,9 @@ from typing import Any from sqlbuild._native import ( + OldNameViewDropReason, + OldNameViewEvent, + OldNameViewEventType, deterministic_old_name_view_event_id, write_migration_state_event, ) @@ -17,11 +20,6 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.migrations.models import OldNameViewEvent -from sqlbuild.compiler.migrations.types import ( - OldNameViewDropReason, - OldNameViewEventType, -) from sqlbuild.executor.janitor.models import ( JanitorOldNameView, JanitorOldNameViewPlanning, diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index 9cc0dbe756..91696eb327 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -15,10 +15,9 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import CostContext, audit_resource_id +from sqlbuild._native import AuditPlanEntry, CostContext, PlanOutput, audit_resource_id from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit diff --git a/src/sqlbuild/executor/pipeline/_helpers/connections.py b/src/sqlbuild/executor/pipeline/_helpers/connections.py index adcbb3d091..e960bd874e 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/connections.py +++ b/src/sqlbuild/executor/pipeline/_helpers/connections.py @@ -8,9 +8,8 @@ from contextvars import copy_context from typing import Any -from sqlbuild._native import BuildPhaseTimingTracker +from sqlbuild._native import BuildPhaseTimingTracker, PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import BuildCallbacks from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from sqlbuild.executor.pipeline.models import BuildConnectionPreparation diff --git a/src/sqlbuild/executor/pipeline/_helpers/graph_width.py b/src/sqlbuild/executor/pipeline/_helpers/graph_width.py index 887e67dec7..53c776211f 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/graph_width.py +++ b/src/sqlbuild/executor/pipeline/_helpers/graph_width.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput def runnable_graph_width(*, plan: PlanOutput) -> int: diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index 60f819b29f..47b2649c76 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -16,7 +16,11 @@ from typing import Any, ClassVar, Protocol from sqlbuild._native import ( + CompiledSqlScenario, + CompilePipelineResult, NativeProgressProjector, + ScenarioExecutionPlan, + SqlLexicalSyntax, current_native_progress_projector, error_code, error_help, @@ -26,10 +30,6 @@ from sqlbuild._native import build_cli_scenario_plan as build_scenario_plan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.pipeline._helpers.connections import ( close_connections, diff --git a/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py b/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py index b792ee08e2..9110443534 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py +++ b/src/sqlbuild/executor/pipeline/_helpers/schema_preparation.py @@ -4,10 +4,9 @@ from typing import Any +from sqlbuild._native import PlanOutput, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SourceEntry def prepare_build_schemas( diff --git a/src/sqlbuild/executor/pipeline/_helpers/seeding.py b/src/sqlbuild/executor/pipeline/_helpers/seeding.py index 0808a3a4cc..e608c77529 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/seeding.py +++ b/src/sqlbuild/executor/pipeline/_helpers/seeding.py @@ -8,9 +8,9 @@ from contextvars import copy_context from typing import Any, cast +from sqlbuild._native import PlanOutput, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import PlanOutput, SeedPlanEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed diff --git a/src/sqlbuild/executor/pipeline/_helpers/settings.py b/src/sqlbuild/executor/pipeline/_helpers/settings.py index fb3fa386ca..f640d45d49 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/settings.py +++ b/src/sqlbuild/executor/pipeline/_helpers/settings.py @@ -4,6 +4,7 @@ from dataclasses import replace +from sqlbuild._native import SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.executor.build.models import ( @@ -13,7 +14,6 @@ BuildRuntimeParams, ) from sqlbuild.executor.pipeline.models import ResolvedBuildInputs -from sqlbuild.spec.contracts.models import SettingsConfig def resolve_promotion_mode( diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 81577014c0..0a8fd02066 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -12,16 +12,18 @@ from typing import Any, cast from uuid import uuid4 -from sqlbuild._native import sql_test_resource_id -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import FunctionInfo -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + FunctionPlanEntry, + PlanOutput, + SqlTestPlanEntry, + sql_test_resource_id, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import FunctionInfo from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import FunctionPlanEntry, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function from sqlbuild.executor.pipeline._helpers.connections import ( diff --git a/src/sqlbuild/executor/pipeline/main/run.py b/src/sqlbuild/executor/pipeline/main/run.py index e2b61b7ec7..eba798e2f5 100644 --- a/src/sqlbuild/executor/pipeline/main/run.py +++ b/src/sqlbuild/executor/pipeline/main/run.py @@ -6,9 +6,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import BuildPhaseTimingTracker, CostContext +from sqlbuild._native import BuildPhaseTimingTracker, CostContext, PlanOutput, SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.main._external_source_loads import ( @@ -65,7 +64,6 @@ from sqlbuild.runtime.observability.main.create_lifecycle_event import create_lifecycle_event from sqlbuild.runtime.observability.main.current_event_dispatcher import current_event_dispatcher from sqlbuild.runtime.observability.main.run_scope import run_scope -from sqlbuild.spec.contracts.models import SettingsConfig def run_build_pipeline( diff --git a/src/sqlbuild/executor/pipeline/models.py b/src/sqlbuild/executor/pipeline/models.py index 6f5b5f519f..9190a261b0 100644 --- a/src/sqlbuild/executor/pipeline/models.py +++ b/src/sqlbuild/executor/pipeline/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any -from sqlbuild.compiler.planner.models import AuditPlanEntry, SqlTestPlanEntry +from sqlbuild._native import AuditPlanEntry, SqlTestPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 1c36b1e9dd..43bd63753c 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -8,9 +8,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredAssetFunction from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction from sqlbuild.compiler.python_nodes.types import ( PythonNodeFanInAction, PythonNodeKind, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py index 7e8e656d48..bb4aa6a85d 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/fingerprinting.py @@ -7,10 +7,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import write_fingerprint +from sqlbuild._native import Fingerprint, PythonNodeIdentity, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity def try_write_python_node_identity_fingerprint( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 3faa8e9f0e..6a24e47e14 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -8,18 +8,17 @@ from functools import partial from typing import Any -from sqlbuild._native import CostContext -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( + CostContext, DiscoveredAssetFunction, DiscoveredLoaderFunction, - DiscoveredTaskFunction, -) -from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, + DiscoveredTaskFunction, PythonNodeGraph, PythonSqlRunLifecyclePlan, + SourceEntry, ) +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.main._execute import execute_source_load @@ -57,7 +56,6 @@ current_execution_identity, ) from sqlbuild.runtime.observability.models import ExecutionIdentity -from sqlbuild.spec.contracts.models import SourceEntry def execute_ingress_python_loader_nodes( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py index 9a1654c004..a82fdd2210 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/lifecycle_nodes.py @@ -2,11 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.python_nodes.models import ( - DiscoveredPythonNode, - PythonNodeGraph, - PythonSqlRunLifecyclePlan, -) +from sqlbuild._native import DiscoveredPythonNode, PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.executor.scheduling.models import LifecycleExecutionNode diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index fe5a64ac54..87027f7f5c 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -6,15 +6,15 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import CostContext -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.python_nodes.models import ( +from sqlbuild._native import ( + CostContext, + DiscoveredCheckFunction, DiscoveredPythonNode, PythonNodeGraph, PythonNodeIdentity, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py index 5e6eead8b3..fd755d2af9 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/relation_guard.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind +from sqlbuild._native import HardCodedRelationOwnerKind from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.models import PythonNodeRuntime diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index bed168a6ee..b85237fa24 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -5,9 +5,8 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredPythonNode, PythonNodeGraph from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( diff --git a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py index 532a4f37b8..a5efd87f3d 100644 --- a/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py +++ b/src/sqlbuild/executor/python_nodes/classes/runtime_relation_guard.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from sqlbuild._native import ( + HardCodedRelationOwnerKind, extract_relation_names, hard_coded_relation_remedy, match_project_relation, @@ -16,7 +17,6 @@ ProjectRelationIndex, RelationName, ) -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef _LOGGER: logging.Logger = logging.getLogger(__name__) diff --git a/src/sqlbuild/executor/python_nodes/main/_checks.py b/src/sqlbuild/executor/python_nodes/main/_checks.py index 8068a7640c..91ac05fd7c 100644 --- a/src/sqlbuild/executor/python_nodes/main/_checks.py +++ b/src/sqlbuild/executor/python_nodes/main/_checks.py @@ -4,8 +4,7 @@ from collections.abc import Mapping -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import DiscoveredCheckFunction, PythonNodeGraph from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes from sqlbuild.executor.python_nodes.models import ( diff --git a/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py b/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py index 8ae8eb7a7e..33dbdf9b53 100644 --- a/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py +++ b/src/sqlbuild/executor/python_nodes/main/_fingerprinting.py @@ -5,8 +5,8 @@ import logging from typing import Any +from sqlbuild._native import PythonNodeIdentity from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint as _try_write_python_node_identity_fingerprint, ) diff --git a/src/sqlbuild/executor/python_nodes/main/_ingress.py b/src/sqlbuild/executor/python_nodes/main/_ingress.py index b213494c47..ea1d03a5f5 100644 --- a/src/sqlbuild/executor/python_nodes/main/_ingress.py +++ b/src/sqlbuild/executor/python_nodes/main/_ingress.py @@ -4,8 +4,7 @@ from collections.abc import Mapping -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import DiscoveredLoaderFunction, PythonNodeGraph, SourceEntry from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( execute_ingress_python_loader_nodes, ) @@ -14,7 +13,6 @@ PythonIngressLoaderExecutorResult, PythonNodeRuntime, ) -from sqlbuild.spec.contracts.models import SourceEntry def run_ingress_python_loader_nodes( diff --git a/src/sqlbuild/executor/python_nodes/main/read_side.py b/src/sqlbuild/executor/python_nodes/main/read_side.py index 46725c23af..cc5613834c 100644 --- a/src/sqlbuild/executor/python_nodes/main/read_side.py +++ b/src/sqlbuild/executor/python_nodes/main/read_side.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import PythonNodeGraph from sqlbuild.executor.python_nodes.classes.read_side_python_execution_tracker import ( ReadSidePythonExecutionTracker, ) diff --git a/src/sqlbuild/executor/python_nodes/types.py b/src/sqlbuild/executor/python_nodes/types.py index 31c512f726..222d443440 100644 --- a/src/sqlbuild/executor/python_nodes/types.py +++ b/src/sqlbuild/executor/python_nodes/types.py @@ -4,8 +4,7 @@ from typing import TYPE_CHECKING, Protocol -from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity +from sqlbuild._native import DiscoveredAssetFunction, DiscoveredTaskFunction, PythonNodeIdentity type ExecutablePythonNode = DiscoveredTaskFunction | DiscoveredAssetFunction diff --git a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py index 52cff927a5..7f0ab92c27 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/final_audits.py +++ b/src/sqlbuild/executor/run/_helpers/execution/final_audits.py @@ -4,11 +4,9 @@ from typing import Any +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, Fingerprint, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.audit import ( @@ -16,7 +14,6 @@ reused_final_audit_results_by_binding_key, ) from sqlbuild.executor.run.models import FinalAuditRun, ModelMaterializationContext -from sqlbuild.spec.contracts.models import SourceEntry def run_final_model_audits( diff --git a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py index ae1d8ae8a2..6841a6b8f4 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hook_phases.py @@ -4,8 +4,8 @@ from collections.abc import Mapping +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/src/sqlbuild/executor/run/_helpers/execution/hooks.py b/src/sqlbuild/executor/run/_helpers/execution/hooks.py index e5cd6aa0b1..a83dee9f83 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/hooks.py +++ b/src/sqlbuild/executor/run/_helpers/execution/hooks.py @@ -5,23 +5,25 @@ from collections.abc import Mapping from typing import Any -from sqlbuild._native import CostContext, render_source_relation +from sqlbuild._native import ( + CompiledRelationLocation, + CostContext, + DiscoveredHookFunction, + HardCodedRelationOwnerKind, + PythonHookEntry, + PythonNodeIdentity, + SourceEntry, + SqlHookEntry, + render_source_relation, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import ( - DiscoveredHookFunction, - PythonHookEntry, - SqlHookEntry, -) from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_HOOK from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard @@ -47,7 +49,6 @@ from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry def execute_hooks( diff --git a/src/sqlbuild/executor/run/_helpers/execution/promotion.py b/src/sqlbuild/executor/run/_helpers/execution/promotion.py index c3c20f17d8..63b72639b1 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/promotion.py +++ b/src/sqlbuild/executor/run/_helpers/execution/promotion.py @@ -4,6 +4,7 @@ from typing import Any +from sqlbuild._native import OPERATION_STRATEGIES from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import PromotionStrategy @@ -14,10 +15,7 @@ OperationAttributes, OperationLifecycle, ) -from sqlbuild.runtime.observability.constants import ( - OPERATION_STRATEGIES, - RENAME_OPERATION_STRATEGY, -) +from sqlbuild.runtime.observability.constants import RENAME_OPERATION_STRATEGY from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) diff --git a/src/sqlbuild/executor/run/_helpers/execution/results.py b/src/sqlbuild/executor/run/_helpers/execution/results.py index 502bb15301..44c120fa46 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/results.py +++ b/src/sqlbuild/executor/run/_helpers/execution/results.py @@ -4,14 +4,17 @@ from dataclasses import replace -from sqlbuild._native import error_code, error_help, error_message, format_waiting_on_empty_inputs -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CursorBounds, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, ModelPlanEntry, + error_code, + error_help, + error_message, + format_waiting_on_empty_inputs, ) +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.constants import ( RUN_AUDIT_FAILED_CODE, diff --git a/src/sqlbuild/executor/run/_helpers/execution/staging.py b/src/sqlbuild/executor/run/_helpers/execution/staging.py index a2c668d98e..8b34bf380e 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/staging.py +++ b/src/sqlbuild/executor/run/_helpers/execution/staging.py @@ -4,9 +4,9 @@ from typing import Any +from sqlbuild._native import Fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.executor.run.models import ModelMaterializationContext from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py index f93a559bb0..4885ea39c1 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_targets.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_targets.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild._native import fit_auxiliary_relation_name +from sqlbuild._native import ModelPlanEntry, fit_auxiliary_relation_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, @@ -10,7 +10,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.executor.run.constants import STAGING_RELATION_SUFFIX from sqlbuild.executor.run.models import TableTargets diff --git a/src/sqlbuild/executor/run/_helpers/materializations/custom.py b/src/sqlbuild/executor/run/_helpers/materializations/custom.py index e3e6fbaa28..d12c469e5a 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/custom.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/custom.py @@ -7,6 +7,13 @@ from dataclasses import replace from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + ModelPlanEntry, + PlanReason, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo @@ -14,9 +21,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.main._execute import execute_audit @@ -46,7 +50,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.provider.main.runtime import _empty_provider_container, invoke_with_providers from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.models import SourceEntry def execute_custom_entry( diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index cf4c517f67..e0c222de4f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -5,12 +5,17 @@ from dataclasses import dataclass, replace from typing import Any -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import ( + CursorBounds, + IncrementalStrategy, + ModelPlanEntry, + OnSchemaChange, + TableType, fit_auxiliary_relation_name, future_cursor_cap_warning, maximum_start_cap_warning, ) +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -21,8 +26,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry -from sqlbuild.compiler.planner.types import IncrementalStrategy, OnSchemaChange from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -69,7 +72,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.types import TableType _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 599fc8b238..27f888d74a 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -12,16 +12,35 @@ from typing import Any from sqlbuild._native import ( + AlignedInterval, + BackfillAction, + CursorBounds, + CursorGrain, + CursorType, + DateValue, + Duration, + IncrementalStrategy, + IntegerValue, + MicrobatchLimitAction, + MicrobatchStrategy, + ModelPlanEntry, + OnSchemaChange, + TableType, + TimestampValue, aligned_timestamp_bounds, compute_query_hash, cursor_bound_display, + deterministic_microbatch_event_id, empty_cursor_window, fit_auxiliary_relation_name, future_cursor_cap_warning, inclusive_cursor_end, + latest_active_replay_requirement, maximum_start_cap_warning, merge_microbatch_intervals, microbatch_limit_warning, + project_microbatch_coverage, + project_replay_requirement, resolve_effective_microbatch_batch_size, ) from sqlbuild._native import cursor_cap_from_end as cap_from_end @@ -46,20 +65,6 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - Duration, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - IncrementalStrategy, - MicrobatchStrategy, - OnSchemaChange, -) -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event @@ -125,14 +130,6 @@ MICROBATCH_REPLAY_GENERATION_PREFIX, ) from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.main.deterministic_event_id import ( - deterministic_microbatch_event_id, -) -from sqlbuild.microbatches.main.latest_replay_requirement import ( - latest_active_replay_requirement, -) -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, @@ -162,7 +159,6 @@ canonicalize_operation_adapter, ) from sqlbuild.runtime.observability.models import MicrobatchLifecycleContext -from sqlbuild.spec.contracts.types import MicrobatchLimitAction, TableType _DEFAULT_ON_SCHEMA_CHANGE: OnSchemaChange = OnSchemaChange.APPEND_NEW_COLUMNS _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 9c15fb0b75..0dcf20cd6d 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -4,7 +4,18 @@ from typing import Any -from sqlbuild._native import fit_auxiliary_relation_name +from sqlbuild._native import ( + SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, + SNAPSHOT_DEFAULT_VALID_TO_COLUMN, + HistoricalInput, + InitialValidFrom, + ModelPlanEntry, + PlanReason, + SnapshotSchemaChangePolicy, + SnapshotsConfig, + SnapshotStrategy, + fit_auxiliary_relation_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -22,18 +33,6 @@ ) from sqlbuild.adapter.type_system.main.normalize_type import normalize_type from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.constants import ( - SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, - SNAPSHOT_DEFAULT_VALID_TO_COLUMN, -) -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - HistoricalInput, - InitialValidFrom, - PlanReason, - SnapshotSchemaChangePolicy, - SnapshotStrategy, -) from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -78,7 +77,6 @@ from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, ) -from sqlbuild.spec.contracts.models import SnapshotsConfig _SCHEMA_CHANGE_STRICTNESS: dict[SnapshotSchemaChangePolicy, int] = { SnapshotSchemaChangePolicy.APPEND_NEW_COLUMNS: 0, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py index d8eb5b41a5..9aca5f50e1 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/table_cursor.py @@ -5,12 +5,13 @@ from dataclasses import replace from sqlbuild._native import ( + CursorBounds, + ModelPlanEntry, empty_cursor_window, future_cursor_cap_warning, replaces_incremental_relation, ) from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index d26eb5257d..4d3d1453b6 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -5,15 +5,19 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + DiscoveredHookFunction, + ModelPlanEntry, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -31,7 +35,6 @@ ) from sqlbuild.executor.run.types import ExecutionPhase, HookPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceEntry def execute_view_entry( diff --git a/src/sqlbuild/executor/run/_helpers/reuse/audit.py b/src/sqlbuild/executor/run/_helpers/reuse/audit.py index 3f6659a223..5f5b0ee20d 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/audit.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/audit.py @@ -2,9 +2,8 @@ from __future__ import annotations -from sqlbuild._native import build_audit_gate_identity +from sqlbuild._native import AuditPlanEntry, build_audit_gate_identity from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope, AuditSeverity -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( reuse_from_audit_gate_reuse_decision, diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py index d1130262d5..e9dc878139 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprint_metadata.py @@ -7,12 +7,15 @@ from collections.abc import Mapping from typing import cast -from sqlbuild._native import build_audit_gate_identity -from sqlbuild.compiler.auditing.models import AuditGateIdentity, AuditIdentity +from sqlbuild._native import ( + AUDIT_GATE_METADATA_KEY, + MIGRATION_FINGERPRINT_METADATA_KEY, + AuditGateIdentity, + AuditIdentity, + AuditPlanEntry, + build_audit_gate_identity, +) from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity -from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( AuditGateMetadata, diff --git a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py index 4950a2afcf..b32c74fb52 100644 --- a/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py +++ b/src/sqlbuild/executor/run/_helpers/reuse/fingerprinting.py @@ -6,11 +6,15 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import compute_query_hash, write_fingerprint +from sqlbuild._native import ( + NODE_TYPE_MODEL, + AuditPlanEntry, + Fingerprint, + ModelPlanEntry, + compute_query_hash, + write_fingerprint, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( model_fingerprint_metadata_with_audit_gate, diff --git a/src/sqlbuild/executor/run/_helpers/validation/contracts.py b/src/sqlbuild/executor/run/_helpers/validation/contracts.py index 0836cc0b4b..ae8fefe93b 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/contracts.py +++ b/src/sqlbuild/executor/run/_helpers/validation/contracts.py @@ -2,11 +2,10 @@ from __future__ import annotations -from sqlbuild._native import matching_dynamic_families +from sqlbuild._native import ModelPlanEntry, SchemaDynamicColumnFamily, matching_dynamic_families from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.constants import ( RUNTIME_CONTRACT_EXTRA_COLUMN_CODE, @@ -14,7 +13,6 @@ RUNTIME_CONTRACT_MISSING_DECLARATIONS_CODE, RUNTIME_CONTRACT_TYPE_MISMATCH_CODE, ) -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily def validate_runtime_contract( diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index edd3b81662..5879375918 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -9,6 +9,27 @@ from typing import Any from sqlbuild._native import ( + GRAIN_ORDER, + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + ZERO_DAY_CURSOR_DURATION, + AlignedInterval, + BackfillAction, + CursorBounds, + CursorGrain, + CursorInputEvidence, + CursorInputRelation, + CursorType, + DateValue, + Duration, + EmptyCursorInputDecision, + IntegerValue, + MaximumStartPolicyInputs, + MicrobatchStrategy, + ModelCursorSnapshot, + ModelPlanEntry, + StartCursorsConfig, + TimestampValue, apply_maximum_start_policy, apply_typed_cursor_replay_policy, cursor_window_override_flags, @@ -33,37 +54,11 @@ from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputEvidence, - CursorInputRelation, - Duration, - EmptyCursorInputDecision, - MaximumStartPolicyInputs, - ModelCursorSnapshot, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - CursorGrain, - CursorType, - MicrobatchStrategy, -) -from sqlbuild.cursor_algebra.constants import GRAIN_ORDER -from sqlbuild.cursor_algebra.models import ( - AlignedInterval, - DateValue, - IntegerValue, - TimestampValue, -) from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode, WatermarkResolver -from sqlbuild.spec.contracts.constants import ZERO_DAY_CURSOR_DURATION -from sqlbuild.spec.contracts.models import StartCursorsConfig def has_runtime_owned_cursor_watermarks( diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index f4a286b14f..8562c2145c 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -5,19 +5,19 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import setting_help -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.discovery.constants import ( +from sqlbuild._native import ( PROJECT_CONFIG_FILENAME, SETTINGS_SECTION, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, + Fingerprint, + ModelPlanEntry, + setting_help, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ModelPlanEntry +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/src/sqlbuild/executor/run/main/_table_targets.py b/src/sqlbuild/executor/run/main/_table_targets.py index 69539ca521..7981a8595f 100644 --- a/src/sqlbuild/executor/run/main/_table_targets.py +++ b/src/sqlbuild/executor/run/main/_table_targets.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.executor.run._helpers.execution.table_targets import ( resolve_table_targets as _resolve_table_targets, ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 1ea931c07e..598dc135be 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -8,25 +8,33 @@ from datetime import datetime from typing import Any, cast -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_parse as parse -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( AuditPlanEntry, + CompiledRelationLocation, CursorBounds, + CursorGrain, + CursorType, + DateValue, + DiscoveredHookFunction, Duration, FutureCursorSafetyEvidence, + FutureCursorsConfig, + IntegerValue, MaximumStartSafetyEvidence, + MicrobatchLimitAction, ModelPlanEntry, + PythonHookEntry, + SourceEntry, + StartCursorsConfig, + TimestampValue, ) -from sqlbuild.compiler.planner.types import CursorGrain, CursorType +from sqlbuild._native import cursor_compare as compare +from sqlbuild._native import cursor_parse as parse +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult @@ -47,8 +55,6 @@ from sqlbuild.microbatches.types import MicrobatchEventStore from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import FutureCursorsConfig, SourceEntry, StartCursorsConfig -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True, kw_only=True, init=False) diff --git a/src/sqlbuild/executor/scenario/_helpers/capture/core.py b/src/sqlbuild/executor/scenario/_helpers/capture/core.py index 7cd3e0e670..8208c2be38 100644 --- a/src/sqlbuild/executor/scenario/_helpers/capture/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/capture/core.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scenario._helpers.lifecycle.failures import first_failure_details from sqlbuild.executor.scenario._helpers.lifecycle.fixtures import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py b/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py index a51ea11a8f..81fec1cca5 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/interrupts.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario.classes.scenario_interrupts import ScenarioInterrupts from sqlbuild.executor.scenario.main._cleanup import execute_scenario_cleanup diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py b/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py index b4c113d910..08df6f107c 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/model_execution.py @@ -5,9 +5,9 @@ import threading from typing import Any +from sqlbuild._native import ModelPlanEntry, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ModelPlanEntry, ScenarioExecutionPlan from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.exceptions import ScenarioStopRequested from sqlbuild.executor.scenario.main._execute import execute_scenario_model diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/run.py b/src/sqlbuild/executor/scenario/_helpers/execution/run.py index 46012a9668..4eb7161627 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/run.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/run.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan, ScenarioRelationMap from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan, ScenarioRelationMap from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.interrupts import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py index 6575d7cb45..7988d10312 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/cleanup.py @@ -7,6 +7,15 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ( + CompiledRelationLocation, + MaterializationType, + ModelPlanEntry, + ScenarioArtifactKind, + ScenarioExecutionPlan, + ScenarioFixturePlan, + SeedPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.contract.types import RelationType @@ -17,14 +26,6 @@ resolve_relation_location_qualified_name, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - ScenarioExecutionPlan, - ScenarioFixturePlan, - SeedPlanEntry, -) -from sqlbuild.compiler.planner.types import MaterializationType, ScenarioArtifactKind from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.run.main._table_targets import resolve_table_targets from sqlbuild.executor.run.main._type_enforcement_relation_name import ( diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py index 3b9b7cb9d4..74997a5385 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/expectations.py @@ -4,12 +4,12 @@ from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.scenario.main._assertion_expectations import ( execute_scenario_assertion_expectation, ) diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py index 7d90a5b52e..4ceed41ace 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/fixtures.py @@ -5,9 +5,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import ScenarioFixturePlan, SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ScenarioFixturePlan, SeedPlanEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_SEED_FAILED from sqlbuild.executor.scenario.main._fixtures import execute_scenario_fixture diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index 90a20fb144..a702cae017 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -6,21 +6,24 @@ from pathlib import Path from typing import Any -from sqlbuild._native import error_code, error_help, error_message -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( + CompiledRelationLocation, FunctionPlanEntry, ModelPlanEntry, + ScenarioArtifactKind, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioRelationPlan, + SourceEntry, + error_code, + error_help, + error_message, ) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.executor.build.models import FunctionExecutionResult from sqlbuild.executor.functions.main._execute import execute_function from sqlbuild.executor.run.models import ModelExecutionResult @@ -69,7 +72,6 @@ from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( ResourceAttemptLifecycle, ) -from sqlbuild.spec.contracts.models import SourceEntry _LOCAL_STEP_FAILED_MESSAGE: str = "local scenario step failed" diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index e24624c68a..313c83e736 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -5,8 +5,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import error_code, error_message -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, error_code, error_message from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.scenario._helpers.snapshots.core import ( is_scenario_snapshot_fresh, diff --git a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py index 78e7096b66..5f0fa53810 100644 --- a/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/snapshots/core.py @@ -11,13 +11,14 @@ from typing import Any from uuid import UUID -from sqlbuild._native import error_code, error_message -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( + ScenarioArtifactKind, ScenarioExecutionPlan, ScenarioFixturePlan, SeedPlanEntry, + error_code, + error_message, ) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.scenario.constants import ( JSONL_BLANK_LINE, diff --git a/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py b/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py index d8e316f66b..20d9abd2ae 100644 --- a/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py +++ b/src/sqlbuild/executor/scenario/classes/prepared_scenario_schemas.py @@ -6,10 +6,9 @@ import threading from typing import Any +from sqlbuild._native import CompiledRelationLocation, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.diagnostics.main.log_debug_event import log_debug_event _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/scenario/classes/running_scenarios.py b/src/sqlbuild/executor/scenario/classes/running_scenarios.py index 174fbe19ff..08f48fb20e 100644 --- a/src/sqlbuild/executor/scenario/classes/running_scenarios.py +++ b/src/sqlbuild/executor/scenario/classes/running_scenarios.py @@ -6,8 +6,8 @@ import threading from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.diagnostics.main.log_debug_event import log_debug_event _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") diff --git a/src/sqlbuild/executor/scenario/main/_assertion_expectations.py b/src/sqlbuild/executor/scenario/main/_assertion_expectations.py index 9b1a1a17ca..c328feb06a 100644 --- a/src/sqlbuild/executor/scenario/main/_assertion_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_assertion_expectations.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioAssertionExpectationPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioAssertionExpectationPlan from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_ASSERTION_ERRORED, SCENARIO_EXEC_ASSERTION_FAILED, diff --git a/src/sqlbuild/executor/scenario/main/_capture_steps.py b/src/sqlbuild/executor/scenario/main/_capture_steps.py index 1054cee4b6..0149702f29 100644 --- a/src/sqlbuild/executor/scenario/main/_capture_steps.py +++ b/src/sqlbuild/executor/scenario/main/_capture_steps.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.capture.core import execute_scenario_snapshot_capture_steps from sqlbuild.executor.scenario.models import ( ScenarioCaptureSettings, diff --git a/src/sqlbuild/executor/scenario/main/_cleanup.py b/src/sqlbuild/executor/scenario/main/_cleanup.py index 78e1b5f68d..291cd7ea0d 100644 --- a/src/sqlbuild/executor/scenario/main/_cleanup.py +++ b/src/sqlbuild/executor/scenario/main/_cleanup.py @@ -4,10 +4,9 @@ from typing import Any +from sqlbuild._native import MaterializationType, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.scenario._helpers.lifecycle.cleanup import ( collect_scenario_cleanup_targets, diff --git a/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py b/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py index 75ad2e6a55..da119bd1a6 100644 --- a/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py +++ b/src/sqlbuild/executor/scenario/main/_cleanup_interrupted.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.execution.interrupts import ( cleanup_interrupted_scenarios as _cleanup_interrupted_scenarios, ) diff --git a/src/sqlbuild/executor/scenario/main/_execute.py b/src/sqlbuild/executor/scenario/main/_execute.py index f629db08d1..e3e0d90ce6 100644 --- a/src/sqlbuild/executor/scenario/main/_execute.py +++ b/src/sqlbuild/executor/scenario/main/_execute.py @@ -5,10 +5,9 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import MaterializationType, ModelPlanEntry, PlanAction, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ModelPlanEntry, ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction from sqlbuild.executor.run.main._execute import execute_table_entry, execute_view_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_MODEL_FAILED diff --git a/src/sqlbuild/executor/scenario/main/_expected_expectations.py b/src/sqlbuild/executor/scenario/main/_expected_expectations.py index d848eb1477..933d59cc38 100644 --- a/src/sqlbuild/executor/scenario/main/_expected_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_expected_expectations.py @@ -4,12 +4,11 @@ from typing import Any -from sqlbuild._native import build_scenario_expected_comparison_sql +from sqlbuild._native import ScenarioExpectedExpectationPlan, build_scenario_expected_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ScenarioExpectedExpectationPlan from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_EXPECTED_ERRORED, SCENARIO_EXEC_EXPECTED_FAILED, diff --git a/src/sqlbuild/executor/scenario/main/_fixtures.py b/src/sqlbuild/executor/scenario/main/_fixtures.py index 5fb4ec4969..7a03b0915e 100644 --- a/src/sqlbuild/executor/scenario/main/_fixtures.py +++ b/src/sqlbuild/executor/scenario/main/_fixtures.py @@ -4,12 +4,12 @@ from typing import Any +from sqlbuild._native import ScenarioFixturePlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import ScenarioFixturePlan from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.scenario.constants import SCENARIO_EXEC_FIXTURE_FAILED from sqlbuild.executor.scenario.models import ScenarioFixtureExecutionResult diff --git a/src/sqlbuild/executor/scenario/main/_local.py b/src/sqlbuild/executor/scenario/main/_local.py index ffd7242821..388ce36cbf 100644 --- a/src/sqlbuild/executor/scenario/main/_local.py +++ b/src/sqlbuild/executor/scenario/main/_local.py @@ -4,9 +4,9 @@ from pathlib import Path +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.local.execution import ( execute_local_scenario_load_only_run as _execute_local_scenario_load_only_run, ) diff --git a/src/sqlbuild/executor/scenario/main/_run.py b/src/sqlbuild/executor/scenario/main/_run.py index 16e6f533ee..05a4f5faca 100644 --- a/src/sqlbuild/executor/scenario/main/_run.py +++ b/src/sqlbuild/executor/scenario/main/_run.py @@ -4,8 +4,8 @@ from typing import Any +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.execution.run import execute_scenario_run_steps from sqlbuild.executor.scenario.models import ScenarioRunOptions, ScenarioRunResult diff --git a/src/sqlbuild/executor/scenario/main/_snapshots.py b/src/sqlbuild/executor/scenario/main/_snapshots.py index e8f1975e65..980693ca12 100644 --- a/src/sqlbuild/executor/scenario/main/_snapshots.py +++ b/src/sqlbuild/executor/scenario/main/_snapshots.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan +from sqlbuild._native import ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.snapshots.core import ( classify_scenario_snapshot_state as _classify_scenario_snapshot_state, ) diff --git a/src/sqlbuild/executor/scenario/models.py b/src/sqlbuild/executor/scenario/models.py index a0fdb38e90..22c2fd7251 100644 --- a/src/sqlbuild/executor/scenario/models.py +++ b/src/sqlbuild/executor/scenario/models.py @@ -5,15 +5,16 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CompiledRelationLocation, + MaterializationType, + ScenarioArtifactKind, ScenarioExecutionPlan, ScenarioRelationMap, + SqlLexicalSyntax, ) -from sqlbuild.compiler.planner.types import MaterializationType, ScenarioArtifactKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.executor.build.models import FunctionExecutionResult, SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario.classes.prepared_scenario_schemas import PreparedScenarioSchemas diff --git a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py index 7e6a2e752e..3bb9f4842d 100644 --- a/src/sqlbuild/executor/seed/_helpers/fingerprinting.py +++ b/src/sqlbuild/executor/seed/_helpers/fingerprinting.py @@ -6,11 +6,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import write_fingerprint +from sqlbuild._native import NODE_TYPE_SEED, Fingerprint, SeedPlanEntry, write_fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import SeedPlanEntry def try_write_seed_fingerprint( diff --git a/src/sqlbuild/executor/seed/main/_execute.py b/src/sqlbuild/executor/seed/main/_execute.py index e159b62808..b4ddb026cd 100644 --- a/src/sqlbuild/executor/seed/main/_execute.py +++ b/src/sqlbuild/executor/seed/main/_execute.py @@ -5,12 +5,12 @@ import time from typing import Any +from sqlbuild._native import SeedPlanEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) -from sqlbuild.compiler.planner.models import SeedPlanEntry from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed._helpers.fingerprinting import try_write_seed_fingerprint diff --git a/src/sqlbuild/executor/testing/_helpers/difference_samples.py b/src/sqlbuild/executor/testing/_helpers/difference_samples.py index 70bb3a2911..9d6f02f246 100644 --- a/src/sqlbuild/executor/testing/_helpers/difference_samples.py +++ b/src/sqlbuild/executor/testing/_helpers/difference_samples.py @@ -6,9 +6,8 @@ from dataclasses import replace from typing import Any, Final -from sqlbuild._native import render_sql_test_difference_sample_sql +from sqlbuild._native import ChainStep, SqlTestPlanEntry, render_sql_test_difference_sample_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.executor.testing.models import ( diff --git a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py index c82b524ce9..27df21d025 100644 --- a/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py +++ b/src/sqlbuild/executor/testing/_helpers/expected_column_probe.py @@ -5,9 +5,12 @@ import logging from typing import Any -from sqlbuild._native import describe_missing_expected_columns, render_sql_test_comparison_sql +from sqlbuild._native import ( + SqlTestPlanEntry, + describe_missing_expected_columns, + render_sql_test_comparison_sql, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index f0c31495c7..69951e0ae0 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -4,10 +4,9 @@ from typing import Any -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.executor.testing._helpers.difference_samples import add_difference_samples from sqlbuild.executor.testing._helpers.expected_column_probe import ( missing_expected_columns_message, diff --git a/src/sqlbuild/executor/testing/main/_sql_length.py b/src/sqlbuild/executor/testing/main/_sql_length.py index 3cb9b9c5e8..1ae52e7a79 100644 --- a/src/sqlbuild/executor/testing/main/_sql_length.py +++ b/src/sqlbuild/executor/testing/main/_sql_length.py @@ -2,9 +2,9 @@ from __future__ import annotations +from sqlbuild._native import CompileInputError from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.types import StatementSizeLimit -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.testing.constants import SQL_TEST_SIZE_BYTES diff --git a/src/sqlbuild/executor/testing/models.py b/src/sqlbuild/executor/testing/models.py index 9935ec7411..40247d1a60 100644 --- a/src/sqlbuild/executor/testing/models.py +++ b/src/sqlbuild/executor/testing/models.py @@ -5,9 +5,8 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration +from sqlbuild._native import SqlTestParameterDeclaration, SqlValue from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.sql_values.models import SqlValue @dataclass(frozen=True) diff --git a/src/sqlbuild/integrations/__init__.py b/src/sqlbuild/integrations/__init__.py index a0dbf8a6fb..e69de29bb2 100644 --- a/src/sqlbuild/integrations/__init__.py +++ b/src/sqlbuild/integrations/__init__.py @@ -1 +0,0 @@ -"""Integration package.""" diff --git a/src/sqlbuild/integrations/dagster/_helpers/assets.py b/src/sqlbuild/integrations/dagster/_helpers/assets.py index 555137015e..c7841a94c6 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/assets.py +++ b/src/sqlbuild/integrations/dagster/_helpers/assets.py @@ -5,16 +5,11 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import dagster_asset_plans, dagster_check_plans from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, ) -from sqlbuild.integrations.dagster.constants import ( - DAGSTER_ASSET_NODE_KINDS, - DAGSTER_DIRECT_KIND_NODE_KINDS, - MODEL_NODE_KIND, - VIEW_MATERIALIZATION_TYPE, -) def build_asset_specs( @@ -23,22 +18,13 @@ def build_asset_specs( """Build materializable Dagster asset specs from a SQLBuild DAG artifact.""" dg: Any = load_dagster() - nodes_by_id: dict[str, Mapping[str, Any]] = _nodes_by_id(dag) - upstream_by_id: dict[str, list[str]] = _upstream_by_id(dag) - project_name: object = dag.get("project_name") specs: list[Any] = [] - for node in dag["nodes"]: - if str(node.get("kind")) not in DAGSTER_ASSET_NODE_KINDS: - continue - translated_node: Mapping[str, Any] = {**node, "project_name": project_name} + for translated_node, upstream_nodes, kinds in dagster_asset_plans(dag): if not translator.is_asset_node(translated_node): continue - deps: list[Any] = [] - for upstream_id in upstream_by_id.get(str(node["id"]), []): - upstream_node: Mapping[str, Any] | None = nodes_by_id.get(upstream_id) - if upstream_node is None: - continue - deps.append(dg.AssetDep(translator.get_asset_key(upstream_node))) + deps: list[Any] = [ + dg.AssetDep(translator.get_asset_key(upstream_node)) for upstream_node in upstream_nodes + ] specs.append( dg.AssetSpec( key=translator.get_asset_key(translated_node), @@ -47,7 +33,7 @@ def build_asset_specs( group_name=translator.get_group_name(translated_node), metadata=translator.get_metadata(translated_node), tags=translator.get_tags(translated_node), - kinds=_asset_spec_kinds(translated_node), + kinds=kinds, ) ) return tuple(specs) @@ -61,12 +47,9 @@ def build_check_specs( ) -> tuple[Any, ...]: """Build Dagster check specs from SQLBuild test/audit/scenario checks.""" - excluded_kinds: set[str] = set() if include_scenarios else {"scenario"} return _build_check_specs( - dag=dag, + plans=dagster_check_plans(dag, include_scenarios=include_scenarios, only_scenarios=False), translator=translator, - kinds=None, - excluded_kinds=excluded_kinds, ) @@ -76,70 +59,30 @@ def build_scenario_check_specs( """Build Dagster check specs for SQLBuild scenario checks only.""" return _build_check_specs( - dag=dag, + plans=dagster_check_plans(dag, include_scenarios=True, only_scenarios=True), translator=translator, - kinds={"scenario"}, - excluded_kinds=set(), ) def _build_check_specs( *, - dag: Mapping[str, Any], + plans: list[tuple[Mapping[str, Any], bool, list[Mapping[str, Any]], str | None]], translator: SqlBuildDagsterTranslator, - kinds: set[str] | None, - excluded_kinds: set[str], ) -> tuple[Any, ...]: - """Build Dagster check specs from SQLBuild checks matching optional kinds.""" - dg: Any = load_dagster() - nodes_by_id: dict[str, Mapping[str, Any]] = _nodes_by_id(dag) specs: list[Any] = [] - for check in dag["checks"]: - check_kind: str = str(check.get("kind")) - if not translator.is_asset_check(check): - continue - if check_kind in excluded_kinds: - continue - if kinds is not None and check_kind not in kinds: + for check, included, checked_nodes, description in plans: + if not translator.is_asset_check(check) or not included: continue - for asset_id in check.get("checked_asset_ids", ()): - node: Mapping[str, Any] | None = nodes_by_id.get(str(asset_id)) - if node is None: - continue + for node in checked_nodes: if not translator.is_asset_node(node): continue specs.append( dg.AssetCheckSpec( name=translator.get_check_name(check), asset=translator.get_asset_key(node), - description=str(check.get("path")) if check.get("path") is not None else None, + description=description, metadata=translator.get_check_metadata(check), ) ) return tuple(specs) - - -def _nodes_by_id(dag: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: - project_name: object = dag.get("project_name") - return {str(node["id"]): {**node, "project_name": project_name} for node in dag["nodes"]} - - -def _upstream_by_id(dag: Mapping[str, Any]) -> dict[str, list[str]]: - upstream: dict[str, list[str]] = {} - edge: Mapping[str, Any] - for edge in dag["edges"]: - upstream.setdefault(str(edge["to_id"]), []).append(str(edge["from_id"])) - return upstream - - -def _asset_spec_kinds(node: Mapping[str, Any]) -> set[str]: - kind: str = str(node.get("kind")) - if kind == MODEL_NODE_KIND: - materialization_type: str = str(node.get("materialization_type") or "table") - if materialization_type == VIEW_MATERIALIZATION_TYPE: - return {"sqlbuild", "view"} - return {"sqlbuild", "table"} - if kind in DAGSTER_DIRECT_KIND_NODE_KINDS: - return {"sqlbuild", kind} - return {"sqlbuild"} diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py index 8c35568cff..8121443831 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation_factory.py @@ -7,10 +7,13 @@ import subprocess from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any +from typing import Any, cast -from sqlbuild._native import dagster_cli_arguments -from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV +from sqlbuild._native import ( + INTEGRATION_RESULT_PATH_ENV, + INVOCATION_CONTEXT_ENV, + dagster_cli_arguments, +) from sqlbuild.integrations.dagster._helpers.invocation import ( _selected_asset_check_keys, _selected_asset_paths, @@ -19,7 +22,6 @@ dagster_invocation_context, ) from sqlbuild.integrations.dagster.classes.sqlbuild_cli_invocation import SqlBuildCliInvocation -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV def start_sqlbuild_cli_invocation( @@ -34,7 +36,7 @@ def start_sqlbuild_cli_invocation( """Start a SQLBuild CLI subprocess and return its invocation wrapper.""" planned: dict[str, Any] = dagster_cli_arguments( - args=list(args), + args=_cli_arguments(args), dag=dag, has_context=context is not None, read_selection=lambda: ( @@ -83,3 +85,19 @@ def start_sqlbuild_cli_invocation( ) invocation.execution_json_owned = execution_json_owned return invocation + + +def _cli_arguments(args: Sequence[object]) -> list[str]: + """Arguments as strings: path-like values pass through, others fail as `str` methods would.""" + + arguments: list[str] = [] + for argument in args: + if isinstance(argument, os.PathLike): + arguments.append(os.fsdecode(cast("os.PathLike[str]", argument))) + elif isinstance(argument, str): + arguments.append(argument) + else: + raise AttributeError( + f"{type(argument).__name__!r} object has no attribute 'startswith'" + ) + return arguments diff --git a/src/sqlbuild/integrations/dagster/_helpers/selection.py b/src/sqlbuild/integrations/dagster/_helpers/selection.py index 9f935ab8ec..a3926d2ed3 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/selection.py +++ b/src/sqlbuild/integrations/dagster/_helpers/selection.py @@ -5,12 +5,11 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlbuild._native import load_sqlbuild_dag, resolve_dag_selection +from sqlbuild._native import dagster_selected_asset_nodes, load_sqlbuild_dag from sqlbuild.integrations.dagster._helpers.imports import load_dagster from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( SqlBuildDagsterTranslator, ) -from sqlbuild.integrations.dagster.constants import DAGSTER_ASSET_NODE_KINDS from sqlbuild.integrations.dagster.exceptions import DagsterDagInputError from sqlbuild.integrations.dagster.types import SqlBuildDagInput @@ -29,20 +28,11 @@ def build_sqlbuild_asset_selection_impl( raise DagsterDagInputError("SQLBuild asset selection requires an asset definition") resolved_dag: Mapping[str, Any] = load_sqlbuild_dag(dag, error_class=DagsterDagInputError) resolved_translator: SqlBuildDagsterTranslator = translator or SqlBuildDagsterTranslator() - selected_ids: frozenset[str] = resolve_dag_selection( - dag=resolved_dag, - select=sqlbuild_select, - exclude=sqlbuild_exclude, - ) - asset_keys: list[Any] = [] - project_name: object = resolved_dag.get("project_name") - for node in resolved_dag.get("nodes", ()): - if str(node.get("id")) not in selected_ids: - continue - if str(node.get("kind")) not in DAGSTER_ASSET_NODE_KINDS: - continue - translated_node: Mapping[str, Any] = {**node, "project_name": project_name} - if not resolved_translator.is_asset_node(translated_node): - continue - asset_keys.append(resolved_translator.get_asset_key(translated_node)) + asset_keys: list[Any] = [ + resolved_translator.get_asset_key(node) + for node in dagster_selected_asset_nodes( + resolved_dag, select=sqlbuild_select, exclude=sqlbuild_exclude + ) + if resolved_translator.is_asset_node(node) + ] return load_dagster().AssetSelection.assets(*asset_keys) diff --git a/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py b/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py index 249462beb6..4cff960df9 100644 --- a/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py +++ b/src/sqlbuild/integrations/dagster/classes/sqlbuild_dagster_translator.py @@ -2,12 +2,19 @@ from __future__ import annotations -import re -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from typing import Any +from sqlbuild._native import ( + dagster_default_asset_key_path, + dagster_default_check_metadata, + dagster_default_check_name, + dagster_default_description, + dagster_default_group_name, + dagster_default_metadata, + dagster_default_tags, +) from sqlbuild.integrations.dagster._helpers.imports import load_dagster -from sqlbuild.integrations.dagster.constants import SOURCE_NODE_KIND class SqlBuildDagsterTranslator: @@ -15,10 +22,7 @@ class SqlBuildDagsterTranslator: def get_asset_key(self, node: Mapping[str, Any]) -> Any: dg: Any = load_dagster() - authored_asset_key: Sequence[object] | None = _authored_dagster_asset_key(node) - if authored_asset_key is not None: - return dg.AssetKey([str(part) for part in authored_asset_key]) - return dg.AssetKey([str(part) for part in node["asset_key"]]) + return dg.AssetKey(dagster_default_asset_key_path(node)) def is_asset_node(self, node: Mapping[str, Any]) -> bool: """Return whether SQLBuild owns this node as a Dagster asset.""" @@ -31,112 +35,19 @@ def is_asset_check(self, check: Mapping[str, Any]) -> bool: return True def get_group_name(self, node: Mapping[str, Any]) -> str | None: - group_name: str = str(node.get("group") or node.get("project_name") or "sqlbuild") - return group_name.replace("-", "_") + return dagster_default_group_name(node) def get_tags(self, node: Mapping[str, Any]) -> Mapping[str, str]: - tags: dict[str, str] = {"sqlbuild/kind": str(node.get("kind", ""))} - for tag in node.get("tags", ()): - tag_key: str = _normalize_tag_key(str(tag)) - if tag_key: - tags[tag_key] = "" - return tags + return dagster_default_tags(node) def get_metadata(self, node: Mapping[str, Any]) -> Mapping[str, Any]: - metadata: dict[str, Any] = { - "sqlbuild_id": node.get("id"), - "sqlbuild_name": node.get("name"), - "sqlbuild_kind": node.get("kind"), - "sqlbuild_selector": node.get("name"), - } - for key in ( - "path", - "target", - "description", - "sql", - "columns", - "column_lineage", - "group", - "language", - "materialization_type", - "return_kind", - "loader", - "meta", - ): - if key in node: - metadata[key] = node[key] - return metadata + return dagster_default_metadata(node) def get_description(self, node: Mapping[str, Any]) -> str | None: - description: str = str(node.get("description") or _fallback_description(node)) - sql: object = node.get("sql") - if isinstance(sql, str) and sql.strip(): - return f"{description}\n\n**SQLBuild SQL:**\n```sql\n{sql.strip()}\n```" - path: object = node.get("path") - if isinstance(path, str) and path: - return f"{description}\n\n**Source file:** {_markdown_code_span(path)}" - return description + return dagster_default_description(node) def get_check_name(self, check: Mapping[str, Any]) -> str: - parts: list[str] = [str(check.get("kind", "check")), str(check.get("name", "check"))] - if check.get("attached_column_name") is not None: - parts.append(str(check["attached_column_name"])) - elif check.get("attached_target_name") is not None: - parts.append(str(check["attached_target_name"])) - return "__".join(_normalize_name(part) for part in parts if part) + return dagster_default_check_name(check) def get_check_metadata(self, check: Mapping[str, Any]) -> Mapping[str, Any]: - return { - "sqlbuild_check_id": check.get("id"), - "sqlbuild_check_kind": check.get("kind"), - "sqlbuild_check_name": check.get("name"), - "sqlbuild_check_selector": check.get("id"), - } - - -def _normalize_tag_key(value: str) -> str: - normalized: str = re.sub(r"[^A-Za-z0-9_.\-/]+", "_", value).strip("_") - return normalized - - -def _authored_dagster_asset_key(node: Mapping[str, Any]) -> Sequence[object] | None: - if str(node.get("kind")) != SOURCE_NODE_KIND: - return None - meta: object = node.get("meta") - if not isinstance(meta, Mapping): - return None - dagster_meta: object = meta.get("dagster") - if not isinstance(dagster_meta, Mapping): - return None - asset_key: object = dagster_meta.get("asset_key") - if not isinstance(asset_key, Sequence) or isinstance(asset_key, (str, bytes)) or not asset_key: - return None - return asset_key - - -def _normalize_name(value: str) -> str: - normalized: str = re.sub(r"[^A-Za-z0-9_]+", "_", value).strip("_") - return normalized or "check" - - -def _markdown_code_span(value: str) -> str: - longest_backtick_run: int = max( - (len(match.group()) for match in re.finditer(r"`+", value)), default=0 - ) - delimiter: str = "`" * (longest_backtick_run + 1) - padding: str = " " if value.startswith(("`", " ")) or value.endswith(("`", " ")) else "" - return f"{delimiter}{padding}{value}{padding}{delimiter}" - - -def _fallback_description(node: Mapping[str, Any]) -> str: - kind: str = str(node.get("kind") or "asset").replace("_", " ") - name: str = str(node.get("name") or "unknown") - target: object = node.get("target") - qualified_name: str = "" - if isinstance(target, Mapping): - qualified_name = str(target.get("qualified_name") or "") - relation_suffix: str = f" at `{qualified_name}`" if qualified_name else "" - materialization: object = node.get("materialization_type") - if materialization is not None: - return f"SQLBuild {kind} `{name}` materialized as `{materialization}`{relation_suffix}." - return f"SQLBuild {kind} `{name}`{relation_suffix}." + return dagster_default_check_metadata(check) diff --git a/src/sqlbuild/integrations/dbt/__init__.py b/src/sqlbuild/integrations/dbt/__init__.py index 3767f64770..e69de29bb2 100644 --- a/src/sqlbuild/integrations/dbt/__init__.py +++ b/src/sqlbuild/integrations/dbt/__init__.py @@ -1 +0,0 @@ -"""dbt interop helpers.""" diff --git a/src/sqlbuild/integrations/dbt/_helpers/__init__.py b/src/sqlbuild/integrations/dbt/_helpers/__init__.py index 7195ee3e4d..e69de29bb2 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/__init__.py +++ b/src/sqlbuild/integrations/dbt/_helpers/__init__.py @@ -1 +0,0 @@ -"""dbt integration helper modules.""" diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/__init__.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/integrations/dbt/_helpers/profile/render.py b/src/sqlbuild/integrations/dbt/_helpers/profile/render.py index 4fe15fb4d3..97a519e8c9 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/profile/render.py +++ b/src/sqlbuild/integrations/dbt/_helpers/profile/render.py @@ -8,13 +8,15 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + ResolvedDbtProfileOutput, + SelectedDbtProfileOutput, + coerce_dbt_profile_bool, + coerce_dbt_profile_number, +) from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput, SelectedDbtProfileOutput -_DBT_FALSE_VALUES: frozenset[str] = frozenset({"false", "0", "no", "n", "off"}) _DBT_JINJA_MARKERS: tuple[str, ...] = ("{{", "{%", "{#") -_DBT_NUMBER_DECIMAL_SEPARATOR: str = "." -_DBT_TRUE_VALUES: frozenset[str] = frozenset({"true", "1", "yes", "y", "on"}) class _Missing: @@ -137,8 +139,8 @@ def var(name: str, default: object = _MISSING) -> object: globals_map["target"] = target environment.filters.update( { - "as_bool": _as_bool, - "as_number": _as_number, + "as_bool": coerce_dbt_profile_bool, + "as_number": coerce_dbt_profile_number, "as_native": _as_native, } ) @@ -152,33 +154,5 @@ def var(name: str, default: object = _MISSING) -> object: ) from error -def _as_bool(value: object) -> bool: - if isinstance(value, bool): - return value - if isinstance(value, str): - normalized: str = value.strip().lower() - if normalized in _DBT_TRUE_VALUES: - return True - if normalized in _DBT_FALSE_VALUES: - return False - raise DbtProfileError(f"Cannot coerce {value!r} to bool") - - -def _as_number(value: object) -> int | float: - if isinstance(value, bool): - raise DbtProfileError(f"Cannot coerce {value!r} to number") - if isinstance(value, int | float): - return value - if isinstance(value, str): - stripped: str = value.strip() - try: - if _DBT_NUMBER_DECIMAL_SEPARATOR in stripped: - return float(stripped) - return int(stripped) - except ValueError as error: - raise DbtProfileError(f"Cannot coerce {value!r} to number") from error - raise DbtProfileError(f"Cannot coerce {value!r} to number") - - def _as_native(value: object) -> object: return value diff --git a/src/sqlbuild/integrations/dbt/main/__init__.py b/src/sqlbuild/integrations/dbt/main/__init__.py deleted file mode 100644 index ea321f6ba0..0000000000 --- a/src/sqlbuild/integrations/dbt/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public dbt interop entrypoints.""" diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/__init__.py b/src/sqlbuild/integrations/dbt/main/pipeline/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py deleted file mode 100644 index 477de1ed86..0000000000 --- a/src/sqlbuild/integrations/dbt/models.py +++ /dev/null @@ -1,33 +0,0 @@ -"""dbt interop domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class SelectedDbtProfileOutput: - """Selected dbt profile target output before rendering.""" - - profile_name: str - target_name: str - output: dict[str, object] - - -@dataclass(frozen=True) -class ResolvedDbtProfileOutput: - """Rendered dbt profile target output.""" - - project_dir: Path - profiles_dir: Path - profile_name: str - target_name: str - output: dict[str, object] - - @property - def adapter_type(self) -> str: - """Return the dbt adapter type from the rendered output.""" - - value: object | None = self.output.get("type") - return value if isinstance(value, str) else "" diff --git a/src/sqlbuild/integrations/dbt/types.py b/src/sqlbuild/integrations/dbt/types.py deleted file mode 100644 index dc4da7b53d..0000000000 --- a/src/sqlbuild/integrations/dbt/types.py +++ /dev/null @@ -1,12 +0,0 @@ -"""dbt integration type aliases.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Protocol - - -class DbtInvoker(Protocol): - """Callable dbt process invocation contract.""" - - def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... diff --git a/src/sqlbuild/integrations/dlt/_helpers/destination.py b/src/sqlbuild/integrations/dlt/_helpers/destination.py index 9c875dee7b..3553745ec4 100644 --- a/src/sqlbuild/integrations/dlt/_helpers/destination.py +++ b/src/sqlbuild/integrations/dlt/_helpers/destination.py @@ -2,17 +2,10 @@ from __future__ import annotations -from typing import Any, cast - -from sqlbuild.adapter.contract.types import BuiltinAdapter +from sqlbuild._native import dlt_destination_arguments from sqlbuild.integrations.dlt.exceptions import DltIntegrationError from sqlbuild.integrations.dlt.models import DltDestinationConfig -_DEFAULT_DESTINATION_OPTIONS: dict[str, object] = {"naming_convention": "sql_ci_v1"} -_SQLBUILD_OWNED_DESTINATION_KEYS: frozenset[str] = frozenset( - {"credentials", "dataset_name", "default_schema_name", "project_id"} -) - def build_dlt_destination( *, @@ -31,199 +24,13 @@ def build_dlt_destination( "pip install 'sqlbuild[dlt]'" ) from error - destination_options: dict[str, object] = _destination_options( - adapter_name=adapter_name, destination_config=destination_config or {} - ) - if adapter_name == BuiltinAdapter.DUCKDB.value: - return DltDestinationConfig( - destination=cast(Any, destinations.duckdb)( - credentials=_required_string( - config=connection_config, - key="database", - adapter_name=BuiltinAdapter.DUCKDB.value, - ), - **destination_options, - ), - dataset_name=dataset_name or "main", - ) - _require_dataset_name(adapter_name=adapter_name, dataset_name=dataset_name) - if adapter_name == BuiltinAdapter.MOTHERDUCK.value: - database: str = _optional_string(config=connection_config, key="database") or "" - token: str | None = _optional_string(config=connection_config, key="token") - credentials: str = database if database.startswith("md:") else f"md:{database}" - if token is not None: - credentials = f"{credentials}?motherduck_token={token}" - return DltDestinationConfig( - destination=cast(Any, destinations.motherduck)( - credentials=credentials, **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.POSTGRES.value: - return DltDestinationConfig( - destination=cast(Any, destinations.postgres)( - credentials=_postgres_connection_string(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.SNOWFLAKE.value: - return DltDestinationConfig( - destination=cast(Any, destinations.snowflake)( - credentials=_snowflake_credentials(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.BIGQUERY.value: - return DltDestinationConfig( - destination=cast(Any, destinations.bigquery)( - **_bigquery_config(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.DATABRICKS.value: - return DltDestinationConfig( - destination=cast(Any, destinations.databricks)( - credentials=_databricks_credentials(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - if adapter_name == BuiltinAdapter.SQLSERVER.value: - return DltDestinationConfig( - destination=cast(Any, destinations.mssql)( - credentials=_sqlserver_credentials(connection_config), **destination_options - ), - dataset_name=dataset_name, - ) - raise DltIntegrationError(f"Adapter '{adapter_name}' does not support dlt integration loaders") - - -def _destination_options( - *, adapter_name: str, destination_config: dict[str, object] -) -> dict[str, object]: - blocked_keys: tuple[str, ...] = tuple( - key for key in destination_config if key in _SQLBUILD_OWNED_DESTINATION_KEYS + factory, arguments, options, resolved_dataset_name = dlt_destination_arguments( + adapter_name=adapter_name, + connection_config=connection_config, + destination_config=destination_config, + dataset_name=dataset_name, ) - if blocked_keys: - blocked: str = ", ".join(sorted(blocked_keys)) - raise DltIntegrationError( - f"Adapter '{adapter_name}' dlt destination config cannot define SQLBuild-owned key(s): " - f"{blocked}" - ) - return {**_DEFAULT_DESTINATION_OPTIONS, **destination_config} - - -def _postgres_connection_string(config: dict[str, object]) -> str: - host: str = _optional_string(config=config, key="host") or "localhost" - port: object = config.get("port", 5432) - user: str = _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.POSTGRES.value + return DltDestinationConfig( + destination=getattr(destinations, factory)(**arguments, **options), + dataset_name=resolved_dataset_name, ) - password: str = _optional_string(config=config, key="password") or "" - database: str = _optional_string(config=config, key="dbname") or _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.POSTGRES.value - ) - return f"postgresql://{user}:{password}@{host}:{port}/{database}" - - -def _snowflake_credentials(config: dict[str, object]) -> dict[str, object]: - account: str = _required_string( - config=config, key="account", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - credentials: dict[str, object] = { - "account": _snowflake_account(account), - "host": _snowflake_account(_optional_string(config=config, key="host") or account), - "user": _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ), - "database": _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ), - "warehouse": _required_string( - config=config, key="warehouse", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ), - } - credentials = _copy_optional(source=config, destination=credentials, key="password") - credentials = _copy_optional(source=config, destination=credentials, key="role") - credentials = _copy_optional(source=config, destination=credentials, key="authenticator") - credentials = _copy_optional(source=config, destination=credentials, key="token") - return credentials - - -def _bigquery_config(config: dict[str, object]) -> dict[str, object]: - options: dict[str, object] = {} - project_id: str | None = _optional_string(config=config, key="project") - location: str | None = _optional_string(config=config, key="location") - if project_id is not None: - options["project_id"] = project_id - if location is not None: - options["location"] = location - return options - - -def _snowflake_account(account: str) -> str: - return account.removesuffix(".snowflakecomputing.com") - - -def _databricks_credentials(config: dict[str, object]) -> dict[str, object]: - credentials: dict[str, object] = { - "server_hostname": _required_string( - config=config, key="server_hostname", adapter_name=BuiltinAdapter.DATABRICKS.value - ), - "http_path": _required_string( - config=config, key="http_path", adapter_name=BuiltinAdapter.DATABRICKS.value - ), - "access_token": _required_string( - config=config, key="token", adapter_name=BuiltinAdapter.DATABRICKS.value - ), - } - catalog: str | None = _optional_string(config=config, key="catalog") - if catalog is not None: - credentials["catalog"] = catalog - return credentials - - -def _sqlserver_credentials(config: dict[str, object]) -> dict[str, object]: - return { - "host": _optional_string(config=config, key="host") or "localhost", - "port": config.get("port", 1433), - "database": _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.SQLSERVER.value - ), - "username": _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.SQLSERVER.value - ), - "password": _optional_string(config=config, key="password") or "", - "query": {"TrustServerCertificate": "yes"}, - } - - -def _require_dataset_name(*, adapter_name: str, dataset_name: str | None) -> None: - if not isinstance(dataset_name, str) or not dataset_name.strip(): - raise DltIntegrationError( - f"Adapter '{adapter_name}' requires an explicit dlt source schema for raw landing" - ) - - -def _copy_optional( - *, source: dict[str, object], destination: dict[str, object], key: str -) -> dict[str, object]: - value: str | None = _optional_string(config=source, key=key) - if value is not None: - destination[key] = value - return destination - - -def _required_string(*, config: dict[str, object], key: str, adapter_name: str) -> str: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - raise DltIntegrationError( - f"Adapter '{adapter_name}' requires connection.{key} for dlt integration loaders" - ) - return value.strip() - - -def _optional_string(*, config: dict[str, object], key: str) -> str | None: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - return None - return value.strip() diff --git a/src/sqlbuild/integrations/dlt/main/_loaders.py b/src/sqlbuild/integrations/dlt/main/_loaders.py index df463458d3..8c8bc80459 100644 --- a/src/sqlbuild/integrations/dlt/main/_loaders.py +++ b/src/sqlbuild/integrations/dlt/main/_loaders.py @@ -4,10 +4,10 @@ from collections.abc import Callable +from sqlbuild._native import SourceEntry from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.dlt._helpers.runner import run_dlt_source from sqlbuild.integrations.dlt.models import DltSourceConfig -from sqlbuild.spec.contracts.models import SourceEntry def dlt_loader_function(*, source_entry: SourceEntry, name: str) -> Callable[[LoaderContext], None]: diff --git a/src/sqlbuild/integrations/ingestr/_helpers/command.py b/src/sqlbuild/integrations/ingestr/_helpers/command.py deleted file mode 100644 index e9d0d733a7..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/command.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Command construction helpers for ingestr integration loaders.""" - -from __future__ import annotations - -from collections.abc import Iterable, Mapping -from dataclasses import replace -from typing import cast - -from sqlbuild._native import expand_template_data -from sqlbuild.integrations.ingestr._helpers.destination import build_destination_uri -from sqlbuild.integrations.ingestr.constants import INGESTR_INTEGRATION_KIND -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError -from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry - - -def resolve_ingestr_config( - *, config: IngestrSourceConfig, vars: dict[str, object], environment: str | None, run_id: str -) -> IngestrSourceConfig: - """Resolve supported templates in ingestr source config values.""" - - values: object = expand_template_data( - value={ - "source_uri": config.source_uri, - "source_table": config.source_table, - "strategy": config.strategy, - "incremental_key": config.incremental_key, - "primary_key": list(config.primary_key), - "columns": config.columns, - "extra_args": list(config.extra_args), - }, - variables=vars, - context_values={"environment": environment, "run_id": run_id}, - context_label="ingestr source config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - ) - if not isinstance(values, Mapping): - return config - resolved_values: Mapping[str, object] = cast(Mapping[str, object], values) - return replace( - config, - source_uri=str(resolved_values["source_uri"]), - source_table=str(resolved_values["source_table"]), - strategy=_optional_str(resolved_values.get("strategy")), - incremental_key=_optional_str(resolved_values.get("incremental_key")), - primary_key=_string_tuple(resolved_values.get("primary_key")), - columns=_optional_str(resolved_values.get("columns")), - extra_args=_string_tuple(resolved_values.get("extra_args")), - ) - - -def build_ingestr_command( - *, - source_entry: SourceEntry, - adapter_name: str, - connection_config: dict[str, object], - destination_table: str, - vars: dict[str, object], - environment: str | None, - run_id: str, - is_reload: bool, -) -> tuple[str, ...]: - """Build the ingestr CLI command for one source entry.""" - - integration_loader: IntegrationLoaderConfig | None = source_entry.integration_loader - if integration_loader is None or integration_loader.kind != INGESTR_INTEGRATION_KIND: - raise IngestrIntegrationError( - f"Source '{source_entry.name}' does not define ingestr config" - ) - if not isinstance(integration_loader.config, IngestrSourceConfig): - raise IngestrIntegrationError( - f"Source '{source_entry.name}' defines invalid ingestr config" - ) - config: IngestrSourceConfig = resolve_ingestr_config( - config=integration_loader.config, - vars=vars, - environment=environment, - run_id=run_id, - ) - args: list[str] = [ - "ingestr", - "ingest", - "--source-uri", - config.source_uri, - "--source-table", - config.source_table, - "--dest-uri", - build_destination_uri(adapter_name=adapter_name, connection_config=connection_config), - "--dest-table", - destination_table, - "--yes", - "--progress", - "log", - ] - if config.strategy is not None: - args.extend(("--incremental-strategy", config.strategy)) - if config.incremental_key is not None: - args.extend(("--incremental-key", config.incremental_key)) - for primary_key in config.primary_key: - args.extend(("--primary-key", primary_key)) - if config.columns is not None: - args.extend(("--columns", config.columns)) - if is_reload: - args.append("--full-refresh") - args.extend(config.extra_args) - return tuple(args) - - -def _optional_str(value: object | None) -> str | None: - if value is None: - return None - text: str = str(value) - return text if text else None - - -def _string_tuple(value: object | None) -> tuple[str, ...]: - if value is None: - return () - if isinstance(value, str): - return (value,) - if isinstance(value, Iterable): - return tuple(str(item) for item in value) - return (str(value),) diff --git a/src/sqlbuild/integrations/ingestr/_helpers/destination.py b/src/sqlbuild/integrations/ingestr/_helpers/destination.py deleted file mode 100644 index 5ccdb6113c..0000000000 --- a/src/sqlbuild/integrations/ingestr/_helpers/destination.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Build ingestr destination URIs from SQLBuild adapter connection config.""" - -from __future__ import annotations - -from urllib.parse import quote, urlencode - -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.integrations.ingestr.constants import ( - SQLSERVER_DRIVER_PARAMETER, - SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER, -) -from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError - - -def build_destination_uri(*, adapter_name: str, connection_config: dict[str, object]) -> str: - """Return the ingestr destination URI for a SQLBuild connection.""" - - if adapter_name == BuiltinAdapter.DUCKDB.value: - return _duckdb_uri(connection_config) - if adapter_name == BuiltinAdapter.MOTHERDUCK.value: - return _motherduck_uri(connection_config) - if adapter_name == BuiltinAdapter.POSTGRES.value: - return _postgres_uri(connection_config) - if adapter_name == BuiltinAdapter.BIGQUERY.value: - return _bigquery_uri(connection_config) - if adapter_name == BuiltinAdapter.SNOWFLAKE.value: - return _snowflake_uri(connection_config) - if adapter_name == BuiltinAdapter.DATABRICKS.value: - return _databricks_uri(connection_config) - if adapter_name == BuiltinAdapter.SQLSERVER.value: - return _sqlserver_uri(connection_config) - raise IngestrIntegrationError( - f"Adapter '{adapter_name}' does not support ingestr integration loaders" - ) - - -def _required_string(*, config: dict[str, object], key: str, adapter_name: str) -> str: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - raise IngestrIntegrationError( - f"Adapter '{adapter_name}' requires connection.{key} for ingestr integration loaders" - ) - return value.strip() - - -def _optional_string(*, config: dict[str, object], key: str) -> str | None: - value: object | None = config.get(key) - if not isinstance(value, str) or not value.strip(): - return None - return value.strip() - - -def _duckdb_uri(config: dict[str, object]) -> str: - database: str = _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.DUCKDB.value - ) - return f"duckdb:///{database}" - - -def _motherduck_uri(config: dict[str, object]) -> str: - database: str = _optional_string(config=config, key="database") or "" - if database.startswith("md:"): - database = database.removeprefix("md:") - token: str | None = _optional_string(config=config, key="token") - query: str = f"?{urlencode({'token': token})}" if token else "" - return f"motherduck://{quote(database)}{query}" - - -def _postgres_uri(config: dict[str, object]) -> str: - host: str = _optional_string(config=config, key="host") or "localhost" - port: object = config.get("port", 5432) - user: str = _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.POSTGRES.value - ) - password: str = _optional_string(config=config, key="password") or "" - database: str = _optional_string(config=config, key="dbname") or _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.POSTGRES.value - ) - sslmode: str | None = _optional_string(config=config, key="sslmode") - query: str = f"?{urlencode({'sslmode': sslmode})}" if sslmode else "" - return f"postgresql://{quote(user)}:{quote(password)}@{host}:{port}/{quote(database)}{query}" - - -def _bigquery_uri(config: dict[str, object]) -> str: - project: str = _optional_string(config=config, key="project") or _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.BIGQUERY.value - ) - params: dict[str, str] = {} - for key in ("location", "credentials_path", "credentials_base64"): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - query: str = f"?{urlencode(params)}" if params else "" - return f"bigquery://{project}{query}" - - -def _snowflake_uri(config: dict[str, object]) -> str: - account: str = _required_string( - config=config, key="account", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - user: str = _required_string( - config=config, key="user", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - password: str = _optional_string(config=config, key="password") or "" - database: str = _required_string( - config=config, key="database", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - schema: str = _required_string( - config=config, key="schema", adapter_name=BuiltinAdapter.SNOWFLAKE.value - ) - params: dict[str, str] = {} - for key in ( - "warehouse", - "role", - "token", - "private_key", - "private_key_passphrase", - "authenticator", - ): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - query: str = f"?{urlencode(params)}" if params else "" - return ( - f"snowflake://{quote(user)}:{quote(password)}@{account}/" - f"{quote(database)}/{quote(schema)}{query}" - ) - - -def _databricks_uri(config: dict[str, object]) -> str: - host: str = _required_string( - config=config, key="server_hostname", adapter_name=BuiltinAdapter.DATABRICKS.value - ) - token: str = _required_string( - config=config, key="token", adapter_name=BuiltinAdapter.DATABRICKS.value - ) - params: dict[str, str] = { - "http_path": _required_string( - config=config, key="http_path", adapter_name=BuiltinAdapter.DATABRICKS.value - ) - } - for key in ("catalog", "schema"): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - return f"databricks://token:{quote(token)}@{host}?{urlencode(params)}" - - -def _sqlserver_uri(config: dict[str, object]) -> str: - host: str = ( - _optional_string(config=config, key="host") - or _optional_string(config=config, key="server") - or "localhost" - ) - port: object = config.get("port", 1433) - user: str = ( - _optional_string(config=config, key="user") - or _optional_string(config=config, key="username") - or "sa" - ) - password: str = _optional_string(config=config, key="password") or "" - database: str = ( - _optional_string(config=config, key="database") - or _optional_string(config=config, key="dbname") - or "master" - ) - params: dict[str, str] = {} - for key in ("driver", "TrustServerCertificate", "Authentication"): - value: str | None = _optional_string(config=config, key=key) - if value is not None: - params[key] = value - if SQLSERVER_DRIVER_PARAMETER not in params: - params[SQLSERVER_DRIVER_PARAMETER] = "ODBC Driver 18 for SQL Server" - if SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER not in params: - params[SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER] = "yes" - query: str = f"?{urlencode(params)}" if params else "" - return f"mssql://{quote(user)}:{quote(password)}@{host}:{port}/{quote(database)}{query}" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/output.py b/src/sqlbuild/integrations/ingestr/_helpers/output.py index a9eed714b8..4c7264204a 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/output.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/output.py @@ -2,18 +2,10 @@ from __future__ import annotations -from shlex import quote - from sqlbuild.executor.load.models import LoaderContext from sqlbuild.integrations.ingestr.models import IngestrCommandResult -def format_ingestr_command(command: tuple[str, ...]) -> str: - """Return a display-safe ingestr command line.""" - - return " ".join(quote(part) for part in command) - - def record_ingestr_output(*, ctx: LoaderContext, result: IngestrCommandResult) -> None: """Attach captured ingestr output to the loader lifecycle.""" diff --git a/src/sqlbuild/integrations/ingestr/_helpers/runner.py b/src/sqlbuild/integrations/ingestr/_helpers/runner.py index 24cd839a74..1f33bc29fa 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/runner.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/runner.py @@ -8,7 +8,7 @@ from os import environ from pathlib import Path -from sqlbuild.integrations.ingestr._helpers.output import format_ingestr_command +from sqlbuild._native import format_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrCommandResult from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle diff --git a/src/sqlbuild/integrations/ingestr/constants.py b/src/sqlbuild/integrations/ingestr/constants.py index 80ead410bc..47b3f3f671 100644 --- a/src/sqlbuild/integrations/ingestr/constants.py +++ b/src/sqlbuild/integrations/ingestr/constants.py @@ -3,5 +3,3 @@ from __future__ import annotations INGESTR_INTEGRATION_KIND: str = "ingestr" -SQLSERVER_DRIVER_PARAMETER: str = "driver" -SQLSERVER_TRUST_SERVER_CERTIFICATE_PARAMETER: str = "TrustServerCertificate" diff --git a/src/sqlbuild/integrations/ingestr/main/_loaders.py b/src/sqlbuild/integrations/ingestr/main/_loaders.py index 916dc95fff..d5704629c2 100644 --- a/src/sqlbuild/integrations/ingestr/main/_loaders.py +++ b/src/sqlbuild/integrations/ingestr/main/_loaders.py @@ -4,13 +4,12 @@ from collections.abc import Callable +from sqlbuild._native import SourceEntry, build_ingestr_command from sqlbuild.executor.load.models import LoaderContext -from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command from sqlbuild.integrations.ingestr._helpers.output import record_ingestr_output from sqlbuild.integrations.ingestr._helpers.runner import run_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrCommandResult -from sqlbuild.spec.contracts.models import SourceEntry def ingestr_loader_function( @@ -23,9 +22,7 @@ def run(ctx: LoaderContext) -> None: source_entry=source_entry, adapter_name=ctx.adapter.adapter_name, connection_config=ctx.connection_config, - destination_table=_destination_table( - source_entry=source_entry, target_name=ctx.destination_name - ), + destination_name=ctx.destination_name, vars=ctx.vars, environment=ctx.target, run_id=ctx.run_id, @@ -42,9 +39,3 @@ def run(ctx: LoaderContext) -> None: run.__name__ = name return run - - -def _destination_table(*, source_entry: SourceEntry, target_name: str) -> str: - if source_entry.schema is not None: - return f"{source_entry.schema}.{target_name}" - return target_name diff --git a/src/sqlbuild/integrations/rivers/_helpers/assets.py b/src/sqlbuild/integrations/rivers/_helpers/assets.py index 29087fd7e8..c3fbdad830 100644 --- a/src/sqlbuild/integrations/rivers/_helpers/assets.py +++ b/src/sqlbuild/integrations/rivers/_helpers/assets.py @@ -5,11 +5,11 @@ from collections.abc import Mapping from typing import Any +from sqlbuild._native import rivers_asset_plans from sqlbuild.integrations.rivers._helpers.imports import load_rivers from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( SqlBuildRiversTranslator, ) -from sqlbuild.integrations.rivers.constants import RIVERS_ASSET_NODE_KINDS def build_asset_defs( @@ -18,23 +18,12 @@ def build_asset_defs( """Build materializable Rivers asset definitions from a SQLBuild DAG artifact.""" rs: Any = load_rivers() - nodes_by_id: dict[str, Mapping[str, Any]] = _nodes_by_id(dag) - upstream_by_id: dict[str, list[str]] = _upstream_by_id(dag) - project_name: object = dag.get("project_name") asset_defs: list[Any] = [] - for node in dag["nodes"]: - if str(node.get("kind")) not in RIVERS_ASSET_NODE_KINDS: - continue - translated_node: Mapping[str, Any] = {**node, "project_name": project_name} - deps: list[Any] = [] - for upstream_id in upstream_by_id.get(str(node["id"]), []): - upstream_node: Mapping[str, Any] | None = nodes_by_id.get(upstream_id) - if ( - upstream_node is None - or str(upstream_node.get("kind")) not in RIVERS_ASSET_NODE_KINDS - ): - continue - deps.append(rs.AssetDef.dep(translator.get_asset_name(upstream_node))) + for translated_node, upstream_nodes in rivers_asset_plans(dag): + deps: list[Any] = [ + rs.AssetDef.dep(translator.get_asset_name(upstream_node)) + for upstream_node in upstream_nodes + ] asset_defs.append( rs.AssetDef( name=translator.get_asset_name(translated_node), @@ -46,15 +35,3 @@ def build_asset_defs( ) ) return tuple(asset_defs) - - -def _nodes_by_id(dag: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: - return {str(node["id"]): node for node in dag["nodes"]} - - -def _upstream_by_id(dag: Mapping[str, Any]) -> dict[str, list[str]]: - upstream: dict[str, list[str]] = {} - edge: Mapping[str, Any] - for edge in dag["edges"]: - upstream.setdefault(str(edge["to_id"]), []).append(str(edge["from_id"])) - return upstream diff --git a/src/sqlbuild/integrations/rivers/constants.py b/src/sqlbuild/integrations/rivers/constants.py index d2f5ad3451..cf147dd8fd 100644 --- a/src/sqlbuild/integrations/rivers/constants.py +++ b/src/sqlbuild/integrations/rivers/constants.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import MaterializationType from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.planner.types import MaterializationType RIVERS_DEPLOYMENT_ENVIRONMENT_VARIABLE: str = "RIVERS_DEPLOYMENT" RIVERS_DEVELOPMENT_DEPLOYMENT: str = "dev" diff --git a/src/sqlbuild/lint/__init__.py b/src/sqlbuild/lint/__init__.py index e06fdd147e..e69de29bb2 100644 --- a/src/sqlbuild/lint/__init__.py +++ b/src/sqlbuild/lint/__init__.py @@ -1 +0,0 @@ -"""Private SQL analysis and source-formatting implementation.""" diff --git a/src/sqlbuild/loaders.pyi b/src/sqlbuild/loaders.pyi index 977f5e3c38..4ca3937bce 100644 --- a/src/sqlbuild/loaders.pyi +++ b/src/sqlbuild/loaders.pyi @@ -1,9 +1,9 @@ from collections.abc import Callable, Sequence from typing import overload +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes.models import LoaderDefinition as LoaderDefinition from sqlbuild.python_nodes.types import LoaderColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry @overload def loader(function: Callable[..., object]) -> Callable[..., object]: ... diff --git a/src/sqlbuild/microbatches/_helpers/projection.py b/src/sqlbuild/microbatches/_helpers/projection.py deleted file mode 100644 index 26b2a14173..0000000000 --- a/src/sqlbuild/microbatches/_helpers/projection.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Internal interval algebra for append-only microbatch history.""" - -from __future__ import annotations - -import heapq -from datetime import datetime - -from sqlbuild._native import cursor_compare as compare -from sqlbuild._native import cursor_max_bound as max_bound -from sqlbuild._native import cursor_min_bound as min_bound -from sqlbuild._native import cursor_parse as parse -from sqlbuild._native import cursor_sort_key -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, - ProjectedMicrobatchInterval, - ReplayRequirementProjection, -) -from sqlbuild.microbatches.types import ( - MicrobatchFingerprintStatus, - MicrobatchRecordType, - ReplayRequirementState, -) - - -def project_microbatch_coverage( - *, - events: tuple[MicrobatchEvent, ...], - expected_intervals: tuple[MicrobatchInterval, ...], - cursor_type: str, -) -> MicrobatchCoverageProjection: - """Fold completion facts and classify uncovered expected intervals.""" - - completions: tuple[MicrobatchEvent, ...] = tuple( - event - for event in events - if event.record_type - in { - MicrobatchRecordType.PARTITION_COMPLETION, - MicrobatchRecordType.SYNTHETIC_COMPLETION, - } - and event.partition_start is not None - and event.partition_end is not None - ) - boundaries: set[str] = set() - for event in completions: - if event.partition_start is not None: - boundaries.add(event.partition_start) - if event.partition_end is not None: - boundaries.add(event.partition_end) - boundaries.update(interval.start for interval in expected_intervals) - boundaries.update(interval.end for interval in expected_intervals) - ordered_boundaries: tuple[str, ...] = tuple( - sorted(boundaries, key=lambda value: cursor_sort_key(raw=value, cursor_type=cursor_type)) - ) - projected: list[ProjectedMicrobatchInterval] = [] - for start, end, latest in _latest_completion_segments( - completions=completions, - ordered_boundaries=ordered_boundaries, - cursor_type=cursor_type, - ): - projected.append( - ProjectedMicrobatchInterval( - start=start, - end=end, - fingerprint_status=latest.fingerprint_status, - model_version_hash=latest.model_version_hash, - record_type=latest.record_type, - completion_type=latest.completion_type, - event_id=latest.event_id, - ) - ) - - accounted: list[MicrobatchInterval] = [ - MicrobatchInterval(start=interval.start, end=interval.end) for interval in projected - ] - contiguous_frontier: str | None = _minimum_bound( - values=tuple(interval.start for interval in expected_intervals), - cursor_type=cursor_type, - ) - if accounted and contiguous_frontier is not None: - ordered_accounted: list[MicrobatchInterval] = sorted( - accounted, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - for interval in ordered_accounted: - if _lte(left=interval.start, right=contiguous_frontier, cursor_type=cursor_type): - if _lt(left=contiguous_frontier, right=interval.end, cursor_type=cursor_type): - contiguous_frontier = interval.end - else: - break - - latest_ordinary_end: str | None = _maximum_bound( - values=tuple( - event.partition_end or "" - for event in completions - if event.record_type == MicrobatchRecordType.PARTITION_COMPLETION - ), - cursor_type=cursor_type, - ) - known_missing, unaccounted = _classify_uncovered_expected_intervals( - expected_intervals=expected_intervals, - accounted=tuple(accounted), - latest_ordinary_end=latest_ordinary_end, - cursor_type=cursor_type, - ) - - return MicrobatchCoverageProjection( - intervals=tuple(projected), - contiguous_frontier=contiguous_frontier, - known_missing=tuple(known_missing), - unaccounted=tuple(unaccounted), - unknown_fingerprints=tuple( - MicrobatchInterval(start=interval.start, end=interval.end) - for interval in projected - if interval.fingerprint_status == MicrobatchFingerprintStatus.UNKNOWN - ), - ) - - -def _latest_completion_segments( - *, - completions: tuple[MicrobatchEvent, ...], - ordered_boundaries: tuple[str, ...], - cursor_type: str, -) -> tuple[tuple[str, str, MicrobatchEvent], ...]: - """Sweep interval boundaries while retaining the latest active completion.""" - - latest_order: tuple[MicrobatchEvent, ...] = tuple( - sorted( - completions, - key=lambda event: (_created_sort_value(event.created_at), event.event_id), - ) - ) - latest_rank_by_identity: dict[int, int] = { - id(event): rank for rank, event in enumerate(latest_order) - } - by_start: tuple[MicrobatchEvent, ...] = tuple( - sorted( - completions, - key=lambda event: cursor_sort_key( - raw=event.partition_start or "", cursor_type=cursor_type - ), - ) - ) - active: list[tuple[int, MicrobatchEvent]] = [] - next_event_index: int = 0 - segments: list[tuple[str, str, MicrobatchEvent]] = [] - for boundary_index in range(len(ordered_boundaries) - 1): - start: str = ordered_boundaries[boundary_index] - end: str = ordered_boundaries[boundary_index + 1] - while next_event_index < len(by_start) and _lte( - left=by_start[next_event_index].partition_start or "", - right=start, - cursor_type=cursor_type, - ): - event: MicrobatchEvent = by_start[next_event_index] - heapq.heappush(active, (-latest_rank_by_identity[id(event)], event)) - next_event_index += 1 - while active and not _lte( - left=end, - right=active[0][1].partition_end or "", - cursor_type=cursor_type, - ): - heapq.heappop(active) - if active: - segments.append((start, end, active[0][1])) - return tuple(segments) - - -def _created_sort_value(value: datetime) -> float: - return value.timestamp() - - -def _classify_uncovered_expected_intervals( - *, - expected_intervals: tuple[MicrobatchInterval, ...], - accounted: tuple[MicrobatchInterval, ...], - latest_ordinary_end: str | None, - cursor_type: str, -) -> tuple[tuple[MicrobatchInterval, ...], tuple[MicrobatchInterval, ...]]: - """Classify sorted expected intervals without rescanning prior coverage.""" - - ordered_expected: tuple[MicrobatchInterval, ...] = tuple( - sorted( - expected_intervals, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - ) - ordered_accounted: tuple[MicrobatchInterval, ...] = tuple( - sorted( - accounted, - key=lambda interval: cursor_sort_key(raw=interval.start, cursor_type=cursor_type), - ) - ) - known_missing: list[MicrobatchInterval] = [] - unaccounted: list[MicrobatchInterval] = [] - accounted_index: int = 0 - for expected in ordered_expected: - while accounted_index < len(ordered_accounted) and _lte( - left=ordered_accounted[accounted_index].end, - right=expected.start, - cursor_type=cursor_type, - ): - accounted_index += 1 - frontier: str = expected.start - scan_index: int = accounted_index - gaps: list[MicrobatchInterval] = [] - while scan_index < len(ordered_accounted): - interval: MicrobatchInterval = ordered_accounted[scan_index] - if not _lt(left=interval.start, right=expected.end, cursor_type=cursor_type): - break - if _lt(left=frontier, right=interval.start, cursor_type=cursor_type): - gaps.append(MicrobatchInterval(start=frontier, end=interval.start)) - if _lt(left=frontier, right=interval.end, cursor_type=cursor_type): - frontier = interval.end - if _lte(left=expected.end, right=frontier, cursor_type=cursor_type): - break - scan_index += 1 - if _lt(left=frontier, right=expected.end, cursor_type=cursor_type): - gaps.append(MicrobatchInterval(start=frontier, end=expected.end)) - for gap in gaps: - has_accounted_before: bool = accounted_index > 0 or _lt( - left=expected.start, right=gap.start, cursor_type=cursor_type - ) - if ( - has_accounted_before - and latest_ordinary_end is not None - and _lte( - left=gap.end, - right=latest_ordinary_end, - cursor_type=cursor_type, - ) - ): - known_missing.append(gap) - else: - unaccounted.append(gap) - return tuple(known_missing), tuple(unaccounted) - - -def project_replay_requirement( - *, - requirement: MicrobatchEvent, - current_model_version_hash: str, - expected_intervals: tuple[MicrobatchInterval, ...], - coverage: MicrobatchCoverageProjection, - cursor_type: str, -) -> ReplayRequirementProjection: - """Derive replay completion without writing mutable requirement status.""" - - if requirement.required_model_version_hash != current_model_version_hash: - return ReplayRequirementProjection( - requirement=requirement, - state=ReplayRequirementState.SUPERSEDED, - ) - missing: list[MicrobatchInterval] = [] - unknown: list[MicrobatchInterval] = [] - for expected in expected_intervals: - pieces: tuple[ProjectedMicrobatchInterval, ...] = tuple( - interval - for interval in coverage.intervals - if _lt(left=interval.start, right=expected.end, cursor_type=cursor_type) - and _lt(left=expected.start, right=interval.end, cursor_type=cursor_type) - ) - if not _is_fully_covered( - expected=expected, - accounted=tuple( - MicrobatchInterval(start=piece.start, end=piece.end) for piece in pieces - ), - cursor_type=cursor_type, - ): - missing.append(expected) - continue - if any(piece.fingerprint_status == MicrobatchFingerprintStatus.UNKNOWN for piece in pieces): - unknown.append(expected) - continue - if any(piece.model_version_hash != current_model_version_hash for piece in pieces): - missing.append(expected) - state: ReplayRequirementState - if missing: - state = ReplayRequirementState.INCOMPLETE - elif unknown: - state = ReplayRequirementState.COMPLETE_WITH_UNKNOWN_FINGERPRINTS - else: - state = ReplayRequirementState.VERIFIED_COMPLETE - return ReplayRequirementProjection( - requirement=requirement, - state=state, - missing=tuple(missing), - unknown_fingerprints=tuple(unknown), - ) - - -def latest_active_replay_requirement( - *, events: tuple[MicrobatchEvent, ...], current_model_version_hash: str -) -> MicrobatchEvent | None: - """Return only the latest requirement created for the currently compiled version.""" - - requirements: tuple[MicrobatchEvent, ...] = tuple( - event - for event in events - if event.record_type == MicrobatchRecordType.REPLAY_REQUIREMENT - and event.required_model_version_hash == current_model_version_hash - ) - if not requirements: - return None - return max(requirements, key=lambda event: (event.created_at, event.event_id)) - - -def _is_fully_covered( - *, - expected: MicrobatchInterval, - accounted: tuple[MicrobatchInterval, ...] | list[MicrobatchInterval], - cursor_type: str, -) -> bool: - frontier: str = expected.start - for interval in sorted( - accounted, key=lambda item: cursor_sort_key(raw=item.start, cursor_type=cursor_type) - ): - if _lte(left=interval.end, right=frontier, cursor_type=cursor_type): - continue - if _lt(left=frontier, right=interval.start, cursor_type=cursor_type): - return False - frontier = interval.end - if _lte(left=expected.end, right=frontier, cursor_type=cursor_type): - return True - return False - - -def _maximum_bound(*, values: tuple[str, ...], cursor_type: str) -> str | None: - non_empty: tuple[str, ...] = tuple(value for value in values if value) - return max_bound(values=non_empty, cursor_type=cursor_type) if non_empty else None - - -def _minimum_bound(*, values: tuple[str, ...], cursor_type: str) -> str | None: - non_empty: tuple[str, ...] = tuple(value for value in values if value) - return min_bound(values=non_empty, cursor_type=cursor_type) if non_empty else None - - -def _lt(*, left: str, right: str, cursor_type: str) -> bool: - return ( - compare( - left=parse(raw=left, cursor_type=cursor_type), - right=parse(raw=right, cursor_type=cursor_type), - ) - < 0 - ) - - -def _lte(*, left: str, right: str, cursor_type: str) -> bool: - return ( - compare( - left=parse(raw=left, cursor_type=cursor_type), - right=parse(raw=right, cursor_type=cursor_type), - ) - <= 0 - ) diff --git a/src/sqlbuild/microbatches/_helpers/sql.py b/src/sqlbuild/microbatches/_helpers/sql.py deleted file mode 100644 index 7f8928e42c..0000000000 --- a/src/sqlbuild/microbatches/_helpers/sql.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Internal portable SQL for direct-mode microbatch history.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild._native import render_state_sql_literal -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.constants import ( - MICROBATCH_COLUMN_TYPES, - MICROBATCH_COLUMNS, - MICROBATCH_GENERATION_WILDCARD, - MICROBATCH_TABLE_NAME, -) -from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.sql_values.types import StateSqlValueType - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - table: str = _qualified_table( - database=database, schema=schema, render_qualified_name=render_qualified_name - ) - text_type: str = render_framework_type(FrameworkType.STRING) - timestamp_type: str = render_framework_type(FrameworkType.TIMESTAMP) - definitions: list[str] = [] - for column in MICROBATCH_COLUMNS: - declared_type: StateSqlValueType = MICROBATCH_COLUMN_TYPES[column] - if declared_type == StateSqlValueType.TIMESTAMP: - column_type: str = timestamp_type - elif declared_type == StateSqlValueType.INTEGER: - column_type = "BIGINT" - else: - column_type = text_type - definitions.append(f"{column} {column_type}") - return f"CREATE TABLE IF NOT EXISTS {table} ({', '.join(definitions)})" - - -def build_insert_sql( - *, event: MicrobatchEvent, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _qualified_table( - database=event.scope.target_database, - schema=_required_schema(event.scope), - render_qualified_name=render_qualified_name, - ) - values: tuple[object | None, ...] = MicrobatchEventCodec.values(event) - literals: str = ", ".join( - _column_literal(column=column, value=value) - for column, value in zip(MICROBATCH_COLUMNS, values, strict=True) - ) - return ( - f"INSERT INTO {table} ({', '.join(MICROBATCH_COLUMNS)}) SELECT " - f"{literals} " - f"WHERE NOT EXISTS (SELECT 1 FROM {table} " - f"WHERE event_id = {_column_literal(column='event_id', value=event.event_id)})" - ) - - -def build_insert_many_sql( - *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] -) -> str: - """Build one guarded set-based insert for events in the same state table.""" - - if not events: - raise MicrobatchStateError("bulk microbatch event insert requires at least one event") - first: MicrobatchEvent = events[0] - table: str = _qualified_table( - database=first.scope.target_database, - schema=_required_schema(first.scope), - render_qualified_name=render_qualified_name, - ) - selections: list[str] = [] - for index, event in enumerate(events): - values: tuple[object | None, ...] = MicrobatchEventCodec.values(event) - aliases: str = ( - "" - if index > 0 - else " " - + ", ".join( - f"{_column_literal(column=column, value=value)} AS {column}" - for column, value in zip(MICROBATCH_COLUMNS, values, strict=True) - ) - ) - selections.append( - "SELECT " - + ", ".join( - _column_literal(column=column, value=value) - for column, value in zip(MICROBATCH_COLUMNS, values, strict=True) - ) - if index > 0 - else f"SELECT{aliases}" - ) - columns: str = ", ".join(MICROBATCH_COLUMNS) - return ( - f"INSERT INTO {table} ({columns}) SELECT {columns} FROM " - f"({' UNION ALL '.join(selections)}) AS incoming " - f"WHERE NOT EXISTS (SELECT 1 FROM {table} AS existing " - "WHERE existing.event_id = incoming.event_id)" - ) - - -def build_existing_event_ids_sql( - *, events: tuple[MicrobatchEvent, ...], render_qualified_name: Callable[..., str | None] -) -> str: - """Build one lookup for event IDs already present before a bulk insert.""" - - if not events: - raise MicrobatchStateError("microbatch event lookup requires at least one event") - first: MicrobatchEvent = events[0] - table: str = _qualified_table( - database=first.scope.target_database, - schema=_required_schema(first.scope), - render_qualified_name=render_qualified_name, - ) - event_ids: str = ", ".join( - _column_literal(column="event_id", value=event.event_id) for event in events - ) - return f"SELECT event_id FROM {table} WHERE event_id IN ({event_ids})" - - -def build_read_scope_sql( - *, scope: MicrobatchScope, render_qualified_name: Callable[..., str | None] -) -> str: - table: str = _qualified_table( - database=scope.target_database, - schema=_required_schema(scope), - render_qualified_name=render_qualified_name, - ) - generation_predicate: str = ( - "" - if scope.physical_generation_id == MICROBATCH_GENERATION_WILDCARD - else "AND physical_generation_id = " - + _column_literal(column="physical_generation_id", value=scope.physical_generation_id) - + " " - ) - return ( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM {table} " - f"WHERE scope_kind = {_column_literal(column='scope_kind', value=scope.scope_kind)} " - f"AND scope_key = {_column_literal(column='scope_key', value=scope.scope_key)} " - f"{generation_predicate}" - "ORDER BY created_at, event_id" - ) - - -def _qualified_table( - *, database: str | None, schema: str, render_qualified_name: Callable[..., str | None] -) -> str: - table: str | None = render_qualified_name( - database=database, schema=schema, name=MICROBATCH_TABLE_NAME - ) - if table is None: - raise MicrobatchStateError("microbatch history requires a target schema") - return table - - -def _required_schema(scope: MicrobatchScope) -> str: - if scope.target_schema is None: - raise MicrobatchStateError("microbatch history requires a target schema") - return scope.target_schema - - -def _column_literal(*, column: str, value: object | None) -> str: - return render_state_sql_literal(value=value, declared_type=MICROBATCH_COLUMN_TYPES[column]) diff --git a/src/sqlbuild/microbatches/classes/direct_store.py b/src/sqlbuild/microbatches/classes/direct_store.py index 40393c4fa7..1fb05f220f 100644 --- a/src/sqlbuild/microbatches/classes/direct_store.py +++ b/src/sqlbuild/microbatches/classes/direct_store.py @@ -6,14 +6,14 @@ from dataclasses import replace from typing import Any -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.microbatches._helpers.sql import ( - build_existing_event_ids_sql, - build_insert_many_sql, - build_read_scope_sql, +from sqlbuild._native import ( + CompiledRelationLocation, + ModelPlanEntry, + build_microbatch_existing_event_ids_sql, + build_microbatch_insert_many_sql, + build_microbatch_read_scope_sql, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import ( DIRECT_MICROBATCH_SCOPE_KIND, @@ -55,7 +55,7 @@ def write_many(self, events: tuple[MicrobatchEvent, ...]) -> MicrobatchWriteResu ] cursor: Any = self._adapter.execute( connection=self._connection, - sql=build_existing_event_ids_sql( + sql=build_microbatch_existing_event_ids_sql( events=chunk, render_qualified_name=self._adapter.render_qualified_name, ), @@ -69,7 +69,7 @@ def write_many(self, events: tuple[MicrobatchEvent, ...]) -> MicrobatchWriteResu if missing: self._adapter.execute( connection=self._connection, - sql=build_insert_many_sql( + sql=build_microbatch_insert_many_sql( events=missing, render_qualified_name=self._adapter.render_qualified_name, ), @@ -109,7 +109,7 @@ def read_scope_history(self, scope: MicrobatchScope) -> tuple[MicrobatchEvent, . self._initialize(database=scope.target_database, schema=schema) cursor: Any = self._adapter.execute( connection=self._connection, - sql=build_read_scope_sql( + sql=build_microbatch_read_scope_sql( scope=scope, render_qualified_name=self._adapter.render_qualified_name ), ) diff --git a/src/sqlbuild/microbatches/constants.py b/src/sqlbuild/microbatches/constants.py index f302c98c0a..f0d2a3b396 100644 --- a/src/sqlbuild/microbatches/constants.py +++ b/src/sqlbuild/microbatches/constants.py @@ -1,6 +1,6 @@ """Stable constants for microbatch state storage and retries.""" -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType MICROBATCH_TABLE_NAME: str = "_sqlbuild_microbatches" DIRECT_MICROBATCH_SCOPE_KIND: str = "direct_logical" diff --git a/src/sqlbuild/microbatches/main/__init__.py b/src/sqlbuild/microbatches/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/microbatches/main/create_index_sqls.py b/src/sqlbuild/microbatches/main/create_index_sqls.py deleted file mode 100644 index a81d0fcc08..0000000000 --- a/src/sqlbuild/microbatches/main/create_index_sqls.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Public direct microbatch-state index DDL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.microbatches.constants import ( - MICROBATCH_DIRECT_INDEXES, - MICROBATCH_TABLE_NAME, -) - - -def build_create_index_sqls( - *, - database: str | None, - schema: str, - render_identifier: Callable[[str], str], - render_qualified_name: Callable[..., str | None], -) -> tuple[str, ...]: - """Build non-unique scope and provenance indexes for indexed warehouses.""" - - table: str = ( - render_qualified_name(database=database, schema=schema, name=MICROBATCH_TABLE_NAME) - or f"{schema}.{MICROBATCH_TABLE_NAME}" - ) - statements: list[str] = [] - for index_name, columns in MICROBATCH_DIRECT_INDEXES.items(): - rendered_columns: tuple[str, ...] = tuple(render_identifier(column) for column in columns) - statements.append( - f"CREATE INDEX IF NOT EXISTS {render_identifier(index_name)} ON {table} " - f"({', '.join(rendered_columns)})" - ) - return tuple(statements) diff --git a/src/sqlbuild/microbatches/main/create_table_sql.py b/src/sqlbuild/microbatches/main/create_table_sql.py deleted file mode 100644 index 16d2e658f8..0000000000 --- a/src/sqlbuild/microbatches/main/create_table_sql.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public direct microbatch-state DDL rendering entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.microbatches._helpers.sql import build_create_table_sql as _build_create_table_sql - - -def build_create_table_sql( - *, - database: str | None, - schema: str, - render_qualified_name: Callable[..., str | None], - render_framework_type: Callable[[FrameworkType], str], -) -> str: - """Build direct microbatch-state DDL through the public entrypoint.""" - - return _build_create_table_sql( - database=database, - schema=schema, - render_qualified_name=render_qualified_name, - render_framework_type=render_framework_type, - ) diff --git a/src/sqlbuild/microbatches/main/deterministic_event_id.py b/src/sqlbuild/microbatches/main/deterministic_event_id.py deleted file mode 100644 index 361f9cefeb..0000000000 --- a/src/sqlbuild/microbatches/main/deterministic_event_id.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Build stable identifiers for immutable logical microbatch events.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.microbatches.models import MicrobatchScope -from sqlbuild.microbatches.types import MicrobatchRecordType - - -def deterministic_microbatch_event_id( - *, - scope: MicrobatchScope, - record_type: MicrobatchRecordType, - partition_start: str | None, - partition_end: str | None, - completion_reason: str, -) -> str: - """Return the content-addressed ID for one logical event.""" - - identity: tuple[object, ...] = ( - scope.scope_kind, - scope.scope_key, - scope.model_name, - scope.target_database, - scope.target_schema, - scope.target_name, - scope.physical_generation_id, - record_type.value, - partition_start, - partition_end, - completion_reason, - ) - encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) - return hashlib.sha256(encoded.encode()).hexdigest() diff --git a/src/sqlbuild/microbatches/main/latest_replay_requirement.py b/src/sqlbuild/microbatches/main/latest_replay_requirement.py deleted file mode 100644 index f1e5a9f331..0000000000 --- a/src/sqlbuild/microbatches/main/latest_replay_requirement.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public active replay-requirement selection entrypoint.""" - -from sqlbuild.microbatches._helpers.projection import ( - latest_active_replay_requirement as _latest_active_replay_requirement, -) -from sqlbuild.microbatches.models import MicrobatchEvent - - -def latest_active_replay_requirement( - *, events: tuple[MicrobatchEvent, ...], current_model_version_hash: str -) -> MicrobatchEvent | None: - """Return the latest requirement for the currently expected model version.""" - - return _latest_active_replay_requirement( - events=events, current_model_version_hash=current_model_version_hash - ) diff --git a/src/sqlbuild/microbatches/main/project_coverage.py b/src/sqlbuild/microbatches/main/project_coverage.py deleted file mode 100644 index a60b4d0fbf..0000000000 --- a/src/sqlbuild/microbatches/main/project_coverage.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public physical-coverage projection entrypoint.""" - -from sqlbuild.microbatches._helpers.projection import ( - project_microbatch_coverage as _project_microbatch_coverage, -) -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, -) - - -def project_microbatch_coverage( - *, - events: tuple[MicrobatchEvent, ...], - expected_intervals: tuple[MicrobatchInterval, ...], - cursor_type: str, -) -> MicrobatchCoverageProjection: - """Project physical and fingerprint coverage for expected intervals.""" - - return _project_microbatch_coverage( - events=events, - expected_intervals=expected_intervals, - cursor_type=cursor_type, - ) diff --git a/src/sqlbuild/microbatches/main/project_replay.py b/src/sqlbuild/microbatches/main/project_replay.py deleted file mode 100644 index 02f5dc2daa..0000000000 --- a/src/sqlbuild/microbatches/main/project_replay.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public replay-requirement projection entrypoint.""" - -from sqlbuild.microbatches._helpers.projection import ( - project_replay_requirement as _project_replay_requirement, -) -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, - ReplayRequirementProjection, -) - - -def project_replay_requirement( - *, - requirement: MicrobatchEvent, - current_model_version_hash: str, - expected_intervals: tuple[MicrobatchInterval, ...], - coverage: MicrobatchCoverageProjection, - cursor_type: str, -) -> ReplayRequirementProjection: - """Project derived completion state for one replay requirement.""" - - return _project_replay_requirement( - requirement=requirement, - current_model_version_hash=current_model_version_hash, - expected_intervals=expected_intervals, - coverage=coverage, - cursor_type=cursor_type, - ) diff --git a/src/sqlbuild/presentation/__init__.py b/src/sqlbuild/presentation/__init__.py index eed713edc9..e69de29bb2 100644 --- a/src/sqlbuild/presentation/__init__.py +++ b/src/sqlbuild/presentation/__init__.py @@ -1 +0,0 @@ -"""Human and terminal presentation domain.""" diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py deleted file mode 100644 index ef8eccfe8f..0000000000 --- a/src/sqlbuild/presentation/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Presentation constants.""" - -from __future__ import annotations - -DEFAULT_MAX_DISPLAY_ENTRIES: int = 20 diff --git a/src/sqlbuild/presentation/types.py b/src/sqlbuild/presentation/types.py deleted file mode 100644 index 467d0c0cfb..0000000000 --- a/src/sqlbuild/presentation/types.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Type-layer declarations for presentation.""" - -from typing import Protocol - - -class TransientLineOwner(Protocol): - """A renderer that owns the transient line currently drawn in place on a terminal.""" - - def clear_transient_line(self) -> None: - """Erase the drawn transient line so a persistent line can take its place.""" - ... - - def redraw_transient_line(self) -> None: - """Draw the transient line again below the persistent output.""" - ... diff --git a/src/sqlbuild/provider/_helpers/session.py b/src/sqlbuild/provider/_helpers/session.py index 6776f19815..732740b559 100644 --- a/src/sqlbuild/provider/_helpers/session.py +++ b/src/sqlbuild/provider/_helpers/session.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING -from sqlbuild.compiler.discovery.models import DiscoveredProvider +from sqlbuild._native import DiscoveredProvider from sqlbuild.provider.classes.session import ProviderSession if TYPE_CHECKING: diff --git a/src/sqlbuild/provider/main/session.py b/src/sqlbuild/provider/main/session.py index 231e165740..31c2712bb6 100644 --- a/src/sqlbuild/provider/main/session.py +++ b/src/sqlbuild/provider/main/session.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.discovery.models import DiscoveredProvider +from sqlbuild._native import DiscoveredProvider from sqlbuild.provider._helpers.session import build_provider_session as _build_provider_session from sqlbuild.provider.classes.session import ProviderSession diff --git a/src/sqlbuild/python_nodes/_helpers/column_normalization.py b/src/sqlbuild/python_nodes/_helpers/column_normalization.py index 50548275b8..6e59ac75b6 100644 --- a/src/sqlbuild/python_nodes/_helpers/column_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/column_normalization.py @@ -2,8 +2,8 @@ from collections.abc import Sequence +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes.types import LoaderColumnSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry def normalize_columns( diff --git a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py index 6af73f223e..ec6adc9837 100644 --- a/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/dependency_normalization.py @@ -2,7 +2,6 @@ from collections.abc import Callable -from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.python_nodes.models import SqlResourceRef @@ -21,18 +20,3 @@ def normalize_loader_dependencies( value: tuple[Callable[..., object], ...] | list[Callable[..., object]], ) -> tuple[Callable[..., object], ...]: return tuple(value) - - -def normalize_hook_reads( - value: SqlResourceRef | tuple[SqlResourceRef, ...] | list[SqlResourceRef], -) -> tuple[SqlResourceRef, ...]: - reads: tuple[object, ...] = (value,) if isinstance(value, SqlResourceRef) else tuple(value) - invalid: object | None = next( - (item for item in reads if not isinstance(item, SqlResourceRef)), None - ) - if invalid is not None: - raise SharedInputError( - "@hook(reads=...) accepts model(...), source(...), and seed(...) references only; " - f"got {type(invalid).__name__}" - ) - return tuple(dict.fromkeys(item for item in reads if isinstance(item, SqlResourceRef))) diff --git a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py index 458112ffa8..a69d729e8f 100644 --- a/src/sqlbuild/python_nodes/_helpers/loader_authoring.py +++ b/src/sqlbuild/python_nodes/_helpers/loader_authoring.py @@ -4,12 +4,12 @@ from collections.abc import Callable, Sequence +from sqlbuild._native import SourceColumnEntry +from sqlbuild._native import SourceWriteStrategy as SourceWriteStrategy from sqlbuild.python_nodes.main._apply_loader import apply_loader from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.models import LoaderDefinition from sqlbuild.python_nodes.types import LoaderColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy as SourceWriteStrategy def loader( diff --git a/src/sqlbuild/python_nodes/_helpers/loader_normalization.py b/src/sqlbuild/python_nodes/_helpers/loader_normalization.py index 0dce329575..334eb4caec 100644 --- a/src/sqlbuild/python_nodes/_helpers/loader_normalization.py +++ b/src/sqlbuild/python_nodes/_helpers/loader_normalization.py @@ -2,7 +2,7 @@ from collections.abc import Sequence -from sqlbuild.spec.contracts.types import SourceWriteStrategy +from sqlbuild._native import SourceWriteStrategy def normalize_unique_key(value: str | Sequence[str] | None) -> tuple[str, ...]: diff --git a/src/sqlbuild/python_nodes/main/_apply_loader.py b/src/sqlbuild/python_nodes/main/_apply_loader.py index f2fc9f444f..67367a9a9e 100644 --- a/src/sqlbuild/python_nodes/main/_apply_loader.py +++ b/src/sqlbuild/python_nodes/main/_apply_loader.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Sequence from typing import Any, cast +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes._helpers.attachment import attach_definition from sqlbuild.python_nodes._helpers.column_normalization import normalize_columns from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_loader_dependencies @@ -15,7 +16,6 @@ ) from sqlbuild.python_nodes.models import LoaderDefinition from sqlbuild.python_nodes.types import LoaderColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry def apply_loader( diff --git a/src/sqlbuild/python_nodes/main/apply_asset.py b/src/sqlbuild/python_nodes/main/apply_asset.py index d077bbe786..708ea4220c 100644 --- a/src/sqlbuild/python_nodes/main/apply_asset.py +++ b/src/sqlbuild/python_nodes/main/apply_asset.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Mapping, Sequence from typing import Any, cast +from sqlbuild._native import SourceColumnEntry from sqlbuild.python_nodes._helpers.attachment import attach_definition from sqlbuild.python_nodes._helpers.column_normalization import normalize_columns from sqlbuild.python_nodes._helpers.dependency_normalization import ( @@ -19,7 +20,6 @@ SqlResourceRef, ) from sqlbuild.python_nodes.types import ColumnLineageRefSpec, PythonNodeColumnSpec -from sqlbuild.spec.contracts.models import SourceColumnEntry def apply_asset( diff --git a/src/sqlbuild/python_nodes/main/apply_hook.py b/src/sqlbuild/python_nodes/main/apply_hook.py index c74abe3473..99dd9459ea 100644 --- a/src/sqlbuild/python_nodes/main/apply_hook.py +++ b/src/sqlbuild/python_nodes/main/apply_hook.py @@ -5,8 +5,8 @@ from collections.abc import Callable from typing import Any, cast +from sqlbuild._native import normalize_hook_reads from sqlbuild.python_nodes._helpers.attachment import attach_definition -from sqlbuild.python_nodes._helpers.dependency_normalization import normalize_hook_reads from sqlbuild.python_nodes._helpers.description_resolution import resolve_description from sqlbuild.python_nodes.models import HookDefinition, SqlResourceRef diff --git a/src/sqlbuild/python_nodes/main/calculate_retry_delay.py b/src/sqlbuild/python_nodes/main/calculate_retry_delay.py index ebe10fa53d..3e7b10876f 100644 --- a/src/sqlbuild/python_nodes/main/calculate_retry_delay.py +++ b/src/sqlbuild/python_nodes/main/calculate_retry_delay.py @@ -1,18 +1,10 @@ """Calculate retry delays for Python-node execution.""" -import random - +from sqlbuild._native import retry_delay_seconds from sqlbuild.python_nodes.models import RetryPolicy def calculate_retry_delay(*, retry_policy: RetryPolicy, retry_index: int) -> float: """Calculate the delay before a retry attempt.""" - delay_seconds: float = retry_policy.initial_delay_seconds * ( - retry_policy.backoff_multiplier**retry_index - ) - if retry_policy.max_delay_seconds is not None: - delay_seconds = min(delay_seconds, retry_policy.max_delay_seconds) - if retry_policy.jitter: - return random.uniform(0, delay_seconds) - return delay_seconds + return retry_delay_seconds(retry_policy=retry_policy, retry_index=retry_index) diff --git a/src/sqlbuild/python_nodes/models.py b/src/sqlbuild/python_nodes/models.py index f297bdd362..2f5c61e697 100644 --- a/src/sqlbuild/python_nodes/models.py +++ b/src/sqlbuild/python_nodes/models.py @@ -2,17 +2,19 @@ from __future__ import annotations -import re from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from sqlbuild.compiler.auditing.models import MeasurementThresholds +from sqlbuild._native import ( + MeasurementThresholds, + SourceColumnEntry, + SourceWriteStrategy, + retry_policy_exceptions, + validate_audit_case, +) from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.python_nodes.constants import RELATION_RENDERER from sqlbuild.python_nodes.types import PythonCheckSeverity, SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceColumnEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy @dataclass(frozen=True) @@ -64,27 +66,14 @@ def __init__( max_elapsed_seconds: float | None = None, jitter: bool = True, ) -> None: - if max_attempts < 1: - raise SharedInputError("RetryPolicy max_attempts must be at least 1") - if initial_delay_seconds < 0: - raise SharedInputError("RetryPolicy initial_delay_seconds must be non-negative") - if backoff_multiplier < 1: - raise SharedInputError("RetryPolicy backoff_multiplier must be at least 1") - if max_delay_seconds is not None and max_delay_seconds < 0: - raise SharedInputError("RetryPolicy max_delay_seconds must be non-negative") - if max_elapsed_seconds is not None and max_elapsed_seconds < 0: - raise SharedInputError("RetryPolicy max_elapsed_seconds must be non-negative") - normalized_retry_on: tuple[type[BaseException], ...] - if isinstance(retry_on, type) and issubclass(retry_on, BaseException): - normalized_retry_on = (retry_on,) - else: - normalized_retry_on = tuple(retry_on) - exception_type: type[BaseException] - for exception_type in normalized_retry_on: - if not isinstance(exception_type, type) or not issubclass( - exception_type, BaseException - ): - raise SharedInputError("RetryPolicy retry_on must contain exception classes") + normalized_retry_on: tuple[type[BaseException], ...] = retry_policy_exceptions( + max_attempts=max_attempts, + retry_on=retry_on, + initial_delay_seconds=initial_delay_seconds, + backoff_multiplier=backoff_multiplier, + max_delay_seconds=max_delay_seconds, + max_elapsed_seconds=max_elapsed_seconds, + ) object.__setattr__(self, "max_attempts", max_attempts) object.__setattr__(self, "retry_on", normalized_retry_on) object.__setattr__(self, "initial_delay_seconds", initial_delay_seconds) @@ -189,22 +178,9 @@ class AuditCase: evidence_limit: int | None = None def __post_init__(self) -> None: - if not re.fullmatch(r"^[a-z][a-z0-9_]*$", self.name): - raise SharedInputError( - "AuditCase name must be a non-empty lowercase identifier using letters, " - "numbers, and underscores" - ) - if not self.definition.strip(): - raise SharedInputError("AuditCase definition must be non-empty") - if self.minimum_samples is not None and ( - not isinstance(self.minimum_samples, int) - or isinstance(self.minimum_samples, bool) - or self.minimum_samples < 0 - ): - raise SharedInputError("AuditCase minimum_samples must be a non-negative integer") - if self.evidence_limit is not None and ( - not isinstance(self.evidence_limit, int) - or isinstance(self.evidence_limit, bool) - or self.evidence_limit < 0 - ): - raise SharedInputError("AuditCase evidence_limit must be a non-negative integer") + validate_audit_case( + name=self.name, + definition=self.definition, + minimum_samples=self.minimum_samples, + evidence_limit=self.evidence_limit, + ) diff --git a/src/sqlbuild/rule_engine/__init__.py b/src/sqlbuild/rule_engine/__init__.py index 5309b45486..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/__init__.py +++ b/src/sqlbuild/rule_engine/__init__.py @@ -1 +0,0 @@ -"""SQLBuild rules engine.""" diff --git a/src/sqlbuild/rule_engine/_helpers/__init__.py b/src/sqlbuild/rule_engine/_helpers/__init__.py index 8c45c20f48..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/_helpers/__init__.py +++ b/src/sqlbuild/rule_engine/_helpers/__init__.py @@ -1 +0,0 @@ -"""Internal rules helpers.""" diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py index c86d8bafe6..58cd57f4b7 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -9,13 +9,12 @@ from pathlib import Path from types import ModuleType +from sqlbuild._native import PYTHON_INIT_MODULE_STEM, Rule, RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.constants import INIT_MODULE_NAME, RULE_DECORATOR_TOKEN -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Rule +from sqlbuild.rule_engine.constants import RULE_DECORATOR_TOKEN def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: @@ -51,7 +50,7 @@ def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: def _rule_module_name(*, file_path: Path, project_dir: Path) -> str: relative: Path = file_path.relative_to(project_dir.resolve()).with_suffix("") parts: tuple[str, ...] = relative.parts - if parts[-1] == INIT_MODULE_NAME: + if parts[-1] == PYTHON_INIT_MODULE_STEM: parts = parts[:-1] return ".".join(parts) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py index d18c5786e7..708754bb50 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_payloads.py @@ -6,11 +6,11 @@ from pathlib import Path from sqlbuild._native import ( + Rule, custom_rule_implementation_fingerprint, custom_rule_import_closure, custom_rule_test_evidence, ) -from sqlbuild.rule_engine.models import Rule def custom_rule_payloads( diff --git a/src/sqlbuild/rule_engine/_helpers/engine/definition.py b/src/sqlbuild/rule_engine/_helpers/engine/definition.py index 973ded3f39..5f7644caba 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/definition.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/definition.py @@ -6,9 +6,8 @@ from collections.abc import Callable from typing import cast, get_type_hints -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Model, Project, Rule, RuleOption -from sqlbuild.rule_engine.types import RuleCheck, RuleContext, RuleSubject +from sqlbuild._native import Model, Project, Rule, RuleOption, RulesError, RuleSubject +from sqlbuild.rule_engine.types import RuleCheck, RuleContext _RULE_PARAMETER_COUNT: int = 2 _RULE_CODE_DIGIT_COUNT: int = 3 diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py index 0832f7e1a2..dc149b703e 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py @@ -9,6 +9,7 @@ import orjson +from sqlbuild._native import FactDigestError, Model, RuleFactViews, tracked_text_file_values from sqlbuild.rule_engine.classes.shared_fact_encoder import SharedFactEncoder from sqlbuild.rule_engine.classes.sql_document import SqlDocument from sqlbuild.rule_engine.constants import ( @@ -42,11 +43,8 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) -from sqlbuild.rule_engine.exceptions import FactDigestError -from sqlbuild.rule_engine.models import Model, RuleFactViews from sqlbuild.rule_engine.types import FactKey -_TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) _ORJSON_OPTIONS: int = orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS _GLOBAL_FACTS: dict[str, Callable[[RuleFactViews], object]] = { @@ -118,9 +116,8 @@ def full_fact_keys(views: RuleFactViews) -> tuple[FactKey, ...]: keys.extend((fact, path) for fact in _MODEL_PATH_FACTS) keys.extend((fact, model.name) for fact in _MODEL_NAME_FACTS) keys.extend( - (FACT_TREE_READ_TEXT, node.value) - for node in views.project.tree.paths() - if node.is_file and Path(node.value).suffix.lower() in _TRACKED_TEXT_SUFFIXES + (FACT_TREE_READ_TEXT, value) + for value in tracked_text_file_values(views.project.tree.paths()) ) return tuple(keys) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py b/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py deleted file mode 100644 index 22173a8d50..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/hermeticity.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Static import verification for the fingerprinted custom-rule source closure.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -from sqlbuild._native import custom_rule_import_closure -from sqlbuild.rule_engine.constants import CUSTOM_RULE_IMPORT_ROOTS -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError, RulesError -from sqlbuild.rule_engine.models import Rule - - -def verify_custom_rules(*, rules: tuple[Rule, ...], project_dir: Path) -> None: - """Reject imports outside the deterministic allowlist in every fingerprinted rule file.""" - - root: Path = project_dir.resolve() - checked: set[Path] = set() - for rule in rules: - if not rule.custom or rule.source is None: - continue - source_path: Path = Path(rule.source).resolve() - closure: tuple[Path, ...] = custom_rule_import_closure(rule=rule, project_dir=root) - for path in (source_path, *closure): - if path in checked: - continue - checked.add(path) - _verify_source(path=path, project_dir=root) - - -def allowed_custom_rule_import(module: str) -> bool: - """Return whether custom-rule code may import one absolute module name.""" - - return any(module == root or module.startswith(f"{root}.") for root in CUSTOM_RULE_IMPORT_ROOTS) - - -def _verify_source(*, path: Path, project_dir: Path) -> None: - if not path.is_relative_to(project_dir): - raise RulesError(f"custom rule source must be repository-owned: {path}") - source: str = path.read_text(encoding="utf-8") - tree: ast.Module = ast.parse(source, filename=str(path)) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - _verify_import(path=path, line=node.lineno, module=alias.name) - elif isinstance(node, ast.ImportFrom) and node.level == 0: - _verify_import(path=path, line=node.lineno, module=node.module or "") - - -def _verify_import(*, path: Path, line: int, module: str) -> None: - if not allowed_custom_rule_import(module): - raise NonHermeticRuleError( - f"non-hermetic custom rule at {path}:{line}: import {module!r} is not allowed" - ) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index fb31d74cc8..4acfa16047 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -7,27 +7,27 @@ from tempfile import TemporaryDirectory from sqlbuild._native import ( + MIN_CUSTOM_RULE_TEST_CASES, + DiscoveredProjectInputs, + Rule, + RuleAssertionError, + RuleCase, + RuleResult, + RulesConfig, + RulesResult, + RuleUsageError, build_compiled_project, discover_project_inputs, evaluate_project_rules, load_rules_config, + rule_option_toml_value, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.exceptions import RuleAssertionError, RuleUsageError -from sqlbuild.rule_engine.models import ( - Rule, - RuleCase, - RuleResult, - RulesConfig, - RulesResult, -) from sqlbuild.rule_engine.types import RuleCheck @@ -76,7 +76,7 @@ def _write_rule_project(*, root: Path, rule: Rule, test_case: RuleCase) -> Path: if test_case.config: config_lines.append(f"[rules.rule_options.{rule.code}]") for name, value in sorted(test_case.config.items()): - config_lines.append(f"{name} = {_toml_value(value=value)}") + config_lines.append(f"{name} = {rule_option_toml_value(value)}") config_lines.append("") project_config: str = "\n".join(config_lines) (root / "sqlbuild_project.toml").write_text(project_config, encoding="utf-8") @@ -96,19 +96,6 @@ def _write_rule_project(*, root: Path, rule: Rule, test_case: RuleCase) -> Path: return rule_path -def _toml_value(*, value: object) -> str: - if isinstance(value, bool): - return str(value).lower() - if isinstance(value, int): - return str(value) - if isinstance(value, str): - escaped: str = value.replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' - if isinstance(value, (list, tuple)): - return "[" + ", ".join(_toml_value(value=item) for item in value) + "]" - raise RuleUsageError(f"RuleCase config value {value!r} cannot be represented as a rule option") - - def _compile_rule_project(*, root: Path) -> CompiledProjectView: discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=root) adapter: DuckDbAdapter = DuckDbAdapter() diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 0f40861c9e..1345c0b19d 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -9,6 +9,21 @@ from itertools import chain from pathlib import Path +from sqlbuild._native import ( + CUSTOM_RULE_PROJECT_SUBJECT, + CustomHostPartition, + CustomRuleEvaluation, + CustomRuleRun, + Finding, + HostCancelledError, + Model, + NonHermeticRuleError, + Project, + Rule, + RulesConfig, + RulesError, + RuleSubject, +) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import rule_namespaces from sqlbuild.rule_engine._helpers.host.tracked_facts import tracked_fact_views @@ -21,26 +36,8 @@ build_rule_fact_views, ) from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard -from sqlbuild.rule_engine.constants import ( - CUSTOM_HOST_CANCELLED_MESSAGE, - CUSTOM_RULE_PROJECT_SUBJECT, -) -from sqlbuild.rule_engine.exceptions import ( - HostCancelledError, - NonHermeticRuleError, - RulesError, -) -from sqlbuild.rule_engine.models import ( - CustomHostPartition, - CustomRuleEvaluation, - CustomRuleRun, - Finding, - Model, - Project, - Rule, - RulesConfig, -) -from sqlbuild.rule_engine.types import CustomRulePlan, RuleSubject +from sqlbuild.rule_engine.constants import CUSTOM_HOST_CANCELLED_MESSAGE +from sqlbuild.rule_engine.types import CustomRulePlan type _FindingKey = tuple[str, str, int, int, str, str] diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 76d0c2cbf9..b00c6ecb87 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,8 +14,20 @@ from pathlib import Path from typing import Any, BinaryIO -from sqlbuild._native import build_rule_catalogue -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild._native import ( + CUSTOM_HOST_LAUNCH_MODULE, + CUSTOM_HOST_PROTOCOL_VERSION, + CUSTOM_HOST_RUNTIME_VERSION, + CompiledProject, + CustomHostPartition, + CustomRuleRun, + Finding, + HostCancelledError, + Rule, + RulesConfig, + RulesError, + build_rule_catalogue, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules @@ -28,18 +40,7 @@ CUSTOM_HOST_INPUT_POLL_SECONDS, CUSTOM_HOST_INPUT_TUPLE_SIZE, CUSTOM_HOST_INPUT_WAIT_SECONDS, - CUSTOM_HOST_LAUNCH_MODULE, CUSTOM_HOST_MAX_TRACKED_READS, - CUSTOM_HOST_PROTOCOL_VERSION, - CUSTOM_HOST_RUNTIME_VERSION, -) -from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError -from sqlbuild.rule_engine.models import ( - CustomHostPartition, - CustomRuleRun, - Finding, - Rule, - RulesConfig, ) from sqlbuild.rule_engine.types import CustomRulePlan diff --git a/src/sqlbuild/rule_engine/_helpers/host/module_state.py b/src/sqlbuild/rule_engine/_helpers/host/module_state.py index eba9b96c5c..93004b792b 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/module_state.py +++ b/src/sqlbuild/rule_engine/_helpers/host/module_state.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import enum import hashlib import os @@ -16,8 +17,7 @@ from types import BuiltinFunctionType, CellType, FunctionType, MethodType, ModuleType from typing import Any, cast -from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError -from sqlbuild.rule_engine.models import ModuleStateFingerprint, ModuleStateTrace +from sqlbuild._native import ModuleStateFingerprint, ModuleStateTrace, OpaqueModuleStateError _MAX_STATE_NODES: int = 50_000 _ATOMIC_TYPES: tuple[type, ...] = ( @@ -557,6 +557,10 @@ def _instance_state(value: object) -> dict[str, object]: for slot in getattr(owner, "__slots__", ()): if isinstance(slot, str) and hasattr(value, slot): state[slot] = getattr(value, slot) + if not state and dataclasses.is_dataclass(value) and not isinstance(value, type): + state.update( + (field.name, getattr(value, field.name)) for field in dataclasses.fields(value) + ) return state diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index 9cdd9567fa..854fc1dc92 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -6,9 +6,18 @@ from pathlib import Path from typing import Any -from sqlbuild.compiler.compile.models import CompiledSqlTest, InferredColumn +from sqlbuild._native import ( + CompiledSqlTest, + ConstantDeclaration, + EnumDeclaration, + FactDigestError, + InferredColumn, + Model, + ModelSql, + ProjectPath, + RuleFactViews, +) from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.constants import ( @@ -42,8 +51,6 @@ FACT_TREE_RELATIVE_PARTS, FACT_TREE_RESOURCES_UNDER, ) -from sqlbuild.rule_engine.exceptions import FactDigestError -from sqlbuild.rule_engine.models import Model, ModelSql, ProjectPath, RuleFactViews class _TrackedView: diff --git a/src/sqlbuild/rule_engine/classes/__init__.py b/src/sqlbuild/rule_engine/classes/__init__.py index 8f088725ac..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/classes/__init__.py +++ b/src/sqlbuild/rule_engine/classes/__init__.py @@ -1 +0,0 @@ -"""Rules runtime classes.""" diff --git a/src/sqlbuild/rule_engine/classes/audit_facts.py b/src/sqlbuild/rule_engine/classes/audit_facts.py index 361b2dce77..bcbb39994a 100644 --- a/src/sqlbuild/rule_engine/classes/audit_facts.py +++ b/src/sqlbuild/rule_engine/classes/audit_facts.py @@ -3,11 +3,11 @@ from functools import cached_property from sqlbuild._native import ( + CompiledAudit, + Model, compiled_audit_record, ) -from sqlbuild.compiler.compile.models import CompiledAudit from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.models import Model class AuditFacts: diff --git a/src/sqlbuild/rule_engine/classes/column_facts.py b/src/sqlbuild/rule_engine/classes/column_facts.py index f6b62b392a..6cad2a9fa7 100644 --- a/src/sqlbuild/rule_engine/classes/column_facts.py +++ b/src/sqlbuild/rule_engine/classes/column_facts.py @@ -2,9 +2,8 @@ from functools import cached_property -from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild._native import InferredColumn, Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.rule_engine.models import Model class ColumnFacts: diff --git a/src/sqlbuild/rule_engine/classes/contract_facts.py b/src/sqlbuild/rule_engine/classes/contract_facts.py index f5e757925a..454bc611e4 100644 --- a/src/sqlbuild/rule_engine/classes/contract_facts.py +++ b/src/sqlbuild/rule_engine/classes/contract_facts.py @@ -1,8 +1,7 @@ """Contract facts for custom rules.""" +from sqlbuild._native import CONTRACT_ENFORCED, Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.rule_engine.constants import CONTRACT_ENFORCED -from sqlbuild.rule_engine.models import Model class ContractFacts: diff --git a/src/sqlbuild/rule_engine/classes/declaration_facts.py b/src/sqlbuild/rule_engine/classes/declaration_facts.py index c410d315b0..e0724599f7 100644 --- a/src/sqlbuild/rule_engine/classes/declaration_facts.py +++ b/src/sqlbuild/rule_engine/classes/declaration_facts.py @@ -1,7 +1,7 @@ """Declaration facts for custom rules.""" +from sqlbuild._native import ConstantDeclaration, EnumDeclaration from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration class DeclarationFacts: diff --git a/src/sqlbuild/rule_engine/classes/fact_reads.py b/src/sqlbuild/rule_engine/classes/fact_reads.py index 8680946276..7fa8ed04e5 100644 --- a/src/sqlbuild/rule_engine/classes/fact_reads.py +++ b/src/sqlbuild/rule_engine/classes/fact_reads.py @@ -4,7 +4,7 @@ import os -from sqlbuild.rule_engine.models import Model +from sqlbuild._native import Model class FactReads: diff --git a/src/sqlbuild/rule_engine/classes/graph_facts.py b/src/sqlbuild/rule_engine/classes/graph_facts.py index 98df4f64ec..1ccc5069fd 100644 --- a/src/sqlbuild/rule_engine/classes/graph_facts.py +++ b/src/sqlbuild/rule_engine/classes/graph_facts.py @@ -2,9 +2,9 @@ from functools import cached_property +from sqlbuild._native import Model from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import public_model -from sqlbuild.rule_engine.models import Model class GraphFacts: diff --git a/src/sqlbuild/rule_engine/classes/model_test_facts.py b/src/sqlbuild/rule_engine/classes/model_test_facts.py index 395bff6d35..46efec3754 100644 --- a/src/sqlbuild/rule_engine/classes/model_test_facts.py +++ b/src/sqlbuild/rule_engine/classes/model_test_facts.py @@ -2,9 +2,8 @@ from functools import cached_property -from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild._native import CompiledSqlTest, Model from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode -from sqlbuild.rule_engine.models import Model class TestFacts: diff --git a/src/sqlbuild/rule_engine/classes/module_state_monitor.py b/src/sqlbuild/rule_engine/classes/module_state_monitor.py index 98435f2506..9bfa9d6385 100644 --- a/src/sqlbuild/rule_engine/classes/module_state_monitor.py +++ b/src/sqlbuild/rule_engine/classes/module_state_monitor.py @@ -5,13 +5,12 @@ from itertools import chain from pathlib import Path +from sqlbuild._native import ModuleStateFingerprint, ModuleStateTrace, OpaqueModuleStateError from sqlbuild.rule_engine._helpers.host.module_state import ( module_state_fingerprint, module_state_snapshot, rules_root_prefix, ) -from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError -from sqlbuild.rule_engine.models import ModuleStateFingerprint, ModuleStateTrace _EMPTY_TRACE: ModuleStateTrace = ModuleStateTrace( touched={}, descended_types=frozenset(), checked_instances=frozenset() diff --git a/src/sqlbuild/rule_engine/classes/project_facts.py b/src/sqlbuild/rule_engine/classes/project_facts.py index b07de8a219..8c5b37873c 100644 --- a/src/sqlbuild/rule_engine/classes/project_facts.py +++ b/src/sqlbuild/rule_engine/classes/project_facts.py @@ -3,6 +3,7 @@ from pathlib import Path from sqlbuild._native import ( + Model, compiled_audit_record, compiled_function_record, compiled_seed_record, @@ -10,7 +11,6 @@ ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import ProjectTree, public_model -from sqlbuild.rule_engine.models import Model class ProjectFacts: diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index f192e64dae..b009d556b0 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -2,19 +2,21 @@ from __future__ import annotations -import fnmatch -import os -import stat from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import ( + Model, + ProjectPath, + RuleUsageError, + project_tree_child_nodes, + project_tree_descendant_nodes, + project_tree_file_path, + project_tree_glob_nodes, + project_tree_paths, + project_tree_relative_parts, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN -from sqlbuild.rule_engine.exceptions import RuleUsageError -from sqlbuild.rule_engine.models import Model, ProjectPath - -_EXCLUDED_ROOTS: frozenset[str] = frozenset({".git", ".venv", "logs", "target", "venv"}) -_TRACKED_TEXT_SUFFIXES: frozenset[str] = frozenset({".py", ".sql", ".toml", ".yaml", ".yml"}) class ProjectTree: @@ -29,7 +31,6 @@ def __init__( ) -> None: self._root: Path = project_dir.resolve() self._discovered: tuple[ProjectPath, ...] | None = None - self._directories: dict[Path, Path | None] = {} model_views: tuple[Model, ...] = ( tuple(public_model(item) for item in project.models) if models is None else models ) @@ -40,37 +41,23 @@ def __init__( @property def _nodes(self) -> tuple[ProjectPath, ...]: if self._discovered is None: - self._discovered = self._discover() + self._discovered = project_tree_paths(self._root) return self._discovered def paths(self) -> tuple[ProjectPath, ...]: return self._nodes def children(self, path: str = "") -> tuple[ProjectPath, ...]: - parent: Path = Path(path) - return tuple(node for node in self._nodes if Path(node.value).parent == parent) + return project_tree_child_nodes(self._nodes, path) def descendants(self, path: str = "") -> tuple[ProjectPath, ...]: - prefix: str = f"{Path(path).as_posix().rstrip('/')}/" if path else "" - return tuple(node for node in self._nodes if node.value.startswith(prefix)) + return project_tree_descendant_nodes(self._nodes, path) def glob(self, pattern: str) -> tuple[ProjectPath, ...]: - self._validate_relative(pattern) - return tuple(node for node in self._nodes if fnmatch.fnmatchcase(node.value, pattern)) + return project_tree_glob_nodes(self._nodes, pattern) def relative_parts(self, *, path: Path | str, under: str = "") -> tuple[str, ...]: - relative: Path = Path(path) - if relative.is_absolute(): - try: - relative = relative.resolve().relative_to(self._root) - except ValueError as error: - raise RuleUsageError(f"project path escapes the project: {path}") from error - if under: - try: - relative = relative.relative_to(Path(under)) - except ValueError as error: - raise RuleUsageError(f"project path {path} is not beneath {under}") from error - return relative.parts + return project_tree_relative_parts(root=self._root, path=path, under=under) def resources_under(self, path: str) -> tuple[Model, ...]: prefix: str = f"{Path(path).as_posix().rstrip('/')}/" @@ -81,78 +68,15 @@ def resources_under(self, path: str) -> tuple[Model, ...]: ) def read_text(self, path: str) -> str: - self._validate_relative(path) - regular: Path | None = self._regular_file(path) - candidate: Path = (self._root / path).resolve() if regular is None else regular - if regular is None and ( - not candidate.is_relative_to(self._root) or not candidate.is_file() - ): - raise RuleUsageError(f"project file does not exist: {path}") - if candidate.suffix.lower() not in _TRACKED_TEXT_SUFFIXES: - supported: str = ", ".join(sorted(_TRACKED_TEXT_SUFFIXES)) - raise RuleUsageError( - f"compiler rules input must be a tracked file type ({supported}): {path}" - ) - if regular is None and candidate.is_symlink(): - raise RuleUsageError(f"project file must not be a symlink: {path}") + file: str = project_tree_file_path(self._root, path) try: - return candidate.read_text(encoding="utf-8") + return Path(file).read_text(encoding="utf-8") except (OSError, UnicodeError) as error: raise RuleUsageError(f"could not read project file {path}: {error}") from error - def _regular_file(self, path: str) -> Path | None: - """Resolve a regular, non-symlink file under the project through its cached directory.""" - - relative: Path = Path(path) - if not relative.name: - return None - directory: Path | None - if relative.parent in self._directories: - directory = self._directories[relative.parent] - else: - resolved: Path = (self._root / relative.parent).resolve() - directory = resolved if resolved.is_relative_to(self._root) else None - self._directories[relative.parent] = directory - if directory is None: - return None - candidate: Path = directory / relative.name - try: - regular: bool = stat.S_ISREG(os.lstat(candidate).st_mode) - except OSError: - return None - return candidate if regular else None - - def _discover(self) -> tuple[ProjectPath, ...]: - nodes: list[ProjectPath] = [] - for root, directories, filenames in os.walk(self._root, followlinks=False): - root_path: Path = Path(root) - directories[:] = sorted( - name - for name in directories - if not name.startswith(".") - and not (root_path == self._root and name in _EXCLUDED_ROOTS) - and not (root_path / name).is_symlink() - ) - for name in directories: - relative: str = (root_path / name).relative_to(self._root).as_posix() - nodes.append(ProjectPath(value=relative, is_file=False, is_dir=True)) - for name in sorted(filenames): - file_path: Path = root_path / name - if file_path.is_symlink(): - continue - relative = file_path.relative_to(self._root).as_posix() - nodes.append(ProjectPath(value=relative, is_file=True, is_dir=False)) - return tuple(sorted(nodes)) - - @staticmethod - def _validate_relative(path: str) -> None: - candidate: Path = Path(path) - if candidate.is_absolute() or PARENT_DIRECTORY_TOKEN in candidate.parts: - raise RuleUsageError(f"project path escapes the project: {path}") - def public_model(model: object) -> Model: - from sqlbuild.compiler.compile.models import CompiledModel + from sqlbuild._native import CompiledModel if not isinstance(model, (CompiledModel, _native.NativeCompiledModel)): raise RuleUsageError("model is not a compiled SQL model") diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index 0fb4e32544..555d7db93e 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -5,6 +5,18 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + Finding, + Model, + ProjectPath, + Rule, + RuleFactViews, + RuleOption, + RulesConfig, + RuleUsageError, + SqlNode, + finding_position, +) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.classes.column_facts import ColumnFacts @@ -15,18 +27,6 @@ from sqlbuild.rule_engine.classes.model_test_facts import TestFacts from sqlbuild.rule_engine.classes.project_facts import ProjectFacts from sqlbuild.rule_engine.classes.sql_facts import SqlFacts -from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN -from sqlbuild.rule_engine.exceptions import RuleUsageError -from sqlbuild.rule_engine.models import ( - Finding, - Model, - ProjectPath, - Rule, - RuleFactViews, - RuleOption, - RulesConfig, - SqlNode, -) def build_rule_fact_views( @@ -114,14 +114,9 @@ def finding( path = Path(subject.value) else: path = Path(subject) - if path.is_absolute() or PARENT_DIRECTORY_TOKEN in path.parts: - raise RuleUsageError(f"finding path must be project-relative: {path}") - resolved_line: int = line if line is not None else int(getattr(node, "line", 1) or 1) - resolved_column: int = ( - column if column is not None else int(getattr(node, "column", 1) or 1) + resolved_line, resolved_column = finding_position( + path=path, line=line, column=column, node=node ) - if resolved_line < 1 or resolved_column < 1: - raise RuleUsageError("finding line and column must be positive") return Finding( code=self._rule.code, path=path, diff --git a/src/sqlbuild/rule_engine/classes/runtime_guard.py b/src/sqlbuild/rule_engine/classes/runtime_guard.py index 847fc4b1a2..03fc03ba1e 100644 --- a/src/sqlbuild/rule_engine/classes/runtime_guard.py +++ b/src/sqlbuild/rule_engine/classes/runtime_guard.py @@ -12,9 +12,8 @@ from pathlib import Path from types import CodeType, FrameType, FunctionType, ModuleType -from sqlbuild.rule_engine._helpers.engine.hermeticity import allowed_custom_rule_import +from sqlbuild._native import NonHermeticRuleError, allowed_custom_rule_import from sqlbuild.rule_engine.classes.project_tree import ProjectTree -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError _IMPORT_EVENT: str = "import" _OPEN_EVENT: str = "open" diff --git a/src/sqlbuild/rule_engine/classes/sql_document.py b/src/sqlbuild/rule_engine/classes/sql_document.py index 29456d171a..e609dfa8a7 100644 --- a/src/sqlbuild/rule_engine/classes/sql_document.py +++ b/src/sqlbuild/rule_engine/classes/sql_document.py @@ -2,10 +2,8 @@ from typing import Any +from sqlbuild._native import AST_CTE_KIND, RuleUsageError, SqlNode from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.rule_engine.constants import AST_CTE_KIND -from sqlbuild.rule_engine.exceptions import RuleUsageError -from sqlbuild.rule_engine.models import SqlNode class SqlDocument: diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index d55c346673..c600b94107 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -2,10 +2,9 @@ from functools import cached_property +from sqlbuild._native import Model, ModelSql, RuleUsageError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.sql_document import SqlDocument -from sqlbuild.rule_engine.exceptions import RuleUsageError -from sqlbuild.rule_engine.models import Model, ModelSql class SqlFacts: diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index c448dbf8f3..042f1b718e 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -1,20 +1,13 @@ """Rules engine constants.""" -CUSTOM_HOST_PROTOCOL_VERSION: int = 1 -CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 -CUSTOM_RULE_PROJECT_SUBJECT: str = "" -CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" -CUSTOM_HOST_ABANDONED_SUFFIX: str = ".abandoned" CUSTOM_HOST_INPUT_POLL_SECONDS: float = 0.002 CUSTOM_HOST_INPUT_WAIT_SECONDS: float = 120.0 CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" -CUSTOM_HOST_CANCELLED_MARKER: str = "cancelled" CUSTOM_HOST_CANCELLED_EXIT_CODE: int = 75 CUSTOM_HOST_CANCELLED_MESSAGE: str = "custom host cancelled after another host failed" -CUSTOM_HOST_LAUNCH_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host_launch" CUSTOM_HOST_HASH_SEED: str = "0" CUSTOM_HOST_EXEC_OS_NAME: str = "posix" CUSTOM_HOST_STARTUP_ENVIRONMENT: tuple[str, ...] = ( @@ -26,50 +19,8 @@ "PYTHONUSERBASE", "SYSTEMROOT", ) -CUSTOM_RULE_IMPORT_ROOTS: frozenset[str] = frozenset( - { - "__future__", - "abc", - "bisect", - "collections", - "copy", - "dataclasses", - "decimal", - "difflib", - "enum", - "fnmatch", - "fractions", - "functools", - "graphlib", - "hashlib", - "heapq", - "itertools", - "json", - "math", - "operator", - "pathlib", - "re", - "rules", - "sqlbuild.rules", - "statistics", - "string", - "textwrap", - "types", - "typing", - "unicodedata", - } -) - -AST_CTE_KIND: str = "cte" -CONTRACT_ENFORCED: str = "enforced" -TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) - - RULE_DECORATOR_TOKEN: str = "@rule" -INIT_MODULE_NAME: str = "__init__" -PARENT_DIRECTORY_TOKEN: str = ".." -MIN_CUSTOM_RULE_TEST_CASES: str = "min_custom_rule_test_cases" FACT_PROJECT_MODELS: str = "project.models" FACT_PROJECT_SOURCES: str = "project.sources" FACT_PROJECT_SEEDS: str = "project.seeds" diff --git a/src/sqlbuild/rule_engine/exceptions.py b/src/sqlbuild/rule_engine/exceptions.py deleted file mode 100644 index 5e32b7cb94..0000000000 --- a/src/sqlbuild/rule_engine/exceptions.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Rules public errors.""" - - -class RulesError(RuntimeError): - """Raised for invalid compiler rules or execution.""" - - -class RulesConfigError(RulesError): - """Raised when rules configuration breaks a project policy; help shows the setting to change.""" - - def __init__(self, message: str, *, help: str) -> None: - super().__init__(message) - self.message = message - self.help = help - - -class RuleUsageError(RulesError): - """Raised when a rule or rule test uses the public API incorrectly.""" - - -class NonHermeticRuleError(RulesError): - """Raised when a custom rule observes state the Rules cache cannot track.""" - - -class RuleAssertionError(RulesError): - """Raised when a custom-rule harness case produces an unexpected result.""" - - -class FactDigestError(RulesError): - """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" - - -class OpaqueModuleStateError(RulesError): - """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" - - -class HostCancelledError(RulesError): - """Raised in a custom-rule host that stops because another host already failed.""" diff --git a/src/sqlbuild/rule_engine/main/__init__.py b/src/sqlbuild/rule_engine/main/__init__.py index c2922b48c1..e69de29bb2 100644 --- a/src/sqlbuild/rule_engine/main/__init__.py +++ b/src/sqlbuild/rule_engine/main/__init__.py @@ -1 +0,0 @@ -"""Rules public entrypoints.""" diff --git a/src/sqlbuild/rule_engine/main/define.py b/src/sqlbuild/rule_engine/main/define.py index 38527a7c2f..9c60a48417 100644 --- a/src/sqlbuild/rule_engine/main/define.py +++ b/src/sqlbuild/rule_engine/main/define.py @@ -4,8 +4,8 @@ from collections.abc import Callable +from sqlbuild._native import RuleOption from sqlbuild.rule_engine._helpers.engine.definition import define_rule -from sqlbuild.rule_engine.models import RuleOption from sqlbuild.rule_engine.types import RuleCheck diff --git a/src/sqlbuild/rule_engine/main/evaluate_rule.py b/src/sqlbuild/rule_engine/main/evaluate_rule.py index d728597f83..de6def3a96 100644 --- a/src/sqlbuild/rule_engine/main/evaluate_rule.py +++ b/src/sqlbuild/rule_engine/main/evaluate_rule.py @@ -1,6 +1,6 @@ """Public custom rule test harness.""" -from sqlbuild.rule_engine.models import Rule, RuleCase, RuleResult +from sqlbuild._native import Rule, RuleCase, RuleResult from sqlbuild.rule_engine.types import RuleCheck diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py deleted file mode 100644 index 362c0a90b3..0000000000 --- a/src/sqlbuild/rule_engine/models.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Structured rules configuration, rule, and evaluation models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - GraphEdgeExclusion, - RuleExemption, - RuleGuidance, - RuleIgnore, - RulesCacheConfig, - SelectStarAllow, - SqlTestRulesConfig, - ThresholdOverride, -) -from sqlbuild.rule_engine.types import RuleCheck, RuleOptionValue, RuleSubject - - -@dataclass(frozen=True, slots=True) -class Model: - """Stable identity and common metadata for one compiled SQL model.""" - - name: str - path: Path - materialization: str | None - - -@dataclass(frozen=True, slots=True) -class Project: - """Stable summary for one compiled project invocation.""" - - model_count: int - target_name: str | None - - -@dataclass(frozen=True, slots=True, order=True) -class ProjectPath: - """One normalized project-relative file or directory.""" - - value: str - is_file: bool - is_dir: bool - - @property - def parts(self) -> tuple[str, ...]: - return Path(self.value).parts - - @property - def name(self) -> str: - return Path(self.value).name - - @property - def depth(self) -> int: - return len(self.parts) - - -@dataclass(frozen=True, slots=True) -class SqlNode: - """Stable source location for one common SQL structure fact.""" - - kind: str - line: int - column: int - - -@dataclass(frozen=True, slots=True) -class ModelSql: - """Authored and compiler-expanded SQL documents for one model.""" - - authored: Any - expanded: Any - - -@dataclass(frozen=True, slots=True) -class RuleFactViews: - """Views shared by every invocation in one custom-rule host evaluation.""" - - project: Any - sql: Any - graph: Any - columns: Any - contracts: Any - tests: Any - audits: Any - declarations: Any - - -@dataclass(frozen=True) -class RuleOption[T]: - """One typed configuration option declared by a rule.""" - - name: str - value_type: type[T] - default: T - description: str - choices: tuple[T, ...] = () - minimum: int | None = None - maximum: int | None = None - minimum_items: int | None = None - item_type: type[object] | None = None - - @classmethod - def boolean(cls, *, name: str, default: bool, description: str) -> RuleOption[bool]: - return RuleOption(name=name, value_type=bool, default=default, description=description) - - @classmethod - def integer( - cls, - *, - name: str, - default: int, - description: str, - minimum: int | None = None, - maximum: int | None = None, - ) -> RuleOption[int]: - return RuleOption( - name=name, - value_type=int, - default=default, - description=description, - minimum=minimum, - maximum=maximum, - ) - - @classmethod - def string( - cls, - *, - name: str, - default: str, - description: str, - choices: tuple[str, ...] = (), - ) -> RuleOption[str]: - return RuleOption( - name=name, - value_type=str, - default=default, - description=description, - choices=choices, - ) - - @classmethod - def string_list( - cls, - *, - name: str, - default: tuple[str, ...], - description: str, - minimum_items: int | None = None, - ) -> RuleOption[tuple[str, ...]]: - return RuleOption( - name=name, - value_type=tuple, - default=default, - description=description, - minimum_items=minimum_items, - item_type=str, - ) - - @classmethod - def integer_list( - cls, - *, - name: str, - default: tuple[int, ...], - description: str, - minimum_items: int | None = None, - ) -> RuleOption[tuple[int, ...]]: - return RuleOption( - name=name, - value_type=tuple, - default=default, - description=description, - minimum_items=minimum_items, - item_type=int, - ) - - -@dataclass(frozen=True) -class Rule: - """Immutable metadata and implementation for one rule.""" - - code: str - family: str - slug: str - message: str - remediation: str - check: RuleCheck - options: tuple[RuleOption[object], ...] = () - enabled_by_default: bool = False - custom: bool = False - source: str | None = None - subject: RuleSubject | None = None - subject_parameter: str | None = None - context_parameter: str | None = None - guidance: RuleGuidance | None = None - - @property - def project_wide(self) -> bool: - """Return whether this rule is invoked once for the compiled project.""" - - return self.subject is RuleSubject.PROJECT - - -@dataclass(frozen=True) -class Finding: - """One deterministic compiler rules violation.""" - - code: str - path: Path - line: int - column: int - message: str - remediation: str - unevaluated: bool = False - affected_rules: tuple[str, ...] = () - fixable: bool = False - - -@dataclass(frozen=True) -class LayoutConfig: - levels: tuple[str, ...] = ( - "staging", - "intermediate/clean", - "intermediate/enriched", - "mart", - ) - domain_roots: tuple[str, ...] = () - - -@dataclass(frozen=True) -class RulesConfig: - select: tuple[str, ...] = () - ignore: tuple[str, ...] = () - thresholds: dict[str, int] = field(default_factory=dict) - threshold_overrides: tuple[ThresholdOverride, ...] = () - rule_options: dict[str, dict[str, RuleOptionValue]] = field(default_factory=dict) - rule_exceptions: tuple[RuleExemption, ...] = () - graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] = () - rule_ignores: tuple[RuleIgnore, ...] = () - allow_exceptions: bool = True - select_star_allow: tuple[SelectStarAllow, ...] = () - domains: tuple[str, ...] = () - approved_source_tokens: tuple[str, ...] = () - retired_source_tokens: dict[str, str] = field(default_factory=dict) - sql_tests: SqlTestRulesConfig = field(default_factory=SqlTestRulesConfig) - layout: LayoutConfig = field(default_factory=LayoutConfig) - cache: RulesCacheConfig = field(default_factory=RulesCacheConfig) - - -@dataclass(frozen=True) -class RulesResult: - findings: tuple[Finding, ...] - evaluated_models: int - unevaluated_resources: int = 0 - cache_hits: int = 0 - cache_misses: int = 0 - built_in_ms: int = 0 - custom_ms: int = 0 - custom_cpu_ms: int = 0 - - -@dataclass(frozen=True) -class RuleFile: - path: str - source: str - - -@dataclass(frozen=True) -class RuleCase: - description: str - source: str - expected_finding_count: int - path: str = "models/mart/example__mart__result.sql" - files: tuple[RuleFile, ...] = () - config: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RuleResult: - findings: tuple[Finding, ...] - - @property - def finding_count(self) -> int: - return len(self.findings) - - -@dataclass(frozen=True) -class CustomRuleEvaluation: - """Findings and observed fact reads of one rule invocation for one subject.""" - - code: str - subject: str | None - findings: tuple[Finding, ...] - reads: frozenset[tuple[object, ...]] | None - - -@dataclass(frozen=True) -class ModuleStateTrace: - """What exact module-state walks touched: object and type identities and their decisions.""" - - touched: dict[int, int] - descended_types: frozenset[int] - checked_instances: frozenset[int] - - -@dataclass(frozen=True) -class ModuleStateFingerprint: - """One exact custom-rule module-state fingerprint and the trace of the walk behind it.""" - - token: str - trace: ModuleStateTrace - - -@dataclass(frozen=True) -class CustomRuleRun: - """Every invocation of one host run plus rules whose reads could not be attributed.""" - - evaluations: tuple[CustomRuleEvaluation, ...] - untracked_codes: frozenset[str] - uncacheable_codes: frozenset[str] = frozenset() - observed: tuple[tuple[tuple[str, ...], str | None], ...] = () - stateful_codes: frozenset[str] = frozenset() - - -@dataclass(frozen=True) -class CustomHostPartition: - """How one host of a split run detects module state and learns the run was cancelled.""" - - cancelled: Callable[[], bool] diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index ccaedcbba2..fec5a01fac 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -3,11 +3,11 @@ from __future__ import annotations from collections.abc import Callable -from enum import StrEnum from pathlib import Path from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: + from sqlbuild._native import Finding, Model, ProjectPath, RuleOption, SqlNode from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.classes.column_facts import ColumnFacts from sqlbuild.rule_engine.classes.contract_facts import ContractFacts @@ -16,26 +16,12 @@ from sqlbuild.rule_engine.classes.model_test_facts import TestFacts from sqlbuild.rule_engine.classes.project_facts import ProjectFacts from sqlbuild.rule_engine.classes.sql_facts import SqlFacts - from sqlbuild.rule_engine.models import ( - Finding, - Model, - ProjectPath, - RuleOption, - SqlNode, - ) type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] type FactKey = tuple[str, ...] type CustomRulePlan = dict[str, frozenset[str] | None] -class RuleSubject(StrEnum): - """Compiler-owned unit used to invoke and cache one custom rule.""" - - MODEL = "model" - PROJECT = "project" - - class RuleContext(Protocol): """Stable typed compiler-fact views available to custom rules.""" diff --git a/src/sqlbuild/rules/__init__.py b/src/sqlbuild/rules/__init__.py index 583061e6ac..1ae437ada4 100644 --- a/src/sqlbuild/rules/__init__.py +++ b/src/sqlbuild/rules/__init__.py @@ -2,9 +2,7 @@ from collections.abc import Callable -from sqlbuild.rule_engine.main.define import rule as _rule -from sqlbuild.rule_engine.main.evaluate_rule import evaluate_rule -from sqlbuild.rule_engine.models import ( +from sqlbuild._native import ( Finding, Model, Project, @@ -15,6 +13,8 @@ RuleResult, SqlNode, ) +from sqlbuild.rule_engine.main.define import rule as _rule +from sqlbuild.rule_engine.main.evaluate_rule import evaluate_rule from sqlbuild.rule_engine.types import RuleCheck, RuleContext __all__ = ( diff --git a/src/sqlbuild/rules/testing.py b/src/sqlbuild/rules/testing.py index 0efe82f090..e316fe23f7 100644 --- a/src/sqlbuild/rules/testing.py +++ b/src/sqlbuild/rules/testing.py @@ -1,6 +1,6 @@ """Public test harness for repository-owned compiler rules.""" +from sqlbuild._native import RuleCase, RuleFile, RuleResult from sqlbuild.rule_engine.main.evaluate_rule import evaluate_rule -from sqlbuild.rule_engine.models import RuleCase, RuleFile, RuleResult __all__ = ("RuleCase", "RuleFile", "RuleResult", "evaluate_rule") diff --git a/src/sqlbuild/runtime/__init__.py b/src/sqlbuild/runtime/__init__.py index 68003a5128..e69de29bb2 100644 --- a/src/sqlbuild/runtime/__init__.py +++ b/src/sqlbuild/runtime/__init__.py @@ -1 +0,0 @@ -"""Runtime integration contracts.""" diff --git a/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py b/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py index 45e6f11387..cc2b396af6 100644 --- a/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py +++ b/src/sqlbuild/runtime/compute_logs/classes/diagnostic_log_handler.py @@ -7,35 +7,11 @@ from datetime import UTC, datetime from functools import cache from importlib.metadata import PackageNotFoundError, version -from typing import cast +from typing import Any, cast -from sqlbuild.diagnostics.classes.diagnostic_record_redactor import DiagnosticRecordRedactor -from sqlbuild.diagnostics.constants import ( - LOGGER_PREFIX, - LOGGER_ROOT_NAME, - REDACTED_SQL_LOG_MESSAGE, - SQL_DIGEST_FIELD, - SQL_PRIVATE_RECORD_FIELD, - SQL_PRIVATE_RECORD_MARKER, -) -from sqlbuild.runtime.compute_logs.constants import SQL_LOG_RECORD_FIELD +from sqlbuild._native import CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, compute_diagnostic_log_fields from sqlbuild.runtime.compute_logs.types import ComputeLogStorage -from sqlbuild.runtime.observability.constants import CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION from sqlbuild.runtime.observability.models import DiagnosticLog -from sqlbuild.runtime.observability.types import JSONValue - -_IDENTITY_FIELDS: tuple[str, ...] = ( - "invocation_id", - "run_id", - "resource_id", - "resource_attempt_id", - "operation_id", - "statement_id", - "log_stream_id", -) -_SQL_SAFE_FIELDS: frozenset[str] = frozenset( - ("sql_action", "sql_digest", "sql_intent", "sql_query_id") -) class ComputeDiagnosticLogHandler(logging.Handler): @@ -59,71 +35,21 @@ def __init__( def emit(self, record: logging.LogRecord) -> None: """Convert eligible records while excluding SQL and arbitrary parameter fields.""" - if self._failed or not isinstance(record.name, str): - return - internal: bool = record.name == LOGGER_ROOT_NAME or record.name.startswith(LOGGER_PREFIX) - if not internal and record.levelno < logging.INFO: + if self._failed: return - contains_sql: bool = ( - SQL_DIGEST_FIELD in record.__dict__ or SQL_LOG_RECORD_FIELD in record.__dict__ - ) - trusted_private_sql: bool = ( - record.__dict__.get(SQL_PRIVATE_RECORD_FIELD) is SQL_PRIVATE_RECORD_MARKER + fields: dict[str, object] | None = compute_diagnostic_log_fields( + record=record, + invocation_id=self._invocation_id, + include_sql_text=self._include_sql_text, ) - if self._include_sql_text and contains_sql and SQL_LOG_RECORD_FIELD not in record.__dict__: + if fields is None: return - structured: dict[str, object] = { - key: value - for key, value in record.__dict__.items() - if key.startswith("sqlbuild_") and key != SQL_LOG_RECORD_FIELD - } - identities: dict[str, str | None] = {} - for field_name in _IDENTITY_FIELDS: - raw_value: object = structured.get(f"sqlbuild_{field_name}") - identities[field_name] = str(raw_value) if raw_value is not None else None - identities["invocation_id"] = identities["invocation_id"] or self._invocation_id - fields: dict[str, JSONValue] = {} - for safe_name in ( - "sql_action", - "sql_digest", - "sql_intent", - "sql_query_id", - "channel", - "subscriber", - "error_type", - ): - raw_value = structured.get(f"sqlbuild_{safe_name}") - safe_value: object = DiagnosticRecordRedactor.value(name=safe_name, value=raw_value) - if isinstance(safe_value, (str, int, float, bool)) or safe_value is None: - if safe_value is not None: - fields[safe_name] = safe_value try: - message: str = ( - REDACTED_SQL_LOG_MESSAGE - if contains_sql - else DiagnosticRecordRedactor.text(record.getMessage()) - ) - logger_name: str = "sqlbuild.sql" if contains_sql else record.name - if contains_sql: - fields = {key: value for key, value in fields.items() if key in _SQL_SAFE_FIELDS} - if not trusted_private_sql: - identities = {field_name: None for field_name in _IDENTITY_FIELDS} - identities["invocation_id"] = self._invocation_id - if self._include_sql_text and SQL_LOG_RECORD_FIELD in record.__dict__: - fields["sql_text"] = str(record.__dict__[SQL_LOG_RECORD_FIELD]) - elif not internal: - fields = cast(dict[str, JSONValue], DiagnosticRecordRedactor.extras(record)) diagnostic: DiagnosticLog = DiagnosticLog( schema_version=CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, - producer="sqlbuild", producer_version=_sqlbuild_version(), occurred_at=datetime.fromtimestamp(record.created, tz=UTC), - severity=_severity(record.levelno), - logger=logger_name, - source="python_logging", - message=message, - fields=fields, - **identities, + **cast(dict[str, Any], fields), ) except Exception: return @@ -135,18 +61,6 @@ def emit(self, record: logging.LogRecord) -> None: self._failure_callback(error) -def _severity(level: int) -> str: - if level >= logging.CRITICAL: - return "critical" - if level >= logging.ERROR: - return "error" - if level >= logging.WARNING: - return "warning" - if level >= logging.INFO: - return "info" - return "debug" - - @cache def _sqlbuild_version() -> str: try: diff --git a/src/sqlbuild/runtime/compute_logs/constants.py b/src/sqlbuild/runtime/compute_logs/constants.py deleted file mode 100644 index 4b6f540a00..0000000000 --- a/src/sqlbuild/runtime/compute_logs/constants.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Compute log format and policy constants.""" - -COMPUTE_LOG_FORMAT_VERSION: int = 1 -DEFAULT_RETENTION_COUNT: int = 20 -MAX_READ_BYTES: int = 1_048_576 -METADATA_FILE_NAME: str = "metadata.json" -COMPLETE_FILE_NAME: str = "complete" -PRUNE_LOCK_FILE_NAME: str = ".prune.lock" -COMPLETED_AT_FIELD: str = "completed_at" -SQL_LOG_RECORD_FIELD: str = "sqlbuild_sql" -REDACTED_SQL_LOG_MESSAGE: str = "SQL diagnostic omitted" diff --git a/src/sqlbuild/runtime/compute_logs/types.py b/src/sqlbuild/runtime/compute_logs/types.py index 3de00c2f04..f91a061af0 100644 --- a/src/sqlbuild/runtime/compute_logs/types.py +++ b/src/sqlbuild/runtime/compute_logs/types.py @@ -6,7 +6,7 @@ from types import TracebackType from typing import TYPE_CHECKING, Protocol, Self -from sqlbuild.runtime.compute_logs.constants import MAX_READ_BYTES +from sqlbuild._native import MAX_READ_BYTES if TYPE_CHECKING: from sqlbuild.runtime.compute_logs.models import ( diff --git a/src/sqlbuild/runtime/contracts/__init__.py b/src/sqlbuild/runtime/contracts/__init__.py index 66ed421c75..e69de29bb2 100644 --- a/src/sqlbuild/runtime/contracts/__init__.py +++ b/src/sqlbuild/runtime/contracts/__init__.py @@ -1 +0,0 @@ -"""Cross-domain runtime contracts.""" diff --git a/src/sqlbuild/runtime/event_exporting/constants.py b/src/sqlbuild/runtime/event_exporting/constants.py index 227df104bd..1467e799be 100644 --- a/src/sqlbuild/runtime/event_exporting/constants.py +++ b/src/sqlbuild/runtime/event_exporting/constants.py @@ -1,7 +1,7 @@ """Lifecycle-event sink declaration constants.""" +from sqlbuild._native import EventExportSeverity from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.spec.contracts.types import EventExportSeverity EVENT_EXPORTER_EVENT_PARAMETER_NAME: str = "event" EVENT_EXPORT_SEVERITIES: tuple[str, ...] = tuple(item.value for item in EventExportSeverity) diff --git a/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py b/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py index 6b3503a88c..d3a8db86bd 100644 --- a/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py +++ b/src/sqlbuild/runtime/event_exporting/main/lifecycle_event_sink.py @@ -6,6 +6,7 @@ from collections.abc import Callable, Iterable from typing import Any, cast +from sqlbuild._native import EventExportSeverity from sqlbuild.python_nodes.main.attach_definition import attach_definition from sqlbuild.runtime.event_exporting.constants import ( EVENT_EXPORT_KINDS, @@ -15,7 +16,6 @@ from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError from sqlbuild.runtime.event_exporting.models import LifecycleEventSinkDefinition from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.spec.contracts.types import EventExportSeverity def lifecycle_event_sink( diff --git a/src/sqlbuild/runtime/event_exporting/models.py b/src/sqlbuild/runtime/event_exporting/models.py index a75ce68508..6f2f6e6725 100644 --- a/src/sqlbuild/runtime/event_exporting/models.py +++ b/src/sqlbuild/runtime/event_exporting/models.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.types import EventExportSeverity +from sqlbuild._native import EventExportSeverity @dataclass(frozen=True) diff --git a/src/sqlbuild/runtime/execution_history/constants.py b/src/sqlbuild/runtime/execution_history/constants.py deleted file mode 100644 index 06f7e8cad7..0000000000 --- a/src/sqlbuild/runtime/execution_history/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Execution history contract limits and schema defaults.""" - -DEFAULT_PAGE_LIMIT: int = 100 -MAX_PAGE_LIMIT: int = 1000 -CURRENT_EVENT_LOG_SCHEMA_VERSION: int = 1 -CURRENT_RUN_STORAGE_SCHEMA_VERSION: int = 1 -RUN_STARTED_EVENT_TYPE: str = "run_started" -RUN_COMPLETED_EVENT_TYPE: str = "run_completed" -RUN_FAILED_EVENT_TYPE: str = "run_failed" diff --git a/src/sqlbuild/runtime/execution_history/types.py b/src/sqlbuild/runtime/execution_history/types.py index b74bbd3984..5349b10614 100644 --- a/src/sqlbuild/runtime/execution_history/types.py +++ b/src/sqlbuild/runtime/execution_history/types.py @@ -7,7 +7,7 @@ from types import TracebackType from typing import TYPE_CHECKING, Protocol, Self -from sqlbuild.runtime.execution_history.constants import DEFAULT_PAGE_LIMIT +from sqlbuild._native import DEFAULT_PAGE_LIMIT if TYPE_CHECKING: from sqlbuild.runtime.execution_history.models import ( diff --git a/src/sqlbuild/runtime/observability/_helpers/factory.py b/src/sqlbuild/runtime/observability/_helpers/factory.py index b1bc0e5631..0901ade0ca 100644 --- a/src/sqlbuild/runtime/observability/_helpers/factory.py +++ b/src/sqlbuild/runtime/observability/_helpers/factory.py @@ -7,11 +7,11 @@ from types import MappingProxyType from uuid import uuid4 +from sqlbuild._native import CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, validate_lifecycle_producer from sqlbuild.runtime.observability._helpers.identity import ( current_execution_identity, current_invocation_external_context, ) -from sqlbuild.runtime.observability.constants import CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity, LifecycleEvent from sqlbuild.runtime.observability.types import JSONValue @@ -33,10 +33,7 @@ def create_lifecycle_event( raise ObservabilityValidationError( "cannot create a lifecycle event without an active invocation identity" ) - if producer != _DEFAULT_PRODUCER and producer_version is None: - raise ObservabilityValidationError( - "producer_version is required when producer is not 'sqlbuild'" - ) + validate_lifecycle_producer(producer=producer, producer_version=producer_version) return LifecycleEvent( event_id=uuid4().hex if event_id is None else event_id, event_type=event_type, diff --git a/src/sqlbuild/runtime/observability/_helpers/identity.py b/src/sqlbuild/runtime/observability/_helpers/identity.py index 599e68933e..58f4436675 100644 --- a/src/sqlbuild/runtime/observability/_helpers/identity.py +++ b/src/sqlbuild/runtime/observability/_helpers/identity.py @@ -10,7 +10,7 @@ from typing import cast from uuid import uuid4 -from sqlbuild.runtime.observability._helpers.validation import freeze_json +from sqlbuild._native import freeze_lifecycle_json from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity from sqlbuild.runtime.observability.types import JSONValue @@ -73,7 +73,7 @@ def invocation_external_context_scope( ) -> Iterator[Mapping[str, JSONValue]]: """Install validated integration context for lifecycle facts in one command.""" - frozen: JSONValue = freeze_json(value=external_context, path="external_context") + frozen: JSONValue = freeze_lifecycle_json(value=external_context, path="external_context") if not isinstance(frozen, Mapping): raise ObservabilityValidationError("external_context must be a JSON object") context: Mapping[str, JSONValue] = cast(Mapping[str, JSONValue], frozen) diff --git a/src/sqlbuild/runtime/observability/_helpers/observability.py b/src/sqlbuild/runtime/observability/_helpers/observability.py index 80c5c8cdf6..81ce115539 100644 --- a/src/sqlbuild/runtime/observability/_helpers/observability.py +++ b/src/sqlbuild/runtime/observability/_helpers/observability.py @@ -7,19 +7,21 @@ from datetime import datetime from typing import Any, cast -from sqlbuild.runtime.observability._helpers.validation import validate_schema_version -from sqlbuild.runtime.observability.constants import ( +from sqlbuild._native import ( CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, + lifecycle_event_catalogued, + lifecycle_event_terminal, + validate_lifecycle_schema_version, +) +from sqlbuild.runtime.observability.constants import ( DIAGNOSTIC_ENVELOPE_FIELDS, LIFECYCLE_ENVELOPE_FIELDS_BY_VERSION, - LIFECYCLE_EVENT_CATALOGS, LIFECYCLE_INVOCATION_METADATA_SCHEMA_VERSION, ) from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ( DiagnosticLog, LifecycleEvent, - LifecycleEventDefinition, OpaqueLifecycleEvent, ) @@ -96,16 +98,9 @@ def lifecycle_event_from_json(raw_json: str) -> LifecycleEvent | OpaqueLifecycle data: dict[str, Any] = _loads_object(raw_json=raw_json, envelope_name="lifecycle event") event_type: object = data.get("event_type") schema_version_raw: Any = data.get("schema_version") - validate_schema_version(value=schema_version_raw) + validate_lifecycle_schema_version(value=schema_version_raw) schema_version: int = cast(int, schema_version_raw) - version_catalog: Mapping[str, LifecycleEventDefinition] | None = LIFECYCLE_EVENT_CATALOGS.get( - schema_version - ) - if ( - version_catalog is None - or not isinstance(event_type, str) - or event_type not in version_catalog - ): + if not lifecycle_event_catalogued(schema_version=schema_version, event_type=event_type): return OpaqueLifecycleEvent(raw=data) _reject_unknown_fields( data=data, @@ -119,7 +114,7 @@ def lifecycle_event_from_json(raw_json: str) -> LifecycleEvent | OpaqueLifecycle occurred_at: datetime = datetime.fromisoformat(occurred_at_raw.replace("Z", "+00:00")) return LifecycleEvent( event_id=data["event_id"], - event_type=event_type, + event_type=cast(str, event_type), schema_version=schema_version, producer=data["producer"], producer_version=data["producer_version"], @@ -176,7 +171,7 @@ def diagnostic_log_from_json(raw_json: str) -> DiagnosticLog: data: dict[str, Any] = _loads_object(raw_json=raw_json, envelope_name="diagnostic log") schema_version_raw: Any = data.get("schema_version") - validate_schema_version(value=schema_version_raw) + validate_lifecycle_schema_version(value=schema_version_raw) schema_version: int = cast(int, schema_version_raw) if schema_version == CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: _reject_unknown_fields( @@ -219,7 +214,9 @@ def diagnostic_log_from_json(raw_json: str) -> DiagnosticLog: def is_terminal_event(event: LifecycleEvent) -> bool: """Return whether the fact closes its correlated lifecycle scope.""" - return LIFECYCLE_EVENT_CATALOGS[event.schema_version][event.event_type].terminal + return lifecycle_event_terminal( + schema_version=event.schema_version, event_type=event.event_type + ) def validate_idempotent_duplicate(*, original: LifecycleEvent, duplicate: LifecycleEvent) -> None: diff --git a/src/sqlbuild/runtime/observability/_helpers/validation.py b/src/sqlbuild/runtime/observability/_helpers/validation.py deleted file mode 100644 index 1dac7d85e3..0000000000 --- a/src/sqlbuild/runtime/observability/_helpers/validation.py +++ /dev/null @@ -1,410 +0,0 @@ -"""Validation support for immutable observability envelopes.""" - -from __future__ import annotations - -import json -import math -import re -from collections.abc import Mapping -from datetime import UTC, datetime -from types import MappingProxyType - -from sqlbuild.runtime.observability.constants import ( - AUDIT_RUN_COUNT_FIELDS, - BOOLEAN_PAYLOAD_FIELDS, - CONFIGURED_CONCURRENCY_FIELD, - DURATION_MS_FIELD, - EXIT_CODE_FIELD, - FINITE_NUMBER_PAYLOAD_FIELDS, - FORBIDDEN_STATEMENT_PAYLOAD_FIELDS, - HOOK_PHASES, - HOOK_TYPES, - JSON_PAYLOAD_FIELDS, - LIFECYCLE_EVENT_CATALOGS, - MAX_METADATA_BYTES, - METADATA_FIELD, - MICROBATCH_EVENT_PREFIX, - MICROBATCH_KINDS, - MICROBATCH_RUN_TYPES, - MICROBATCH_STRATEGIES, - NONNEGATIVE_INTEGER_PAYLOAD_FIELDS, - OPERATION_ADAPTERS, - OPERATION_EVENT_PREFIX, - OPERATION_KINDS, - OPERATION_METADATA_FIELDS, - OPERATION_NAMES, - OPERATION_PHASES, - OPERATION_SCOPES, - OPERATION_STRATEGIES, - OPERATION_TARGET_KINDS, - RESOURCE_ATTEMPT_SKIPPED_EVENT, - RESOURCE_SKIP_CODES, - RESOURCE_SKIP_MODES, - RETRY_SCHEDULED_EVENT, - RUN_STARTED_EVENT, - RUN_TERMINALS, - STATEMENT_EVENT_PREFIX, - STRING_PAYLOAD_FIELDS, -) -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -from sqlbuild.runtime.observability.models import LifecycleEvent, LifecycleEventDefinition -from sqlbuild.runtime.observability.types import JSONValue - -_HOOK_NAME_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") - - -def validate_schema_version(*, value: object) -> None: - """Validate the shared positive integer schema-version contract.""" - - if isinstance(value, bool) or not isinstance(value, int) or value <= 0: - raise ObservabilityValidationError( - "schema_version must be a positive integer excluding bool" - ) - - -def freeze_json(*, value: object, path: str) -> JSONValue: - """Validate and recursively freeze a JSON-compatible value.""" - - return _freeze_json(value=value, path=(path,)) - - -def _freeze_json(*, value: object, path: tuple[str | int, ...]) -> JSONValue: - if value is None or isinstance(value, (str, bool, int)): - return value - if isinstance(value, float): - if not math.isfinite(value): - raise ObservabilityValidationError( - f"{_render_path(path)} must not contain NaN or infinity" - ) - return value - if isinstance(value, Mapping): - frozen: dict[str, JSONValue] = {} - for key, item in value.items(): - if not isinstance(key, str): - raise ObservabilityValidationError(f"{_render_path(path)} keys must be strings") - frozen[key] = ( - item - if item is None or isinstance(item, (str, bool, int)) - else _freeze_json(value=item, path=(*path, key)) - ) - return MappingProxyType(frozen) - if isinstance(value, (list, tuple)): - return tuple( - _freeze_json(value=item, path=(*path, index)) for index, item in enumerate(value) - ) - raise ObservabilityValidationError( - f"{_render_path(path)} contains non-JSON value of type {type(value).__name__}" - ) - - -def _render_path(path: tuple[str | int, ...]) -> str: - return "".join( - f"[{segment}]" if isinstance(segment, int) else f".{segment}" if index else segment - for index, segment in enumerate(path) - ) - - -def validate_required_text(*, value: object, field_name: str) -> None: - """Validate one required non-empty string field.""" - - if not isinstance(value, str) or not value: - raise ObservabilityValidationError(f"{field_name} must be a non-empty string") - - -def validate_optional_text(*, value: object, field_name: str) -> None: - """Validate one optional non-empty string field.""" - - if value is not None: - validate_required_text(value=value, field_name=field_name) - - -def validate_timestamp(*, value: datetime) -> None: - """Validate a timezone-aware UTC timestamp.""" - - if not isinstance(value, datetime) or value.tzinfo is None: - raise ObservabilityValidationError("occurred_at must be a timezone-aware UTC datetime") - if value.utcoffset() != UTC.utcoffset(value): - raise ObservabilityValidationError("occurred_at must use UTC") - - -def _forbidden_statement_fields(*, value: object) -> set[str]: - if isinstance(value, Mapping): - found: set[str] = { - key - for key in value - if isinstance(key, str) and key in FORBIDDEN_STATEMENT_PAYLOAD_FIELDS - } - for item in value.values(): - found.update(_forbidden_statement_fields(value=item)) - return found - if isinstance(value, (list, tuple)): - found = set() - for item in value: - found.update(_forbidden_statement_fields(value=item)) - return found - return set() - - -def _plain_json(*, value: object) -> object: - if isinstance(value, Mapping): - return {key: _plain_json(value=item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_plain_json(value=item) for item in value] - return value - - -def _validate_payload_field(*, field_name: str, value: object) -> None: - if field_name in STRING_PAYLOAD_FIELDS: - if not isinstance(value, str) or not value: - raise ObservabilityValidationError( - f"payload field {field_name!r} must be a non-empty string" - ) - return - if field_name in NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ObservabilityValidationError( - f"payload field {field_name!r} must be a nonnegative integer excluding bool" - ) - return - if field_name in FINITE_NUMBER_PAYLOAD_FIELDS: - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - ): - raise ObservabilityValidationError( - f"payload field {field_name!r} must be a finite number excluding bool" - ) - return - if field_name in BOOLEAN_PAYLOAD_FIELDS: - if not isinstance(value, bool): - raise ObservabilityValidationError(f"payload field {field_name!r} must be a boolean") - return - if field_name in JSON_PAYLOAD_FIELDS: - return - if field_name == DURATION_MS_FIELD: - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - or value < 0 - ): - raise ObservabilityValidationError( - "payload field 'duration_ms' must be a nonnegative finite number excluding bool" - ) - return - if field_name == EXIT_CODE_FIELD: - if isinstance(value, bool) or not isinstance(value, int): - raise ObservabilityValidationError( - "payload field 'exit_code' must be an integer excluding bool" - ) - return - if field_name == METADATA_FIELD: - if not isinstance(value, Mapping): - raise ObservabilityValidationError("payload field 'metadata' must be a JSON object") - encoded: bytes = json.dumps( - _plain_json(value=value), - ensure_ascii=True, - allow_nan=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8") - if len(encoded) > MAX_METADATA_BYTES: - raise ObservabilityValidationError( - f"payload field 'metadata' must encode to at most {MAX_METADATA_BYTES} bytes" - ) - - -def _validate_operation_payload(*, payload: Mapping[str, JSONValue]) -> None: - operation_kind: JSONValue | None = payload.get("operation_kind") - if operation_kind is not None and ( - not isinstance(operation_kind, str) or operation_kind not in OPERATION_KINDS - ): - raise ObservabilityValidationError("operation_kind must be a catalogued value") - operation_name: JSONValue | None = payload.get("operation_name") - if operation_name is not None and ( - not isinstance(operation_name, str) or operation_name not in OPERATION_NAMES - ): - raise ObservabilityValidationError("operation_name must be a catalogued value") - for field_name, allowed in ( - ("phase", OPERATION_PHASES), - ("strategy", OPERATION_STRATEGIES), - ("target_kind", OPERATION_TARGET_KINDS), - ("adapter", OPERATION_ADAPTERS), - ("scope", OPERATION_SCOPES), - ): - value: JSONValue | None = payload.get(field_name) - if value is not None and (not isinstance(value, str) or value not in allowed): - raise ObservabilityValidationError(f"{field_name} must be a catalogued value") - hook_phase: JSONValue | None = payload.get("hook_phase") - hook_type: JSONValue | None = payload.get("hook_type") - hook_index: JSONValue | None = payload.get("hook_index") - hook_name: JSONValue | None = payload.get("hook_name") - hook_fields_present: bool = any( - value is not None for value in (hook_phase, hook_type, hook_index, hook_name) - ) - if hook_fields_present: - if hook_phase not in HOOK_PHASES: - raise ObservabilityValidationError("hook_phase must be a catalogued value") - if hook_type not in HOOK_TYPES: - raise ObservabilityValidationError("hook_type must be a catalogued value") - if isinstance(hook_index, bool) or not isinstance(hook_index, int) or hook_index < 0: - raise ObservabilityValidationError("hook_index must be a nonnegative integer") - if hook_name is not None and ( - not isinstance(hook_name, str) or _HOOK_NAME_PATTERN.fullmatch(hook_name) is None - ): - raise ObservabilityValidationError("hook_name must be a non-empty safe token") - metadata: JSONValue | None = payload.get(METADATA_FIELD) - if metadata is None: - return - if not isinstance(metadata, Mapping): - raise ObservabilityValidationError("operation metadata must be a JSON object") - if set(metadata) - OPERATION_METADATA_FIELDS: - raise ObservabilityValidationError("operation metadata contains a non-allowlisted field") - for metadata_value in metadata.values(): - if ( - isinstance(metadata_value, bool) - or not isinstance(metadata_value, int) - or metadata_value < 0 - ): - raise ObservabilityValidationError( - "operation metadata values must be nonnegative integers excluding bool" - ) - - -def validate_known_lifecycle_event(*, event: LifecycleEvent) -> None: - """Validate correlations and safe payload fields for a catalogued event.""" - - version_catalog: Mapping[str, LifecycleEventDefinition] | None = LIFECYCLE_EVENT_CATALOGS.get( - event.schema_version - ) - definition: LifecycleEventDefinition | None = ( - version_catalog.get(event.event_type) if version_catalog is not None else None - ) - if definition is None: - raise ObservabilityValidationError( - f"unknown lifecycle event_type {event.event_type!r}; decode it as an opaque envelope" - ) - missing: list[str] = sorted( - field_name - for field_name in definition.required_correlations - if getattr(event, field_name) is None - ) - if missing: - raise ObservabilityValidationError( - f"known event {event.event_type!r} requires correlation field(s): {', '.join(missing)}" - ) - missing_payload: list[str] = sorted(definition.required_payload_fields - set(event.payload)) - if missing_payload: - raise ObservabilityValidationError( - f"known event {event.event_type!r} requires payload field(s): " - f"{', '.join(missing_payload)}" - ) - unexpected: list[str] = sorted(set(event.payload) - definition.allowed_payload_fields) - if unexpected: - allowed: str = ", ".join(sorted(definition.allowed_payload_fields)) or "none" - raise ObservabilityValidationError( - f"payload field(s) not allowed for known event {event.event_type!r}: " - f"{', '.join(unexpected)}; allowed fields: {allowed}" - ) - if event.event_type.startswith(STATEMENT_EVENT_PREFIX): - forbidden: list[str] = sorted(_forbidden_statement_fields(value=event.payload)) - if forbidden: - raise ObservabilityValidationError( - f"statement lifecycle payload must not contain SQL or parameter values; " - f"forbidden field(s): {', '.join(forbidden)}" - ) - if event.event_type.startswith(OPERATION_EVENT_PREFIX): - _validate_operation_payload(payload=event.payload) - if event.event_type.startswith(MICROBATCH_EVENT_PREFIX): - _validate_microbatch_payload(payload=event.payload) - if event.event_type == RESOURCE_ATTEMPT_SKIPPED_EVENT: - if event.payload.get("skip_code") not in RESOURCE_SKIP_CODES: - raise ObservabilityValidationError("skip_code must be a catalogued value") - skip_mode: JSONValue | None = event.payload.get("skip_mode") - if skip_mode is not None and skip_mode not in RESOURCE_SKIP_MODES: - raise ObservabilityValidationError("skip_mode must be a catalogued value") - if event.event_type == RETRY_SCHEDULED_EVENT: - failed_attempt: JSONValue | None = event.payload.get("failed_attempt_number") - next_attempt: JSONValue | None = event.payload.get("next_attempt_number") - if ( - not isinstance(failed_attempt, int) - or isinstance(failed_attempt, bool) - or failed_attempt <= 0 - ): - raise ObservabilityValidationError("failed_attempt_number must be a positive integer") - if ( - not isinstance(next_attempt, int) - or isinstance(next_attempt, bool) - or next_attempt != failed_attempt + 1 - ): - raise ObservabilityValidationError( - "next_attempt_number must equal failed_attempt_number + 1" - ) - for field_name, value in event.payload.items(): - _validate_payload_field(field_name=field_name, value=value) - _validate_run_payload(event=event) - - -def _validate_microbatch_payload(*, payload: Mapping[str, JSONValue]) -> None: - batch_index: JSONValue | None = payload.get("batch_index") - batch_count: JSONValue | None = payload.get("batch_count") - if isinstance(batch_index, bool) or not isinstance(batch_index, int) or batch_index < 1: - raise ObservabilityValidationError("batch_index must be a positive integer") - if isinstance(batch_count, bool) or not isinstance(batch_count, int) or batch_count < 1: - raise ObservabilityValidationError("batch_count must be a positive integer") - if batch_index > batch_count: - raise ObservabilityValidationError("batch_index must not exceed batch_count") - for field_name, allowed in ( - ("microbatch_strategy", MICROBATCH_STRATEGIES), - ("microbatch_run_type", MICROBATCH_RUN_TYPES), - ("batch_kind", MICROBATCH_KINDS), - ): - value: JSONValue | None = payload.get(field_name) - if not isinstance(value, str) or value not in allowed: - raise ObservabilityValidationError(f"{field_name} must be a catalogued value") - - -def _validate_run_payload(*, event: LifecycleEvent) -> None: - payload: Mapping[str, JSONValue] = event.payload - if event.event_type == RUN_STARTED_EVENT and CONFIGURED_CONCURRENCY_FIELD in payload: - configured: JSONValue = payload[CONFIGURED_CONCURRENCY_FIELD] - selected: JSONValue | None = payload.get("selected_count") - workers: JSONValue | None = payload.get("worker_count") - if not isinstance(configured, int) or isinstance(configured, bool) or configured < 1: - raise ObservabilityValidationError( - "configured_concurrency must be a positive integer excluding bool" - ) - if not isinstance(selected, int) or isinstance(selected, bool) or selected < 0: - raise ObservabilityValidationError( - "selected_count must be a nonnegative integer excluding bool" - ) - if not isinstance(workers, int) or isinstance(workers, bool): - raise ObservabilityValidationError("worker_count must be an integer excluding bool") - if workers != min(configured, selected): - raise ObservabilityValidationError( - "worker_count must equal min(configured_concurrency, selected_count)" - ) - if event.event_type in RUN_TERMINALS and AUDIT_RUN_COUNT_FIELDS & set(payload): - if not AUDIT_RUN_COUNT_FIELDS <= set(payload): - raise ObservabilityValidationError( - "audit run terminals must include pass_count, warn_count, and fail_count together" - ) - pass_count: int = _required_run_count(payload=payload, field_name="pass_count") - warn_count: int = _required_run_count(payload=payload, field_name="warn_count") - fail_count: int = _required_run_count(payload=payload, field_name="fail_count") - succeeded_count: int = _required_run_count(payload=payload, field_name="succeeded_count") - failed_count: int = _required_run_count(payload=payload, field_name="failed_count") - if succeeded_count != pass_count + warn_count: - raise ObservabilityValidationError("succeeded_count must equal pass_count + warn_count") - if failed_count != fail_count: - raise ObservabilityValidationError("failed_count must equal fail_count") - - -def _required_run_count(*, payload: Mapping[str, JSONValue], field_name: str) -> int: - value: JSONValue | None = payload.get(field_name) - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ObservabilityValidationError( - f"{field_name} must be a nonnegative integer excluding bool" - ) - return value diff --git a/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py b/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py index 32568b62cb..89431b62b8 100644 --- a/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/operation_lifecycle.py @@ -10,6 +10,17 @@ from types import TracebackType from typing import Any +from sqlbuild._native import ( + OPERATION_ADAPTERS, + OPERATION_KINDS, + OPERATION_NAMES, + OPERATION_PHASES, + OPERATION_SCOPES, + OPERATION_STRATEGIES, + OPERATION_TARGET_KINDS, + validated_operation_hook_fields, + validated_operation_metadata, +) from sqlbuild.runtime.observability._helpers.dispatcher import ( current_event_dispatcher, dispatcher_scope, @@ -21,25 +32,12 @@ operation_scope, ) from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.constants import ( - HOOK_PHASES, - HOOK_TYPES, - OPERATION_ADAPTERS, - OPERATION_KINDS, - OPERATION_METADATA_FIELDS, - OPERATION_NAMES, - OPERATION_PHASES, - OPERATION_SCOPES, - OPERATION_STRATEGIES, - OPERATION_TARGET_KINDS, - RETRY_SCHEDULED_EVENT, -) +from sqlbuild.runtime.observability.constants import RETRY_SCHEDULED_EVENT from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity, OperationAttributes from sqlbuild.runtime.observability.types import JSONValue _ERROR_TOKEN_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") -_HOOK_NAME_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,127}$") class OperationLifecycle: @@ -78,9 +76,9 @@ def __init__( ) self._operation_kind: str = operation_kind self._operation_name: str = operation_name - self._metadata: dict[str, int] = _validated_metadata(metadata) + self._metadata: dict[str, int] = validated_operation_metadata(metadata=metadata) self._operation_id: str | None = operation_id - self._hook_fields: dict[str, JSONValue] = _validated_hook_fields( + self._hook_fields: dict[str, JSONValue] = validated_operation_hook_fields( hook_phase=hook_phase, hook_index=hook_index, hook_type=hook_type, @@ -239,7 +237,7 @@ def _terminal_payload( ) -> dict[str, JSONValue]: payload: dict[str, JSONValue] = self._base_payload() payload.update(_validated_attribute_fields(attributes)) - terminal_metadata: dict[str, int] = _validated_metadata(metadata) + terminal_metadata: dict[str, int] = validated_operation_metadata(metadata=metadata) if terminal_metadata: payload["metadata"] = {**self._metadata, **terminal_metadata} started: float = self._started_monotonic if self._started_monotonic is not None else 0.0 @@ -309,50 +307,6 @@ def _validated_attribute_fields( return fields -def _validated_metadata(metadata: Mapping[str, int] | None) -> dict[str, int]: - if metadata is None: - return {} - unexpected: set[str] = set(metadata) - OPERATION_METADATA_FIELDS - if unexpected: - raise ObservabilityValidationError("operation metadata contains a non-allowlisted field") - result: dict[str, int] = {} - for key, value in metadata.items(): - if isinstance(value, bool) or not isinstance(value, int) or value < 0: - raise ObservabilityValidationError( - "operation metadata values must be nonnegative integers excluding bool" - ) - result[key] = value - return result - - -def _validated_hook_fields( - *, - hook_phase: str | None, - hook_index: int | None, - hook_type: str | None, - hook_name: str | None, -) -> dict[str, JSONValue]: - supplied: bool = any( - value is not None for value in (hook_phase, hook_index, hook_type, hook_name) - ) - if not supplied: - return {} - if hook_phase not in HOOK_PHASES: - raise ObservabilityValidationError("hook_phase must be a catalogued value") - if hook_type not in HOOK_TYPES: - raise ObservabilityValidationError("hook_type must be a catalogued value") - if isinstance(hook_index, bool) or not isinstance(hook_index, int) or hook_index < 0: - raise ObservabilityValidationError("hook_index must be a nonnegative integer") - result: dict[str, JSONValue] = { - "hook_phase": hook_phase, - "hook_index": hook_index, - "hook_type": hook_type, - } - if hook_name is not None and _HOOK_NAME_PATTERN.fullmatch(hook_name) is not None: - result["hook_name"] = hook_name - return result - - def _safe_error_token(*, value: object, fallback: str) -> str: if not isinstance(value, str) or _ERROR_TOKEN_PATTERN.fullmatch(value) is None: return fallback diff --git a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py index e4dd7b4c2e..9b0bf9aa84 100644 --- a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py @@ -8,6 +8,7 @@ from types import TracebackType from uuid import uuid4 +from sqlbuild._native import RESOURCE_SKIP_CODES, RESOURCE_SKIP_MODES from sqlbuild.runtime.observability._helpers.dispatcher import ( current_event_dispatcher, dispatcher_scope, @@ -20,11 +21,7 @@ run_scope, ) from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.constants import ( - RESOURCE_ATTEMPT_SKIPPED_EVENT, - RESOURCE_SKIP_CODES, - RESOURCE_SKIP_MODES, -) +from sqlbuild.runtime.observability.constants import RESOURCE_ATTEMPT_SKIPPED_EVENT from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import ExecutionIdentity from sqlbuild.runtime.observability.types import JSONValue diff --git a/src/sqlbuild/runtime/observability/classes/run_lifecycle.py b/src/sqlbuild/runtime/observability/classes/run_lifecycle.py index 887519d012..ee9f74ad2b 100644 --- a/src/sqlbuild/runtime/observability/classes/run_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/run_lifecycle.py @@ -5,10 +5,10 @@ import time from types import TracebackType +from sqlbuild._native import validate_run_bounds from sqlbuild.runtime.observability._helpers.dispatcher import current_event_dispatcher from sqlbuild.runtime.observability._helpers.factory import create_lifecycle_event from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.types import JSONValue @@ -23,7 +23,7 @@ def __init__( configured_concurrency: int, worker_count: int, ) -> None: - _validate_run_bounds( + validate_run_bounds( selected_count=selected_count, configured_concurrency=configured_concurrency, worker_count=worker_count, @@ -108,25 +108,3 @@ def _publish(self, *, event_type: str, payload: dict[str, JSONValue]) -> None: self._dispatcher.publish_lifecycle( create_lifecycle_event(event_type=event_type, payload=payload) ) - - -def _validate_run_bounds( - *, selected_count: int, configured_concurrency: int, worker_count: int -) -> None: - values: tuple[tuple[str, int], ...] = ( - ("selected_count", selected_count), - ("configured_concurrency", configured_concurrency), - ("worker_count", worker_count), - ) - for field_name, value in values: - if isinstance(value, bool) or not isinstance(value, int): - raise ObservabilityValidationError(f"{field_name} must be an integer excluding bool") - if selected_count < 0: - raise ObservabilityValidationError("selected_count must be nonnegative") - if configured_concurrency < 1: - raise ObservabilityValidationError("configured_concurrency must be positive") - expected_worker_count: int = min(configured_concurrency, selected_count) - if worker_count != expected_worker_count: - raise ObservabilityValidationError( - "worker_count must equal min(configured_concurrency, selected_count)" - ) diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index d09e09cbca..47ecb4b533 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -1,332 +1,13 @@ -"""Versioned runtime observability catalogs and decision values.""" +"""Lifecycle envelope, diagnostic log and emitter constants; the event catalogue is native.""" from collections.abc import Mapping from types import MappingProxyType -from sqlbuild.runtime.observability.models import LifecycleEventDefinition - -CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: int = 2 LIFECYCLE_INVOCATION_METADATA_SCHEMA_VERSION: int = 2 -CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: int = 1 -DURATION_MS_FIELD: str = "duration_ms" -EXIT_CODE_FIELD: str = "exit_code" -PROCESS_ID_FIELD: str = "process_id" -SIGNAL_NUMBER_FIELD: str = "signal_number" -METADATA_FIELD: str = "metadata" -CONFIGURED_CONCURRENCY_FIELD: str = "configured_concurrency" -RUN_STARTED_EVENT: str = "run_started" -RUN_TERMINALS: frozenset[str] = frozenset({"run_completed", "run_failed"}) -AUDIT_RUN_COUNT_FIELDS: frozenset[str] = frozenset({"pass_count", "warn_count", "fail_count"}) -STATEMENT_EVENT_PREFIX: str = "statement_" -MAX_METADATA_BYTES: int = 4096 -DIAGNOSTIC_SEVERITIES: frozenset[str] = frozenset( - {"trace", "debug", "info", "warning", "error", "critical"} -) -ERROR_FIELDS: frozenset[str] = frozenset({"error_code", "error_type"}) -DURATION_FIELDS: frozenset[str] = frozenset({DURATION_MS_FIELD}) -RESOURCE_FIELDS: frozenset[str] = frozenset({"resource_kind", "resource_name", "attempt_number"}) -RESOURCE_SKIP_CODES: frozenset[str] = frozenset({"dependency", "explicit", "fan_in", "scheduler"}) -RESOURCE_SKIP_MODES: frozenset[str] = frozenset({"hard", "soft"}) RESOURCE_ATTEMPT_SKIPPED_EVENT: str = "resource_attempt_skipped" -RESOURCE_TERMINALS: frozenset[str] = frozenset( - {"resource_attempt_completed", "resource_attempt_failed", RESOURCE_ATTEMPT_SKIPPED_EVENT} -) -MICROBATCH_EVENT_PREFIX: str = "microbatch_" -MICROBATCH_FIELDS: frozenset[str] = frozenset( - { - "cursor_start", - "cursor_end_exclusive", - "planned_cursor_start", - "planned_cursor_end_exclusive", - "batch_index", - "batch_count", - "configured_batch_size", - "effective_batch_size", - "microbatch_strategy", - "microbatch_run_type", - "batch_kind", - } -) -MICROBATCH_STRATEGIES: frozenset[str] = frozenset({"rolling_window", "watermark"}) -MICROBATCH_RUN_TYPES: frozenset[str] = frozenset({"normal", "backfill", "replay_on_change"}) -MICROBATCH_KINDS: frozenset[str] = frozenset({"ordinary", "recovery"}) -HOOK_PHASES: frozenset[str] = frozenset({"post_hooks", "pre_hooks"}) -HOOK_TYPES: frozenset[str] = frozenset({"python", "sql"}) -OPERATION_FIELDS: frozenset[str] = frozenset( - { - "operation_kind", - "operation_name", - METADATA_FIELD, - "hook_phase", - "hook_index", - "hook_type", - "hook_name", - "phase", - "strategy", - "adapter", - "target_kind", - "scope", - } -) -OPERATION_EVENT_PREFIX: str = "operation_" RENAME_OPERATION_STRATEGY: str = "rename" RETRY_SCHEDULED_EVENT: str = "retry_scheduled" -STATEMENT_FAILED_EVENT: str = "statement_failed" STATEMENT_HEARTBEAT_THRESHOLD_SECONDS: float = 30.0 -OPERATION_KINDS: frozenset[str] = frozenset( - { - "clone", - "freshness", - "janitor", - "loader", - "project", - "python_node", - "quality", - "scenario", - "subprocess", - "warehouse", - } -) -OPERATION_NAMES: frozenset[str] = frozenset( - { - "clone_finalization", - "clone_execution", - "clone_namespace_preparation", - "clone_relation_inspection", - "clone_relation_transfer", - "clone_retention_reconciliation", - "clone_state_connection", - "clone_state_inspection", - "clone_target_connection", - "dbt_command", - "discovery_declaration_parse", - "discovery_filesystem_walk", - "discovery_project_assembly", - "discovery_python_import", - "external_manifest_discovery", - "external_source_load", - "ingestr_command", - "janitor_candidate_planning", - "janitor_cleanup_action", - "janitor_execution", - "janitor_state_inspection", - "janitor_target_connection", - "janitor_warehouse_inspection", - "managed_source_load", - "project_compile", - "project_discovery", - "python_asset", - "python_check", - "python_hook", - "python_materialization", - "python_task", - "sql_hook", - "sql_test_planning", - "scenario_capture", - "scenario_cleanup", - "scenario_execution", - "scenario_relation_read", - "scenario_schema_inspection", - "scenario_snapshot_serialization", - "scenario_snapshot_write", - "scenario_target_connection", - "source_freshness_metadata_observation", - "source_freshness_query_observation", - "source_freshness_publication", - "audit_evaluation", - "sql_test_setup", - "sql_test_assertion", - "retention_inspection", - "retention_application", - "table_type_inspection", - "table_type_conversion", - "runtime_schema_inspection", - "schema_synchronization", - "staging_creation", - "relation_promotion", - } -) -OPERATION_METADATA_FIELDS: frozenset[str] = frozenset( - { - "attempt_number", - "byte_count", - "item_count", - "row_count", - "changed_count", - "added_count", - "removed_count", - "altered_count", - "unknown_count", - } -) -OPERATION_PHASES: frozenset[str] = frozenset( - { - "inspect", - "apply", - "assert", - "convert", - "create", - "evaluate", - "observe", - "promote", - "publish", - "reconcile", - "setup", - } -) -OPERATION_STRATEGIES: frozenset[str] = frozenset( - { - "atomic_replace", - "atomic_swap", - RENAME_OPERATION_STRATEGY, - "build_aside", - "create_new", - "append_new_columns", - "sync_all_columns", - "adapter", - "column", - "sql", - } -) -OPERATION_TARGET_KINDS: frozenset[str] = frozenset( - { - "audit", - "namespace", - "relation", - "source", - "sql_test", - "staging_relation", - "state_batch", - } -) -OPERATION_SCOPES: frozenset[str] = frozenset( - {"delta", "end", "final", "model", "seed", "source", "standalone"} -) -OPERATION_ADAPTERS: frozenset[str] = frozenset( - { - "bigquery", - "custom", - "databricks", - "duckdb", - "motherduck", - "postgres", - "snowflake", - "sqlserver", - } -) -STATEMENT_FIELDS: frozenset[str] = frozenset( - { - "adapter", - "affected_rows", - "batch_size", - "intent", - "job_id", - METADATA_FIELD, - "query_id", - "row_count", - "sql_digest", - "statement_kind", - } -) -STRING_PAYLOAD_FIELDS: frozenset[str] = frozenset( - { - "adapter", - "command", - "error_code", - "error_type", - "hook_name", - "hook_phase", - "hook_type", - "intent", - "job_id", - "operation_kind", - "operation_name", - "query_id", - "resource_kind", - "resource_name", - "run_kind", - "scope", - "sql_digest", - "statement_kind", - "audit_name", - "audit_definition_name", - "audit_description", - "evaluation_mode", - "outcome", - "severity", - "run_scope_phase", - "attachment_kind", - "binding_key", - "definition_fingerprint", - "execution_fingerprint", - "attached_target_kind", - "attached_target_name", - "attached_column_name", - "target_database", - "target_schema", - "target_name", - "sample_unit", - "evidence_error", - "measurement_sql", - "evidence_sql", - "executed_sql", - "execution_error", - "result_id", - "cursor_start", - "cursor_end_exclusive", - "planned_cursor_start", - "planned_cursor_end_exclusive", - "configured_batch_size", - "effective_batch_size", - "microbatch_strategy", - "microbatch_run_type", - "batch_kind", - } -) -NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: frozenset[str] = frozenset( - { - "affected_rows", - "attempt_number", - "batch_size", - "delay_ms", - "failed_count", - "fail_count", - "failed_attempt_number", - "hook_index", - "next_attempt_number", - PROCESS_ID_FIELD, - "row_count", - "selected_count", - "configured_concurrency", - "pass_count", - SIGNAL_NUMBER_FIELD, - "skipped_count", - "succeeded_count", - "warn_count", - "worker_count", - "violation_count", - "sample_count", - "minimum_samples", - "evidence_count", - "batch_index", - "batch_count", - } -) -FINITE_NUMBER_PAYLOAD_FIELDS: frozenset[str] = frozenset({"measured_value"}) -BOOLEAN_PAYLOAD_FIELDS: frozenset[str] = frozenset({"evidence_truncated", "reused"}) -JSON_PAYLOAD_FIELDS: frozenset[str] = frozenset({"thresholds", "evidence"}) -FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: frozenset[str] = frozenset( - { - "bindings", - "full_sql", - "parameter_values", - "parameters", - "params", - "query_sql", - "sql", - "statement_sql", - } -) LIFECYCLE_ENVELOPE_FIELDS_V1: frozenset[str] = frozenset( { "event_id", @@ -370,212 +51,3 @@ "statement_id", } ) -LIFECYCLE_EVENT_CATALOG_V1: Mapping[str, LifecycleEventDefinition] = MappingProxyType( - { - "invocation_started": LifecycleEventDefinition.create(allowed=frozenset({"command"})), - "invocation_completed": LifecycleEventDefinition.create( - allowed=frozenset({"command", EXIT_CODE_FIELD}) | DURATION_FIELDS, terminal=True - ), - "invocation_failed": LifecycleEventDefinition.create( - allowed=frozenset({"command", EXIT_CODE_FIELD}) | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - "run_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - allowed=frozenset( - {"run_kind", "selected_count", "configured_concurrency", "worker_count"} - ), - ), - "run_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - allowed=frozenset( - { - "run_kind", - "succeeded_count", - "failed_count", - "skipped_count", - "pass_count", - "warn_count", - "fail_count", - } - ) - | DURATION_FIELDS, - terminal=True, - ), - "run_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - allowed=frozenset( - { - "run_kind", - "succeeded_count", - "failed_count", - "skipped_count", - "pass_count", - "warn_count", - "fail_count", - } - ) - | DURATION_FIELDS - | ERROR_FIELDS, - terminal=True, - ), - "resource_attempt_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - allowed=RESOURCE_FIELDS, - ), - "resource_attempt_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - allowed=RESOURCE_FIELDS | DURATION_FIELDS, - terminal=True, - ), - "resource_attempt_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - allowed=RESOURCE_FIELDS | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - RESOURCE_ATTEMPT_SKIPPED_EVENT: LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=RESOURCE_FIELDS | frozenset({"skip_code"}), - allowed=RESOURCE_FIELDS | DURATION_FIELDS | frozenset({"skip_code", "skip_mode"}), - terminal=True, - ), - "operation_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"operation_id"}), allowed=OPERATION_FIELDS - ), - "operation_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"operation_id"}), - allowed=OPERATION_FIELDS - | DURATION_FIELDS - | frozenset({EXIT_CODE_FIELD, PROCESS_ID_FIELD, SIGNAL_NUMBER_FIELD}), - terminal=True, - ), - "operation_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"operation_id"}), - allowed=OPERATION_FIELDS - | DURATION_FIELDS - | ERROR_FIELDS - | frozenset({EXIT_CODE_FIELD, PROCESS_ID_FIELD, SIGNAL_NUMBER_FIELD}), - terminal=True, - ), - RETRY_SCHEDULED_EVENT: LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=frozenset( - { - "failed_attempt_number", - "next_attempt_number", - "delay_ms", - "error_type", - } - ), - allowed=frozenset( - { - "failed_attempt_number", - "next_attempt_number", - "delay_ms", - "error_type", - "error_code", - } - ), - ), - "audit_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id"}), - required_payload=frozenset( - { - "audit_name", - "audit_definition_name", - "evaluation_mode", - "outcome", - "severity", - "run_scope_phase", - "attachment_kind", - } - ), - allowed=frozenset( - { - "audit_name", - "audit_definition_name", - "audit_description", - "evaluation_mode", - "outcome", - "severity", - "run_scope_phase", - "attachment_kind", - "binding_key", - "definition_fingerprint", - "execution_fingerprint", - "attached_target_kind", - "attached_target_name", - "attached_column_name", - "target_database", - "target_schema", - "target_name", - "violation_count", - "measured_value", - "sample_count", - "sample_unit", - "minimum_samples", - "thresholds", - "evidence", - "evidence_count", - "evidence_truncated", - "evidence_error", - "measurement_sql", - "evidence_sql", - "executed_sql", - "sql_digest", - "execution_error", - "reused", - "result_id", - } - ), - terminal=True, - ), - "statement_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), allowed=STATEMENT_FIELDS - ), - "statement_submitted": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS - frozenset({"row_count"}), - ), - "statement_heartbeat": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS | DURATION_FIELDS, - ), - "statement_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS | DURATION_FIELDS, - terminal=True, - ), - "statement_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"statement_id"}), - allowed=STATEMENT_FIELDS | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - } -) -LIFECYCLE_EVENT_CATALOG_V2: Mapping[str, LifecycleEventDefinition] = MappingProxyType( - { - **LIFECYCLE_EVENT_CATALOG_V1, - "microbatch_started": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=MICROBATCH_FIELDS, - allowed=MICROBATCH_FIELDS, - ), - "microbatch_completed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=MICROBATCH_FIELDS, - allowed=MICROBATCH_FIELDS | DURATION_FIELDS | frozenset({"affected_rows"}), - terminal=True, - ), - "microbatch_failed": LifecycleEventDefinition.create( - required_correlations=frozenset({"run_id", "resource_id", "resource_attempt_id"}), - required_payload=MICROBATCH_FIELDS | frozenset({"error_type"}), - allowed=MICROBATCH_FIELDS | DURATION_FIELDS | ERROR_FIELDS, - terminal=True, - ), - } -) -LIFECYCLE_EVENT_CATALOGS: Mapping[int, Mapping[str, LifecycleEventDefinition]] = MappingProxyType( - {1: LIFECYCLE_EVENT_CATALOG_V1, 2: LIFECYCLE_EVENT_CATALOG_V2} -) -LIFECYCLE_EVENT_CATALOG: Mapping[str, LifecycleEventDefinition] = LIFECYCLE_EVENT_CATALOG_V2 diff --git a/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py b/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py index 411f9259ff..76636ad78f 100644 --- a/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py +++ b/src/sqlbuild/runtime/observability/main/canonicalize_operation_adapter.py @@ -1,6 +1,6 @@ """Canonical operation adapter identity.""" -from sqlbuild.runtime.observability.constants import OPERATION_ADAPTERS +from sqlbuild._native import OPERATION_ADAPTERS from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError diff --git a/src/sqlbuild/runtime/observability/models.py b/src/sqlbuild/runtime/observability/models.py index 1efbb621d0..d855c62daa 100644 --- a/src/sqlbuild/runtime/observability/models.py +++ b/src/sqlbuild/runtime/observability/models.py @@ -6,8 +6,16 @@ from collections.abc import Mapping from dataclasses import dataclass, field from datetime import datetime -from typing import Literal, Self - +from typing import Literal + +from sqlbuild._native import ( + freeze_lifecycle_json, + validate_diagnostic_log, + validate_lifecycle_event, + validate_lifecycle_invocation_sequence, + validate_lifecycle_optional_text, + validate_lifecycle_required_text, +) from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.types import JSONValue @@ -68,12 +76,8 @@ class ExecutionIdentity: log_stream_id: str | None = None def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import ( - validate_optional_text, - validate_required_text, - ) - validate_required_text(value=self.invocation_id, field_name="invocation_id") + validate_lifecycle_required_text(value=self.invocation_id, field_name="invocation_id") for field_name in ( "run_id", "resource_id", @@ -82,30 +86,7 @@ def __post_init__(self) -> None: "statement_id", "log_stream_id", ): - validate_optional_text(value=getattr(self, field_name), field_name=field_name) - - -@dataclass(frozen=True) -class LifecycleEventDefinition: - """Validation contract for one known lifecycle event type.""" - - required_correlations: frozenset[str] - required_payload_fields: frozenset[str] - allowed_payload_fields: frozenset[str] - terminal: bool - - @classmethod - def create( - cls, - *, - required_correlations: frozenset[str] = frozenset(), - required_payload: frozenset[str] = frozenset(), - allowed: frozenset[str] = frozenset(), - terminal: bool = False, - ) -> Self: - """Create one immutable catalog definition.""" - - return cls(required_correlations, required_payload, allowed, terminal) + validate_lifecycle_optional_text(value=getattr(self, field_name), field_name=field_name) @dataclass(frozen=True) @@ -129,75 +110,12 @@ class LifecycleEvent: payload: Mapping[str, JSONValue] = field(default_factory=dict) def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import ( - freeze_json, - validate_known_lifecycle_event, - validate_optional_text, - validate_required_text, - validate_schema_version, - validate_timestamp, - ) - from sqlbuild.runtime.observability.constants import ( - CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION, - LIFECYCLE_EVENT_CATALOGS, - ) - - for field_name in ( - "event_id", - "event_type", - "producer", - "producer_version", - "invocation_id", - ): - validate_required_text(value=getattr(self, field_name), field_name=field_name) - validate_schema_version(value=self.schema_version) - if self.schema_version not in LIFECYCLE_EVENT_CATALOGS: - raise ObservabilityValidationError( - "LifecycleEvent only represents known schema versions through " - f"{CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION}; decode other versions as " - "OpaqueLifecycleEvent" - ) - if self.schema_version == 1: - if self.invocation_sequence is not None or self.external_context: - raise ObservabilityValidationError( - "schema version 1 lifecycle events cannot contain invocation metadata" - ) - elif ( - isinstance(self.invocation_sequence, bool) - or not isinstance(self.invocation_sequence, int) - or self.invocation_sequence < 0 - ): - raise ObservabilityValidationError( - "invocation_sequence must be a non-negative integer excluding bool" - ) - validate_timestamp(value=self.occurred_at) - for field_name in ( - "run_id", - "resource_id", - "resource_attempt_id", - "operation_id", - "statement_id", - ): - validate_optional_text(value=getattr(self, field_name), field_name=field_name) - frozen_context: JSONValue = freeze_json( - value=self.external_context, path="external_context" - ) - if not isinstance(frozen_context, Mapping): - raise ObservabilityValidationError("external_context must be a JSON object") - object.__setattr__(self, "external_context", frozen_context) - frozen_payload: JSONValue = freeze_json(value=self.payload, path="payload") - if not isinstance(frozen_payload, Mapping): - raise ObservabilityValidationError("payload must be a JSON object") - object.__setattr__(self, "payload", frozen_payload) - validate_known_lifecycle_event(event=self) + validate_lifecycle_event(event=self) def with_invocation_sequence(self, sequence: int) -> LifecycleEvent: """Return this validated event with a dispatcher-assigned invocation sequence.""" - if isinstance(sequence, bool) or not isinstance(sequence, int) or sequence < 0: - raise ObservabilityValidationError( - "invocation_sequence must be a non-negative integer excluding bool" - ) + validate_lifecycle_invocation_sequence(value=sequence) sequenced: LifecycleEvent = copy.copy(self) object.__setattr__(sequenced, "invocation_sequence", sequence) return sequenced @@ -210,9 +128,7 @@ class OpaqueLifecycleEvent: raw: Mapping[str, JSONValue] def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import freeze_json - - frozen_raw: JSONValue = freeze_json(value=self.raw, path="event") + frozen_raw: JSONValue = freeze_lifecycle_json(value=self.raw, path="event") if not isinstance(frozen_raw, Mapping): raise ObservabilityValidationError("event must be a JSON object") object.__setattr__(self, "raw", frozen_raw) @@ -240,51 +156,7 @@ class DiagnosticLog: statement_id: str | None = None def __post_init__(self) -> None: - from sqlbuild.runtime.observability._helpers.validation import ( - freeze_json, - validate_optional_text, - validate_required_text, - validate_schema_version, - validate_timestamp, - ) - from sqlbuild.runtime.observability.constants import ( - CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION, - DIAGNOSTIC_SEVERITIES, - ) - - validate_schema_version(value=self.schema_version) - if self.schema_version != CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION: - raise ObservabilityValidationError( - f"diagnostic schema_version must be {CURRENT_DIAGNOSTIC_LOG_SCHEMA_VERSION}" - ) - for field_name in ( - "producer", - "producer_version", - "severity", - "logger", - "source", - "message", - ): - validate_required_text(value=getattr(self, field_name), field_name=field_name) - if self.severity not in DIAGNOSTIC_SEVERITIES: - raise ObservabilityValidationError( - "severity must be one of trace, debug, info, warning, error, critical" - ) - validate_timestamp(value=self.occurred_at) - for field_name in ( - "log_stream_id", - "invocation_id", - "run_id", - "resource_id", - "resource_attempt_id", - "operation_id", - "statement_id", - ): - validate_optional_text(value=getattr(self, field_name), field_name=field_name) - frozen_fields: JSONValue = freeze_json(value=self.fields, path="fields") - if not isinstance(frozen_fields, Mapping): - raise ObservabilityValidationError("fields must be a JSON object") - object.__setattr__(self, "fields", frozen_fields) + validate_diagnostic_log(record=self) @dataclass(frozen=True) diff --git a/src/sqlbuild/runtime/output_capture/constants.py b/src/sqlbuild/runtime/output_capture/constants.py index 440d894096..efbe81b295 100644 --- a/src/sqlbuild/runtime/output_capture/constants.py +++ b/src/sqlbuild/runtime/output_capture/constants.py @@ -2,39 +2,5 @@ from importlib.metadata import version -DEFAULT_OUTPUT_BATCH_SIZE: int = 64 -DEFAULT_OUTPUT_MAX_RECORD_BYTES: int = 64 * 1024 -DEFAULT_OUTPUT_FLUSH_INTERVAL_SECONDS: float = 0.25 -DEFAULT_OUTPUT_QUEUE_CAPACITY: int = 1024 -DEFAULT_OUTPUT_SHUTDOWN_TIMEOUT_SECONDS: float = 2.0 -MIN_OUTPUT_RECORD_BYTES: int = 4 -UTF8_CONTINUATION_BYTE_MASK: int = 0xC0 -UTF8_CONTINUATION_BYTE_TAG: int = 0x80 -INVOCATION_CONTEXT_ENV: str = "SQLBUILD_INVOCATION_CONTEXT_JSON" -MAX_INVOCATION_CONTEXT_BYTES: int = 16 * 1024 -MAX_INVOCATION_CONTEXT_DEPTH: int = 8 -CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int = 1 -COMMAND_OUTPUT_RECORD_TYPE: str = "command_output" -COMMAND_OUTPUT_LOSS_RECORD_TYPE: str = "command_output_loss" COMMAND_OUTPUT_SINK_RECORD_PARAMETER_NAME: str = "record" -COMMAND_OUTPUT_ENVELOPE_FIELDS: frozenset[str] = frozenset( - { - "record_id", - "record_type", - "schema_version", - "producer", - "producer_version", - "occurred_at", - "invocation_id", - "run_id", - "sequence", - "stream", - "message", - "chunk_index", - "chunk_count", - "priority", - "dropped_records", - "external_context", - } -) SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION: str = version("sqlbuild") diff --git a/src/sqlbuild/runtime/output_capture/models.py b/src/sqlbuild/runtime/output_capture/models.py index 7675bcda3b..a3e508bbc3 100644 --- a/src/sqlbuild/runtime/output_capture/models.py +++ b/src/sqlbuild/runtime/output_capture/models.py @@ -6,12 +6,12 @@ from dataclasses import dataclass, field from datetime import datetime -from sqlbuild._native import freeze_command_output_record -from sqlbuild.runtime.output_capture.constants import ( +from sqlbuild._native import ( COMMAND_OUTPUT_RECORD_TYPE, CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION, - SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION, + freeze_command_output_record, ) +from sqlbuild.runtime.output_capture.constants import SQLBUILD_COMMAND_OUTPUT_PRODUCER_VERSION from sqlbuild.runtime.output_capture.types import CommandOutputStream, OutputRecordPriority diff --git a/src/sqlbuild/spec/__init__.py b/src/sqlbuild/spec/__init__.py index 0c4a535a34..e69de29bb2 100644 --- a/src/sqlbuild/spec/__init__.py +++ b/src/sqlbuild/spec/__init__.py @@ -1 +0,0 @@ -"""Spec package.""" diff --git a/src/sqlbuild/spec/contracts/__init__.py b/src/sqlbuild/spec/contracts/__init__.py index 584e9830ad..e69de29bb2 100644 --- a/src/sqlbuild/spec/contracts/__init__.py +++ b/src/sqlbuild/spec/contracts/__init__.py @@ -1 +0,0 @@ -"""Specification contracts.""" diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index d20a1ff03e..6626f8339b 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -2,7 +2,6 @@ from __future__ import annotations -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() -ZERO_DAY_CURSOR_DURATION: str = "0d" diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py deleted file mode 100644 index 84b803ef0d..0000000000 --- a/src/sqlbuild/spec/contracts/models.py +++ /dev/null @@ -1,634 +0,0 @@ -"""Structured project specification models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from decimal import Decimal -from pathlib import Path -from typing import Literal - -from sqlbuild._native import CollectionRendering -from sqlbuild.compiler.auditing.models import MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.cost.constants import DEFAULT_USD_PER_CREDIT -from sqlbuild.spec.contracts.types import ( - ColumnContractMode, - EventExportSeverity, - FutureCursorAction, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - RetentionDecreasePolicy, - SourceFreshnessStrategy, - SourceFreshnessValueKind, - SourceWriteStrategy, - TableType, - TableTypeDowngradePolicy, - TableTypeSource, - TimeTravelRetentionSource, - WarehouseGroup, -) - - -@dataclass(frozen=True) -class ClonePolicy: - """Environment clone policy.""" - - allow_as_clone_origin: bool = False - allow_as_clone_destination: bool = False - - -@dataclass(frozen=True) -class LocalClonePolicy: - """Local environment clone policy overrides.""" - - allow_as_clone_origin: bool | None = None - allow_as_clone_destination: bool | None = None - - -@dataclass(frozen=True) -class AuthoredTimeTravelRetention: - """One authored time-travel retention policy.""" - - desired_days: int | None = None - unmanaged: bool = False - - -@dataclass(frozen=True) -class MaterializationRetentionDefaults: - """Retention defaults for one built-in materialization.""" - - time_travel_retention: AuthoredTimeTravelRetention | None = None - table_type: TableType | None = None - - -@dataclass(frozen=True) -class MaterializationDefaultsConfig: - """Project retention defaults by materialization type.""" - - table: MaterializationRetentionDefaults = field( - default_factory=MaterializationRetentionDefaults - ) - incremental: MaterializationRetentionDefaults = field( - default_factory=MaterializationRetentionDefaults - ) - snapshot: MaterializationRetentionDefaults = field( - default_factory=MaterializationRetentionDefaults - ) - - -@dataclass(frozen=True) -class ResolvedTimeTravelRetention: - """Effective retention policy attached to a compiled model.""" - - desired_days: int | None = None - unmanaged: bool = True - source: TimeTravelRetentionSource | None = None - - -@dataclass(frozen=True) -class ResolvedTableType: - """Effective Snowflake table type attached to a compiled model.""" - - value: TableType = TableType.TRANSIENT - source: TableTypeSource = TableTypeSource.DEFAULT - declared: bool = False - - -@dataclass(frozen=True) -class ExecutionLimitsConfig: - """Optional target-scoped build execution limits.""" - - max_models: int | None = None - max_duration: str | None = None - max_duration_seconds: int | None = None - remediation: str | None = None - - -@dataclass(frozen=True) -class TargetDiffConfig: - """Optional target-scoped data-diff limits.""" - - max_full_rows: int | Literal["unlimited"] | None = None - - -@dataclass(frozen=True) -class TargetWarehousesConfig: - """Optional per-command-group default warehouses for one target.""" - - build: str | None = None - query: str | None = None - - def for_group(self, group: WarehouseGroup) -> str | None: - return self.build if group == WarehouseGroup.BUILD else self.query - - @property - def configured(self) -> bool: - return self.build is not None or self.query is not None - - -@dataclass(frozen=True) -class TargetConfig: - """One named target configuration.""" - - connection: dict[str, object] = field(default_factory=dict) - connection_name: str | None = None - vars: dict[str, str] = field(default_factory=dict) - database: str | None = None - schema: str | None = None - loader_schema: str | None = None - defer_sources_to: str | None = None - defer_clone_from: str | None = None - clone: ClonePolicy = field(default_factory=ClonePolicy) - compile_cache: bool | None = None - time_travel_retention: AuthoredTimeTravelRetention | None = None - time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] = field( - default_factory=dict - ) - owns_time_travel_retention_namespace: bool = False - default_table_type: TableType | None = None - table_type_downgrade: TableTypeDowngradePolicy = TableTypeDowngradePolicy.REQUIRE_CONFIRMATION - time_travel_retention_decrease: RetentionDecreasePolicy = RetentionDecreasePolicy.DENY - missing_migration_origin: MissingMigrationOriginPolicy = MissingMigrationOriginPolicy.ALLOW - execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) - diff: TargetDiffConfig = field(default_factory=TargetDiffConfig) - warehouses: TargetWarehousesConfig = field(default_factory=TargetWarehousesConfig) - - -@dataclass(frozen=True) -class LocalTargetConfig: - """Local developer overrides for one named target.""" - - connection: dict[str, object] = field(default_factory=dict) - connection_name: str | None = None - vars: dict[str, str] = field(default_factory=dict) - database: str | None = None - schema: str | None = None - loader_schema: str | None = None - defer_sources_to: str | None = None - defer_clone_from: str | None = None - clone: LocalClonePolicy = field(default_factory=LocalClonePolicy) - compile_cache: bool | None = None - time_travel_retention: AuthoredTimeTravelRetention | None = None - time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] | None = None - owns_time_travel_retention_namespace: bool | None = None - default_table_type: TableType | None = None - table_type_downgrade: TableTypeDowngradePolicy | None = None - time_travel_retention_decrease: RetentionDecreasePolicy | None = None - missing_migration_origin: MissingMigrationOriginPolicy | None = None - execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) - diff: TargetDiffConfig = field(default_factory=TargetDiffConfig) - warehouses: TargetWarehousesConfig = field(default_factory=TargetWarehousesConfig) - - -@dataclass(frozen=True) -class SettingsConfig: - """Global feature toggles.""" - - sql_analysis: bool = True - require_sql_analysis: bool = False - query_change_tracking: bool = True - column_contract_mode: ColumnContractMode = ColumnContractMode.IMPLICIT - concurrency: int = 1 - auto_load_sources: bool = True - microbatch_concurrency: bool = False - microbatch_unaccounted_partition_policy: str = "synthesize" - table_promotion_mode: str | None = None - default_audit_severity: AuditSeverity | None = None - default_audit_run_scope: str | None = None - - @property - def sql_validation(self) -> bool: - """Compatibility alias for the unified SQL analysis gate.""" - - return self.sql_analysis - - -@dataclass(frozen=True) -class ScopesConfig: - """Declaration scope policy configuration.""" - - enforce_placement: bool = True - - -@dataclass(frozen=True) -class ReferencesConfig: - """Explicit-reference enforcement policy configuration.""" - - enforce_explicit: bool = True - - -@dataclass(frozen=True) -class CostConfig: - """Snowflake compute estimate configuration.""" - - usd_per_credit: Decimal = DEFAULT_USD_PER_CREDIT - usd_per_credit_is_default: bool = True - - -@dataclass(frozen=True) -class ConstantsConfig: - """Project-wide constant rendering configuration.""" - - collection_rendering: CollectionRendering = CollectionRendering.VALUE_LIST - - -@dataclass(frozen=True) -class FutureCursorsConfig: - """Safety policy for automatically discovered future cursor watermarks.""" - - max_distance: str | None = None - action: FutureCursorAction = FutureCursorAction.ERROR - - -@dataclass(frozen=True) -class StartCursorsConfig: - """Safety policy for automatically discovered incremental starts.""" - - max_ahead: str | None = None - action: FutureCursorAction = FutureCursorAction.ERROR - - -@dataclass(frozen=True) -class CursorsConfig: - """Project-wide cursor safety configuration.""" - - future: FutureCursorsConfig = field(default_factory=FutureCursorsConfig) - start: StartCursorsConfig = field(default_factory=StartCursorsConfig) - - -@dataclass(frozen=True) -class MicrobatchLimitsConfig: - """Per-model microbatch planning safety limit.""" - - max_batches: int | None = None - action: MicrobatchLimitAction = MicrobatchLimitAction.ERROR - - -@dataclass(frozen=True) -class MicrobatchesConfig: - """Project-wide microbatch configuration.""" - - limits: MicrobatchLimitsConfig = field(default_factory=MicrobatchLimitsConfig) - - -@dataclass(frozen=True) -class DefaultsConfig: - """Project-wide resource defaults.""" - - materialized: str | None = None - database: str | None = None - schema: str | None = None - seed_database: str | None = None - seed_schema: str | None = None - function_database: str | None = None - function_schema: str | None = None - incremental_strategy: str | None = None - incremental_mode: str | None = None - microbatch_strategy: str | None = None - cursor_watermark_mode: str | None = None - merge_exclude_columns: tuple[str, ...] = field(default_factory=tuple) - full_refresh: bool | None = None - append_cursor_inclusive: object | None = None - cursor_start: object | None = None - cursor_end: object | None = None - cursor_start_max_ahead: str | None = None - cursor_start_max_action: str | None = None - cursor_future_max_distance: str | None = None - cursor_future_action: str | None = None - lookback: str | None = None - batch_size: str | int | None = None - batch_concurrency: int | None = None - max_microbatches: int | None = None - unaccounted_partition_policy: str | None = None - replay_on_change: str | None = None - row_diff_exclude_columns: tuple[str, ...] = field(default_factory=tuple) - row_diff_tolerances: dict[str, object] = field(default_factory=dict) - row_diff_sample_rows: int | None = None - row_diff_sample_seed: int | None = None - tags: tuple[str, ...] = field(default_factory=tuple) - contract: str | None = None - pre_hooks: object | None = None - post_hooks: object | None = None - - -@dataclass(frozen=True) -class DiffConfig: - """Project-wide data-diff configuration.""" - - query_artifact_ttl: str = "24h" - - -@dataclass(frozen=True) -class JanitorConfig: - """Janitor command defaults.""" - - enabled: bool = False - retention_days: int | None = None - archive_retention_days: int = 14 - direct_state_history_versions: int = 20 - delete_tracked_only: bool = True - exclude_patterns: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class MigrationsConfig: - """Model migration defaults.""" - - old_name_views: str | None = "30d" - - -@dataclass(frozen=True) -class SnapshotsConfig: - """Snapshot materialization safety defaults.""" - - current_state_full_refresh: str = "deny" - historical_full_refresh: str = "require_confirmation" - schema_change: str = "append_new_columns" - wildcard_check_schema_change: str = "require_confirmation" - - -@dataclass(frozen=True) -class ScenarioSnapshotLimitsConfig: - """Scenario snapshot capture safety limits.""" - - max_rows_per_relation: int | None = None - max_total_rows: int | None = None - max_bytes_per_relation: int | None = None - max_total_bytes: int | None = None - - -@dataclass(frozen=True) -class ScenarioConfig: - """Scenario command configuration.""" - - run_namespace: str | None = None - local_type_overrides: dict[str, dict[str, str]] = field(default_factory=dict) - snapshot_limits: ScenarioSnapshotLimitsConfig = field( - default_factory=ScenarioSnapshotLimitsConfig - ) - - -@dataclass(frozen=True) -class DbtConfig: - """dbt interop configuration.""" - - project_dir: str | None = None - profiles_dir: str | None = None - target: str | None = None - target_path: str | None = None - vars: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class LocalDbtConfig: - """Local dbt interop configuration overrides.""" - - target: str | None = None - vars: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class LifecycleEventSinkFilterConfig: - """Destination-neutral runtime narrowing for lifecycle-event sinks.""" - - event_kinds: frozenset[str] | None = None - min_severity: EventExportSeverity | None = None - - -@dataclass(frozen=True) -class LifecycleEventSinksConfig: - """Global and named lifecycle-event sink runtime filters and delivery settings.""" - - defaults: LifecycleEventSinkFilterConfig = field(default_factory=LifecycleEventSinkFilterConfig) - named: dict[str, LifecycleEventSinkFilterConfig] = field(default_factory=dict) - shutdown_timeout: str | None = None - shutdown_timeout_seconds: int | None = None - - -@dataclass(frozen=True) -class SinksConfig: - """Typed destination sink configuration.""" - - lifecycle: LifecycleEventSinksConfig = field(default_factory=LifecycleEventSinksConfig) - - -@dataclass(frozen=True) -class ProjectConfig: - """Shared project configuration loaded from sqlbuild_project.toml.""" - - name: str - adapter: str - default_target: str | None = None - connection: dict[str, object] = field(default_factory=dict) - connections: dict[str, dict[str, object]] = field(default_factory=dict) - settings: SettingsConfig = field(default_factory=SettingsConfig) - scopes: ScopesConfig = field(default_factory=ScopesConfig) - references: ReferencesConfig = field(default_factory=ReferencesConfig) - cost: CostConfig = field(default_factory=CostConfig) - constants: ConstantsConfig = field(default_factory=ConstantsConfig) - cursors: CursorsConfig = field(default_factory=CursorsConfig) - microbatches: MicrobatchesConfig = field(default_factory=MicrobatchesConfig) - defaults: DefaultsConfig = field(default_factory=DefaultsConfig) - diff: DiffConfig = field(default_factory=DiffConfig) - materialization_defaults: MaterializationDefaultsConfig = field( - default_factory=MaterializationDefaultsConfig - ) - path_defaults: dict[str, dict[str, object]] = field(default_factory=dict) - vars: dict[str, str] = field(default_factory=dict) - targets: dict[str, TargetConfig] = field(default_factory=dict) - janitor: JanitorConfig = field(default_factory=JanitorConfig) - migrations: MigrationsConfig = field(default_factory=MigrationsConfig) - snapshots: SnapshotsConfig = field(default_factory=SnapshotsConfig) - scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - dbt: DbtConfig = field(default_factory=DbtConfig) - sinks: SinksConfig = field(default_factory=SinksConfig) - - -@dataclass(frozen=True) -class LocalConfig: - """Local developer overrides from sqlbuild_local.toml.""" - - target: str | None = None - adapter: str | None = None - connection: dict[str, object] = field(default_factory=dict) - connections: dict[str, dict[str, object]] = field(default_factory=dict) - targets: dict[str, LocalTargetConfig] = field(default_factory=dict) - settings: SettingsConfig = field(default_factory=SettingsConfig) - setting_overrides: frozenset[str] = field(default_factory=frozenset) - vars: dict[str, str] = field(default_factory=dict) - dbt: LocalDbtConfig = field(default_factory=LocalDbtConfig) - scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - - -@dataclass(frozen=True) -class SourceLocation: - """An authored source location for compiler diagnostics.""" - - path: Path - line: int - column: int - end_line: int | None = None - end_column: int | None = None - - -@dataclass(frozen=True) -class SeedCsvSettings: - """CSV reader settings for a seed file.""" - - delimiter: str | None = None - quotechar: str | None = None - doublequote: bool | None = None - escapechar: str | None = None - skipinitialspace: bool | None = None - lineterminator: str | None = None - encoding: str | None = None - na_values: tuple[object, ...] | dict[str, tuple[object, ...]] | None = None - keep_default_na: bool | None = None - - -@dataclass(frozen=True) -class SchemaAuditInstance: - """One audit instance attached to model, column, or seed metadata.""" - - definition_name: str - arguments: dict[str, object] = field(default_factory=dict) - name: str | None = None - description: str | None = None - severity: AuditSeverity | None = None - run_scope: str | None = None - always_run: bool = False - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - evidence_limit: int | None = None - location: SourceLocation | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True) -class SchemaColumn: - """One declared model, seed, or source column entry.""" - - name: str - type: str | None = None - nullable: bool | None = None - description: str | None = None - meta: dict[str, object] = field(default_factory=dict) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - location: SourceLocation | None = None - migrate_from: str | None = None - - -@dataclass(frozen=True) -class SchemaDynamicColumnFamily: - """One declared family of columns generated by a runtime-dynamic pivot.""" - - name: str - pivot_column: str - value_column: str - aggregate: str - type: str - name_pattern: str | None = None - - -@dataclass(frozen=True) -class SchemaModelEntry: - """One model metadata entry normalized from MODEL(...).""" - - name: str - model_schema: str | None = None - description: str | None = None - type_enforcement: bool | None = None - meta: dict[str, object] = field(default_factory=dict) - columns: tuple[SchemaColumn, ...] = field(default_factory=tuple) - dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = field(default_factory=tuple) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SchemaSeedEntry: - """One seed metadata entry from seed YAML.""" - - name: str - description: str | None = None - database: str | None = None - schema: str | None = None - meta: dict[str, object] = field(default_factory=dict) - csv_settings: SeedCsvSettings = field(default_factory=SeedCsvSettings) - columns: tuple[SchemaColumn, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class IntegrationLoaderConfig: - """Typed configuration for one declarative integration loader.""" - - kind: str - config: object - - -@dataclass(frozen=True) -class SourceFreshnessAgePolicy: - """Age-based source freshness warning/error thresholds.""" - - warn_after: str | None = None - error_after: str | None = None - - -@dataclass(frozen=True) -class SourceFreshnessConfig: - """Configured source freshness observation for planning.""" - - strategy: SourceFreshnessStrategy - value_kind: SourceFreshnessValueKind | None = None - column: str | None = None - query: str | None = None - filter: str | None = None - lag_tolerance: str | None = None - age_policy: SourceFreshnessAgePolicy | None = None - - -@dataclass(frozen=True) -class SourceColumnEntry: - """One source column entry from sources/*.yml.""" - - name: str - type: str | None = None - nullable: bool | None = None - description: str | None = None - meta: dict[str, object] = field(default_factory=dict) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SourceEntry: - """One source declaration from sources/*.yml.""" - - name: str - database: str | None = None - schema: str | None = None - table: str | None = None - loader: str | None = None - managed: bool = False - integration_loader: IntegrationLoaderConfig | None = None - freshness: SourceFreshnessConfig | None = None - write_strategy: SourceWriteStrategy | None = None - load_batch_size: int | None = None - cursor_column: str | None = None - unique_key: tuple[str, ...] = field(default_factory=tuple) - expression: str | None = None - description: str | None = None - type_enforcement: bool | None = None - contract: str | None = None - meta: dict[str, object] = field(default_factory=dict) - columns: tuple[SourceColumnEntry, ...] = field(default_factory=tuple) - audits: tuple[SchemaAuditInstance, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class LoaderDestinationParts: - """Parsed warehouse relation parts for a loader destination.""" - - database: str | None - schema: str | None - table: str diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py deleted file mode 100644 index 38e32f06c2..0000000000 --- a/src/sqlbuild/spec/contracts/types.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Project specification types.""" - -from __future__ import annotations - -from enum import StrEnum - - -class EventExportSeverity(StrEnum): - """Severity used to filter exported lifecycle events.""" - - DEBUG = "debug" - INFO = "info" - WARNING = "warning" - ERROR = "error" - - -class WarehouseGroup(StrEnum): - """Command group that selects a target's default warehouse.""" - - BUILD = "build" - QUERY = "query" - - -class SourceWriteStrategy(StrEnum): - APPEND = "append" - DELETE_INSERT = "delete_insert" - MERGE = "merge" - TABLE = "table" - - -class FutureCursorAction(StrEnum): - """Response when an effective cursor lies beyond the future horizon.""" - - CAP = "cap" - ERROR = "error" - - -class MicrobatchLimitAction(StrEnum): - """Response when one model plans more than the configured batch limit.""" - - CAP_FROM_END = "cap_from_end" - CAP_FROM_START = "cap_from_start" - ERROR = "error" - WARN = "warn" - - -class ColumnContractMode(StrEnum): - """Default contract behavior for model column declarations.""" - - IMPLICIT = "implicit" - EXPLICIT = "explicit" - - -class SourceFreshnessStrategy(StrEnum): - ADAPTER = "adapter" - COLUMN = "column" - SQL = "sql" - - -class SourceFreshnessValueKind(StrEnum): - TIMESTAMP = "timestamp" - INTEGER = "integer" - STRING = "string" - - -class TimeTravelRetentionSource(StrEnum): - """Authored layer that supplied the effective retention policy.""" - - TARGET = "target" - MATERIALIZATION = "materialization" - MODEL = "model" - - -class TableType(StrEnum): - """Snowflake managed table type.""" - - PERMANENT = "permanent" - TRANSIENT = "transient" - - -class TableTypeSource(StrEnum): - """Authored layer that supplied the effective table type.""" - - DEFAULT = "default" - TARGET = "target" - MATERIALIZATION = "materialization" - MODEL = "model" - - -class TableTypeDowngradePolicy(StrEnum): - """Permanent-to-transient conversion policy.""" - - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" - - -class MissingMigrationOriginPolicy(StrEnum): - """Policy for a declared migrate_from whose origin does not exist in a target.""" - - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" - - -class RetentionDecreasePolicy(StrEnum): - """Policy for lowering live time-travel retention.""" - - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" diff --git a/src/sqlbuild/sql_values/models.py b/src/sqlbuild/sql_values/models.py deleted file mode 100644 index e548e25395..0000000000 --- a/src/sqlbuild/sql_values/models.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Immutable models for validated and authored typed SQL values.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.sql_values.types import SqlValueKind, SqlValuePayload - - -@dataclass(frozen=True) -class SqlLogicalType: - """Logical type inferred for a normalized SQL value.""" - - kind: SqlValueKind - element_type: SqlLogicalType | None = None - - @property - def display_name(self) -> str: - """Return a stable human-readable type name.""" - - if self.element_type is None: - return self.kind.value - return f"{self.kind.value}<{self.element_type.display_name}>" - - -@dataclass(frozen=True) -class SqlValue: - """One validated, canonical SQL value.""" - - logical_type: SqlLogicalType - value: SqlValuePayload - - @property - def kind(self) -> SqlValueKind: - """Return the outer logical kind.""" - - return self.logical_type.kind diff --git a/src/sqlbuild/sql_values/types.py b/src/sqlbuild/sql_values/types.py deleted file mode 100644 index 2e6643497a..0000000000 --- a/src/sqlbuild/sql_values/types.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Immutable logical values shared by SQLBuild-authored SQL features.""" - -from __future__ import annotations - -from decimal import Decimal -from enum import StrEnum - - -class SqlValueKind(StrEnum): - """Closed logical kinds accepted by the typed SQL value boundary.""" - - STRING = "string" - INTEGER = "integer" - BOOLEAN = "boolean" - FLOAT = "float" - DECIMAL = "decimal" - NULL = "null" - LIST = "list" - SET = "set" - OBJECT = "object" - - -class StateSqlValueType(StrEnum): - """Closed physical value kinds used by warehouse state columns.""" - - STRING = "string" - INTEGER = "integer" - BOOLEAN = "boolean" - TIMESTAMP = "timestamp" - TEXT_TIMESTAMP = "text_timestamp" - DATE = "date" - JSON = "json" - - -type SqlScalar = str | int | bool | float | Decimal | None -type SqlValuePayload = SqlScalar | tuple[object, ...] diff --git a/tests/conftest.py b/tests/conftest.py index d1b10120dc..7660cdbf82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,9 +6,7 @@ import pytest -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR def pytest_configure(config: pytest.Config) -> None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index 542142b45c..a3222d6af4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -7,9 +7,7 @@ from pathlib import Path from typing import NamedTuple, cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb _TIMING: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 86149d8555..fd248cbac8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -40,14 +40,12 @@ semantic_compile_fingerprint, ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from sqlbuild._native import ContractValidationResult, compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py index 22e2a5d2bb..471586bd10 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py @@ -6,9 +6,7 @@ from pathlib import Path from typing import Any, NamedTuple, cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 5e0749c357..e5e44d18b7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -14,8 +14,13 @@ from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( assert_required_cgroup_memory_limit, ) -from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules -from sqlbuild.rule_engine.models import Rule, RulesConfig +from sqlbuild._native import ( + Rule, + RulesConfig, + build_rule_catalogue, + load_rules_config, + select_catalogue_rules, +) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, DenseWarmEditCompileGuardTestCase, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index f785980e47..dd575d49d8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -10,13 +10,13 @@ from types import MappingProxyType from sqlbuild._native import ( + Fingerprint, + SourceFreshnessRecord, SourceFreshnessRenderers, write_fingerprint, write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, prepare_inline_project, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index d3e3787872..6440dbe38c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -14,10 +14,8 @@ from pathlib import Path from typing import IO, NamedTuple, cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR +from sqlbuild._native import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_input_fingerprint, ) diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index f55f2a1e62..593f0693a7 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -8,6 +8,10 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + Fingerprint, + FingerprintSet, + SourceFreshnessValueKind, normalize_source_freshness_data_version, read_latest_fingerprints, source_freshness_data_version_hash, @@ -27,10 +31,7 @@ ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryBuildFlowTestCase, BigQueryDeleteInsertCursorTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index 13f1b813ed..415ed47ebe 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -7,6 +7,10 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + Fingerprint, + FingerprintSet, + SourceFreshnessValueKind, normalize_source_freshness_data_version, read_latest_fingerprints, source_freshness_data_version_hash, @@ -26,10 +30,7 @@ ) from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.databricks._test_types import ( DatabricksBuildFlowTestCase, DatabricksExpressionNullabilityRuleTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index abae01ff09..38119cc4da 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -1,5 +1,6 @@ from dataclasses import dataclass, field +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -9,7 +10,6 @@ SchemaDiffResult, ) from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SeedCsvSettings @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 2462ff668c..606a918d2c 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -7,6 +7,7 @@ import duckdb import pytest +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( ColumnInfo, @@ -23,7 +24,6 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.integration.src.sqlbuild.adapters.duckdb._test_types import ( CompositeRowDiffSamplingTestCase, ConnectSettingsTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py b/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py index a138efe157..b7015b1a08 100644 --- a/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/snapshot_sql_rendering/helpers.py @@ -13,6 +13,7 @@ import duckdb +from sqlbuild._native import TableType from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import SnapshotChangeTarget from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -21,7 +22,6 @@ from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.spec.contracts.types import TableType from tests.integration.src.sqlbuild.adapters.snapshot_sql_rendering._test_types import ( SnapshotExecutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py index 75e2928189..24169d45bf 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/_test_types.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import TableType from sqlbuild.adapter.contract.models import ( ColumnInfo, QueryResult, @@ -10,7 +11,6 @@ ) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.cost.types import CostStatus -from sqlbuild.spec.contracts.types import TableType @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py index 03070bec2b..5d5c601f5d 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py @@ -8,6 +8,7 @@ import pytest from sqlbuild._native import ( + SourceFreshnessValueKind, normalize_source_freshness_data_version, source_freshness_data_version_hash, ) @@ -26,7 +27,6 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeBuildFlowTestCase, SnowflakeExpressionNullabilityRuleTestCase, diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py index 467aaba1a1..7a8bb03edc 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_table_type_conversion.py @@ -6,13 +6,15 @@ import pytest -from sqlbuild._native import table_type_copy_name +from sqlbuild._native import ( + CompiledRelationLocation, + TableType, + TableTypePlanEntry, + table_type_copy_name, +) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import TableTypePlanEntry from sqlbuild.executor.build._helpers.retention import apply_table_type_conversion from sqlbuild.executor.clone.main._clone_relation_operation import clone_relation_by_names -from sqlbuild.spec.contracts.types import TableType from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeCloneTableTypeTestCase, SnowflakeTableTypeConversionTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 088cb42294..0b688f3b54 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild._native import CursorOverrides @dataclass(frozen=True) @@ -928,6 +928,7 @@ class DiscoveryWorkTestCase: @dataclass(frozen=True) class NativeAllocatorEntryTestCase: description: str + entry_module: str inherited_environment: tuple[tuple[str, str], ...] expected_purge_delay: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index 6dab1de89a..f8e6902028 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -11,15 +11,14 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import PreparedCompileArtifacts, compiler_cache_directory +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR +from sqlbuild._native import ( + STAGE_CAPTURE_DIR_ENV_VAR, + PreparedCompileArtifacts, + compiler_cache_directory, +) from sqlbuild._native import run_cli as main from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module -from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR STAGING_PREFIX: str = ".sqlbuild-staging-" NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" @@ -205,11 +204,7 @@ def change_runtime(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Report a different interpreter from now on.""" del project_dir - - def changed() -> dict[str, str]: - return {**runtime_identity(), "python": "3.0.0 (simulated)"} - - monkeypatch.setattr(runtime_identity_module, "runtime_identity", changed) + monkeypatch.setattr(sys, "version", "3.0.0 (main, Jan 1 2026, 00:00:00) [simulated]") def change_native_build(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index cfc9777725..09311ab352 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -21,6 +21,7 @@ CustomHostObserver, NativeQueryColumns, NativeSqlTestArtifact, + NativeSqlTestPlanningError, PlannedStaticSqlTests, PreparedCompileArtifacts, ) @@ -33,7 +34,6 @@ ) from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 3146c1624c..2090faf3ca 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -14,11 +14,11 @@ from _pytest.capture import CaptureResult import sqlbuild._native as fingerprint_cache_module +from sqlbuild._native import FINGERPRINT_TABLE_NAME from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.executor.build._helpers import scheduler as scheduler_module DATABASE_FILE: str = "orders.duckdb" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py index 0b236fa5de..76a1faa8c7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/old_name_views/helpers.py @@ -9,8 +9,7 @@ import pytest -from sqlbuild.compiler.migrations.models import OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewEventType +from sqlbuild._native import OldNameViewEvent, OldNameViewEventType from sqlbuild.executor.migrations._helpers import old_name_views as old_name_steps from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( CliRun, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py index 7ec8185905..4820e84708 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_discovery_work.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import STAGE_CAPTURE_DIR_ENV_VAR from sqlbuild._native import run_cli as main -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import DiscoveryWorkTestCase from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( eager_output_column_scans, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index cf4ca69fc0..5e172c1c6b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -17,9 +17,14 @@ AuditCommandRequest, BuildCommandRequest, CliUserError, + CursorOverrides, + DiscoveredProjectInputs, + DiscoveredSourceFile, LoadCommandRequest, PlanCommandRequest, + PlannerInputError, ScenarioTestCommandRequest, + SourceEntry, TestCommandRequest, discover_project_inputs, format_load_execution_json, @@ -33,16 +38,12 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorOverrides from sqlbuild.executor.load.main.run import run_load_pipeline from sqlbuild.executor.load.models import ( LoadCallbacks, LoadExecutionResult, LoadRuntimeParams, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( BuildRunAutoLoadFailureTestCase, BuildRunAutoLoadFlagTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py b/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py index f284fb5cef..802f5d1af7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py @@ -14,14 +14,18 @@ ) _PURGE_DELAY_VARIABLE: str = "MIMALLOC_PURGE_DELAY" +_ENTRY_MODULE: str = "sqlbuild.cli.entry.main.entry" +_LAUNCHED_MODULE: str = "sqlbuild.cli.entry.main.launched" _SCRIPT: str = """ +import importlib import json import os import sys -from sqlbuild.cli.entry.main.entry import main +entry = importlib.import_module(sys.argv[1]) loaded_at_import = "sqlbuild._native" in sys.modules -exit_code = main(["--version"]) +sys.argv = ["sqlbuild-python", "--version"] +exit_code = entry.main() if entry.__name__.endswith(".launched") else entry.main(["--version"]) loaded_by_version = "sqlbuild._native" in sys.modules import sqlbuild._native print(json.dumps({ @@ -38,11 +42,25 @@ [ NativeAllocatorEntryTestCase( description="unset purge delay defaults to ten milliseconds", + entry_module=_ENTRY_MODULE, inherited_environment=(), expected_purge_delay="10", ), NativeAllocatorEntryTestCase( description="user purge delay is preserved", + entry_module=_ENTRY_MODULE, + inherited_environment=((_PURGE_DELAY_VARIABLE, "250"),), + expected_purge_delay="250", + ), + NativeAllocatorEntryTestCase( + description="launched entry defaults the purge delay before native load", + entry_module=_LAUNCHED_MODULE, + inherited_environment=(), + expected_purge_delay="10", + ), + NativeAllocatorEntryTestCase( + description="launched entry preserves a user purge delay", + entry_module=_LAUNCHED_MODULE, inherited_environment=((_PURGE_DELAY_VARIABLE, "250"),), expected_purge_delay="250", ), @@ -57,7 +75,7 @@ def test_given_fresh_process_when_running_cli_entry_then_purge_delay_is_set_befo environment.update(test_case.inherited_environment) result: subprocess.CompletedProcess[str] = subprocess.run( - [sys.executable, "-c", _SCRIPT], + [sys.executable, "-c", _SCRIPT, test_case.entry_module], check=True, capture_output=True, env=environment, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py index 6a4d7ddab6..07ed61665f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_provider_annotation_tolerance.py @@ -6,9 +6,12 @@ import pytest -from sqlbuild._native import discover_provider_classes, discover_python_node_functions +from sqlbuild._native import ( + DiscoveredPythonNodeFunctions, + discover_provider_classes, + discover_python_node_functions, +) from sqlbuild._native import run_cli as main -from sqlbuild.compiler.discovery.models import DiscoveredPythonNodeFunctions from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ProviderAnnotationToleranceTestCase, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index bd995a0a75..90b0042362 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -17,14 +17,15 @@ from sqlbuild._native import ( CustomHostObserver, DeclarationScopeBuild, + DiscoveredMacroFile, + DiscoveredProjectInputs, LintRunResult, + LoadedMacro, + SqlLexicalSyntax, active_lint_stop, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main -from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostFailureIntegrationTestCase, CustomHostInterruptIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 005153b360..85d8a7db1d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -11,9 +11,7 @@ import pytest -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( ReusedCompileImportFootprintTestCase, StartupImportFootprintTestCase, @@ -41,9 +39,6 @@ "pydantic", "yaml", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.compiler.compile.models", - "sqlbuild.compiler.discovery.models", - "sqlbuild.compiler.planner.models", "sqlbuild.lint", "sqlbuild.rule_engine", "sqlbuild.runtime.compute_logs", @@ -65,9 +60,7 @@ "polyglot_sql", "pydantic", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.compiler.compile.models", "sqlbuild.compiler.pipeline", - "sqlbuild.compiler.planner.models", ) REUSED_COMPILE_FORBIDDEN: tuple[str, ...] = ( *WAREHOUSE_AND_INTEGRATION_MODULES, diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 9e25be6485..f43dbaea72 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -7,9 +7,7 @@ from pathlib import Path import sqlbuild -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR type LauncherRun = tuple[int, bytes, bytes] diff --git a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py index c1b903e346..aeb5db14ee 100644 --- a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py +++ b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py @@ -6,8 +6,9 @@ import pytest -from sqlbuild._native import IntegrationCheckResult, IntegrationResultEnvelope -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( + IntegrationCheckResult, + IntegrationResultEnvelope, MeasurementThresholdBound, MeasurementThresholds, ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 77b3164b53..a9d2585e20 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -18,12 +18,15 @@ AnalysisCacheContext, CollectionRendering, ColumnLineageMode, + CompactLineageFacts, CompileAdapterContext, DynamicColumnContractProof, + InferredColumn, ModelSqlAnalysis, NativeModelAnalyses, NativeModelAnalysisRequest, NativePivotTables, + SchemaDynamicColumnFamily, analyze_native_model_sql, build_analysis_cache_context, build_compile_inputs, @@ -40,10 +43,8 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompactLineageFacts, InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( AdapterRuleCallbackTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index 173909af0d..ac846e8dba 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -8,10 +8,14 @@ from typing import Any import sqlbuild._native as _native -from sqlbuild._native import SqlReferenceScan, find_macro_call_names, scan_sql_reference_calls +from sqlbuild._native import ( + SqlLexicalSyntax, + SqlReferenceScan, + find_macro_call_names, + scan_sql_reference_calls, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _SQL_PIECES: tuple[str, ...] = ( "SELECT * FROM t WHERE ", diff --git a/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py index ccfd8fdc5e..14697b5a0d 100644 --- a/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/auditing/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild._native import SourceEntry @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py b/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py index 392ba22039..bf4b6bb07b 100644 --- a/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py +++ b/tests/integration/src/sqlbuild/compiler/auditing/main/test_rendering.py @@ -6,9 +6,7 @@ import pytest -from sqlbuild._native import render_audit_sql -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild._native import CompiledRelationLocation, SourceEntry, render_audit_sql from tests.integration.src.sqlbuild.compiler.auditing.main._test_types import ( ExecuteRenderedAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py index ac2167540c..5bc6993ece 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py @@ -9,9 +9,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import extract_sql_references -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import CompileInputError, SqlLexicalSyntax, extract_sql_references from tests.integration.src.sqlbuild.compiler.compile._helpers.refs._test_types import ( CraftedReferenceTestCase, ReferenceExtractionTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index f732d103b2..1079860f2b 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -17,7 +17,11 @@ from sqlbuild._native import ( CompileAdapterContext, CompilerStage, + DiscoveredProjectInputs, + LoadedMacro, MacroExpansionResult, + ResourceIdentity, + ResourceKind, build_compile_inputs, compile_frontier, discover_project_inputs, @@ -28,12 +32,9 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.compile.models import MacroContext from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE -from sqlbuild.compiler.scopes.models import ResourceIdentity -from sqlbuild.compiler.scopes.types import ResourceKind if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index 6b1afe12aa..94cd47b87d 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -8,15 +8,15 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileModelConfig, + CompileSqlReference, + DiscoveredSqlModelFile, ModelValidationRequest, NativeModelConfigInputs, NativeModelConfigSession, + ProjectConfig, + SqlReferenceKind, native_model_config_session, ) -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import ProjectConfig _ACCEPTED: str = "accepted" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index d95f5c52d3..72f66866e1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -18,7 +18,17 @@ CollectionRendering, CompileAdapterContext, CompileModelConfig, + CompileModelInput, CompileProjectInputs, + CompilerDiagnostic, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, + LocalConfig, + MeasurementThresholdBound, + MeasurementThresholds, + ProjectConfig, + SourceColumnEntry, + SqlLexicalSyntax, assemble_project, build_compile_inputs, build_compiled_project, @@ -27,18 +37,12 @@ ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds -from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic from sqlbuild.compiler.compile.types import ( CompiledAuditView, CompiledFunctionView, CompiledModelView, CompiledProjectView, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -116,7 +120,7 @@ def count_legacy_constructions(*, monkeypatch: pytest.MonkeyPatch) -> Counter[st counts: Counter[str] = Counter() for name in LEGACY_PROJECT_TYPES: - model_type: type = getattr(compile_models, name, None) or getattr(_native, name) + model_type: type = getattr(_native, name) monkeypatch.setattr( model_type, "__init__", _counted_init(counts=counts, name=name, model_type=model_type) ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index e20cb23d88..2c9d07b14f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -11,6 +11,7 @@ import pytest from sqlbuild._native import ( + InferredColumn, InvalidQualifiedIdentityError, build_compiled_project, discover_project_inputs, @@ -18,7 +19,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( ImmediateSyntaxErrorTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py index 6ecd0837a2..e8dac20c6e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CollectionRendering, CompileAdapterContext, + SqlHookEntry, build_compile_inputs, discover_project_inputs, ) @@ -18,7 +19,6 @@ from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.discovery.models import SqlHookEntry from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( AdapterParameterBoundaryTestCase, QuotedHookBoundaryTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py index d34587030e..eed81341b3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py @@ -6,7 +6,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSourceView, SourceInput from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeSourceAttachmentTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index f5d3da81f7..eef1bc48dd 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -13,9 +13,19 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + ColumnContractMode, + CompilerDiagnostic, + ContractPolicy, ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, + IncrementalMode, + InferredColumn, + MaterializationType, + SchemaColumn, + SchemaDynamicColumnFamily, + SchemaModelEntry, + SourceLocation, build_project_graph, compiled_model_record, compiled_project_record, @@ -23,17 +33,8 @@ ) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic, InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.planner.types import ContractPolicy, IncrementalMode, MaterializationType -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaDynamicColumnFamily, - SchemaModelEntry, - SourceLocation, -) -from sqlbuild.spec.contracts.types import ColumnContractMode _PROJECT_TOML: str = ( 'name = "orders_contracts"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 95f2f46ee1..3d3e0ba719 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -3,7 +3,7 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild._native import DiscoveredSqlModelFile @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 195e1d6238..f6996f4d4b 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -23,8 +23,13 @@ from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, + DiscoveredProjectInputs, + DiscoveredSourceFile, + DiscoveredSqlModelFile, DiscoveryFileFault, NamedSqlHookEntry, + PythonHookEntry, + SqlHookEntry, TolerantScopeDiscovery, discover_audit_files, discover_constant_files, @@ -37,13 +42,6 @@ discover_sql_hook_files, render_stage_capture, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSourceFile, - DiscoveredSqlModelFile, - PythonHookEntry, - SqlHookEntry, -) from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( GeneratedDeclarationFileTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 8e5e73f284..e4902eed90 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -8,8 +8,7 @@ import pytest from sqlbuild import _native -from sqlbuild._native import DiscoveryFileFault, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs, DiscoveryFileFault, discover_project_inputs from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, DeepDeclarationHeaderTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py index b3bb0494b7..a38c6bb87d 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_discovery.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + DiscoveredSqlModelFile, directory_snapshot_scope, discover_model_files, discover_python_module_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( NativeRuntimeTestCase, SharedSnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py index 439ca19d06..1132003328 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_model_file_record.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild._native import DiscoveredSqlModelFile from tests.integration.src.sqlbuild.compiler.discovery._test_types import RecordCopyTestCase from tests.integration.src.sqlbuild.compiler.discovery.helpers import model_file_record diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 1c26a35c58..adfeddc89a 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -10,12 +10,13 @@ from sqlbuild._native import ( CollectionRendering, CompileAdapterContext, + CompilerDiagnostic, + LiteralSqlRelation, assemble_project, build_compile_inputs, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic, LiteralSqlRelation from sqlbuild.compiler.compile.types import CompiledProjectView _ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py index 495329ab8c..ee54d48b01 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild._native import Fingerprint @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py index 7402cfe234..09488d9cc3 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/helpers.py @@ -6,9 +6,8 @@ from pathlib import Path from typing import Any -from sqlbuild._native import write_fingerprint +from sqlbuild._native import Fingerprint, write_fingerprint from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint def write_fingerprint_from_own_connection( diff --git a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py index 7f9a6f1e29..effd6e49d8 100644 --- a/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/fingerprints/main/test_read_write.py @@ -8,6 +8,9 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + Fingerprint, + FingerprintSet, read_latest_fingerprints, select_latest_fingerprints, write_fingerprint, @@ -15,8 +18,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from tests.integration.src.sqlbuild.compiler.fingerprints.main._test_types import ( BatchWriteTestCase, InvalidDefinitionStorageTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index c3001f2298..34f74aa3be 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -5,7 +5,7 @@ import random import re from collections.abc import Callable, Iterator -from dataclasses import dataclass, replace +from dataclasses import dataclass, fields, replace from pathlib import Path from typing import TYPE_CHECKING, TypeGuard, cast @@ -14,17 +14,21 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileAdapterContext, + CompileInputError, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, + SchemaModelEntry, + SourceLocation, build_compile_inputs, discover_project_inputs, parse_audit_instances, resolve_effective_collection_rendering, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation if TYPE_CHECKING: + from _typeshed import DataclassInstance + from sqlbuild._native import CompileInputsView _TEXTS: tuple[object, ...] = ( @@ -361,9 +365,17 @@ def _attribute_orders(value: object) -> object: return _item_orders(_flattened(value)) +def _field_names(item: object) -> tuple[str, ...]: + return tuple(field.name for field in fields(cast("DataclassInstance", item))) + + def _item_orders(items: tuple[object, ...]) -> object: return tuple( - (type(item).__name__, tuple(vars(item)), _item_orders(getattr(item, "audits", ()))) + ( + type(item).__name__, + _field_names(item), + _item_orders(getattr(item, "audits", ())), + ) for item in items ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 18dbd2ce24..9487e17ec9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -14,6 +14,8 @@ CompileAdapterContext, CompiledLineageColumnFact, CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, build_compile_inputs, build_compiled_project, build_manifest, @@ -26,8 +28,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks if TYPE_CHECKING: diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py index 0a3f70c70f..b73a779ad9 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_attached_audit_gates.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import CompilePipelineResult from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SingularAuditAttachmentIntegrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index 259e866ba7..2716700155 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, build_compiled_project, discover_project_inputs, resolve_diff_model_names, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( DiffSelectorIntegrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index 3c250b11c2..379ec2345d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -5,14 +5,17 @@ import pytest -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + CompileInputError, + CompilePipelineResult, + DiscoveredProjectInputs, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( GroupedDeclarationCompileTestCase, MacroDeclarationContextErrorTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py index e26bf73da4..3484fefca2 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py @@ -6,9 +6,8 @@ import pytest +from sqlbuild._native import CompilePipelineResult, DeclarationRecord from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.scopes.models import DeclarationRecord from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( MacroCompositionIntegrationTestCase, MacroLoadCountIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 66bd89520c..b8e8be854b 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -8,6 +8,11 @@ import pytest from sqlbuild._native import ( + CompilePipelineResult, + DiscoveredProjectInputs, + InferredColumn, + ModelPlanEntry, + RulesError, build_compiled_project, build_project_graph, discover_project_inputs, @@ -15,13 +20,8 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.rule_engine.exceptions import RulesError from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AppendCursorPipelineIntegrationTestCase, CompileProgressIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py index 3006b453fd..3881d06fef 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py @@ -5,11 +5,10 @@ import pytest +from sqlbuild._native import AuditPlanEntry, CompilePipelineResult from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import AuditPlanEntry from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( MeasurementCompileErrorIntegrationTestCase, MeasurementCompileIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index fec1e83cdd..ad43c91fb6 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -8,6 +8,8 @@ import sqlbuild._native as sql_analysis_models from sqlbuild._native import ( + CompiledObjectKey, + CompilerDiagnostic, build_compiled_project, discover_project_inputs, source_binding_diagnostics, @@ -15,7 +17,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( NativeCatalogCase, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py index 6503f5f1e2..ff5ca6cf19 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_python_source_reads.py @@ -7,9 +7,8 @@ import pytest +from sqlbuild._native import CompilePipelineResult, SourceEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( PythonSourceReadPlanTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py index 9f6331740c..2d2b1984d1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_selection_lineage.py @@ -5,8 +5,8 @@ import pytest +from sqlbuild._native import CompilePipelineResult from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( SelectionLineageIntegrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index f6d98e54f3..298e7d7a45 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -9,6 +9,7 @@ import sqlbuild._native as native from sqlbuild._native import ( + CompilerDiagnostic, build_compiled_project, discover_project_inputs, source_binding_diagnostics, @@ -16,7 +17,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( FunctionArgumentTypeCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index fb54d3e2c9..9f9f5861ca 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -1,15 +1,14 @@ from dataclasses import dataclass, field from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, CursorSnapshotScope, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( + Fingerprint, ModelCursorSnapshot, + SourceEntry, ) from sqlbuild.cursor_algebra.types import CursorScalar -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 607dc40b84..40739c6815 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -10,11 +10,22 @@ from typing import Any, ClassVar from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, + CompileSqlReference, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SourceEntry, + SqlReferenceKind, WarehouseSnapshot, compiled_model_record, gather_warehouse_snapshot, @@ -23,13 +34,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, @@ -37,14 +41,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceEntry, -) class RecordingDuckDbAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index 62d9cf8e2d..057c8e071b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -5,8 +5,8 @@ from dataclasses import dataclass from datetime import datetime +from sqlbuild._native import SourceEntry from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, field -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index ed8184d490..8a7992d288 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -7,12 +7,20 @@ from typing import Any, cast from sqlbuild._native import ( + BackfillAction, BackfillResult, CollectionRendering, CompileAdapterContext, + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, + CompileSqlReference, CursorOverridePair, + ModelCursorSnapshot, ModelPlanContext, + SourceEntry, + SqlReferenceKind, WarehouseSnapshot, assemble_project, build_compile_inputs, @@ -21,23 +29,11 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledRelationLocation, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry class OrchestrationOverrideAdapter(DuckDbAdapter): diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py index 1b1ca53e1b..910416c611 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_adapter_overrides.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_source_references +from sqlbuild._native import SourceColumnEntry, SourceEntry, resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterOverrideTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py index 4b3e677317..bef84b5758 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_planning.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.planner.models import CursorBounds, ModelPlanEntry, PlanOutput +from sqlbuild._native import CursorBounds, ModelPlanEntry, PlanOutput from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorPlannerTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py index b55a6cc093..6d9d293942 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py @@ -7,9 +7,11 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, CursorOverridePair, ModelPlanContext, + SqlHookEntry, WarehouseSnapshot, build_function_locations, build_model_locations, @@ -19,8 +21,6 @@ resolve_model_sql, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.types import BackfillAction from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( OrchestrationTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index 361406c3c8..b8a6a90c92 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -7,19 +7,16 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, + CursorType, + ModelCursorSnapshot, + PlannerInputError, + SourceColumnEntry, + SourceEntry, WarehouseSnapshot, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ResolveAndExecuteTestCase, ResolveBoundedOverrideTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 3b7cdc52ba..4a542a445f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -5,25 +5,23 @@ from pathlib import Path from sqlbuild._native import ( - CompiledModelSqlTestPayload, - CompileModelConfig, - CompileSqlTestCte, - DiscoveredSqlTestBlock, -) -from sqlbuild.compiler.compile.models import ( CompiledModel, + CompiledModelSqlTestPayload, CompiledObjectKey, CompiledProject, CompiledRelationLocation, CompiledSqlTest, + CompileModelConfig, + CompileSqlTestCte, + DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + SqlReferenceKind, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index b185ed8256..f174224260 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -7,17 +7,20 @@ import pytest from sqlbuild._native import ( + ChainStep, CompiledDirectLogicSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + SqlTestPlanEntry, build_compiled_project, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 07b92d16ff..6c223c9ada 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -5,16 +5,18 @@ import pytest -from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile, plan_audit -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + AuditPlanEntry, CompiledAudit, CompiledObjectKey, CompiledRelationLocation, + DiscoveredAuditBlock, + DiscoveredAuditFile, + SourceEntry, + plan_audit, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.spec.contracts.models import SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ExecuteAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 8f26166f98..6cf18838c5 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -6,8 +6,10 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, RelationFixtureCompletion, RelationFixturePlanningContext, + SqlTestPlanEntry, SqlTestSelection, build_compiled_project, build_relation_fixture_completion, @@ -17,8 +19,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, FixtureSchemaIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index e92bdd06ff..475af48f3e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -6,18 +6,16 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, CursorSnapshotScope, + ModelCursorSnapshot, + PlannerInputError, WarehouseSnapshot, gather_warehouse_snapshot, ) from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ConcurrentCursorReadsTestCase, EligibleMaxFailureTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index e9d59dc840..b82f927619 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -7,7 +7,16 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + CursorOverrides, CursorSnapshotScope, + Fingerprint, + ModelCursorSnapshot, + PlannerInputError, + SchemaColumn, + SchemaModelEntry, + TimestampValue, WarehouseSnapshot, compiled_model_record, gather_source_columns, @@ -17,23 +26,11 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorOverrides, - ModelCursorSnapshot, -) -from sqlbuild.cursor_algebra.models import TimestampValue -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( GatherCursorSnapshotTestCase, GatherEmptySnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index b6e7da3b27..acc3e29866 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -1,14 +1,12 @@ from dataclasses import dataclass, field from sqlbuild._native import ( - WarningSeverity, -) -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.types import ( BackfillAction, PlanAction, PlanReason, + WarningSeverity, ) +from sqlbuild.compiler.compile.types import FunctionLanguage @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 008c274148..803a661c0b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -14,17 +14,31 @@ from sqlbuild._native import ( CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, CompilePipelineOptions, + CompilePipelineResult, + CompileSqlReference, DeferralInputs, + DiscoveredProjectInputs, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, + Fingerprint, FunctionArgument, PlannerOverrides, PlannerPolicies, PlannerSelection, + PlanOutput, + SchemaSeedEntry, + SettingsConfig, + SourceEntry, + SqlReferenceKind, build_compiled_function_fingerprint_sql, build_execution_plan, compiled_model_record, @@ -36,13 +50,6 @@ ) from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, @@ -52,17 +59,7 @@ CompiledSourceView, FunctionLanguage, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.models import ( - SchemaSeedEntry, - SettingsConfig, - SourceEntry, -) from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, FormatPlanIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 8697f8058f..b005a94aba 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -9,7 +9,23 @@ import sqlbuild._native as identity_planning from sqlbuild._native import ( + BackfillAction, + CursorBounds, + CursorsConfig, + FutureCursorAction, + FutureCursorsConfig, + LocalConfig, + ModelPlanEntry, + PlanAction, + PlannerInputError, + PlanOutput, + PlanReason, PlanWarning, + ProjectConfig, + ResolvedTableType, + TableType, + TableTypeSource, + TargetConfig, WarningSeverity, compiled_model_record, format_plan, @@ -18,26 +34,6 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - ModelPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) -from sqlbuild.spec.contracts.models import ( - CursorsConfig, - FutureCursorsConfig, - LocalConfig, - ProjectConfig, - ResolvedTableType, - TargetConfig, -) -from sqlbuild.spec.contracts.types import FutureCursorAction, TableType, TableTypeSource from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, FormatPlanIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py index 30a4a9bcd0..67ce789fab 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_snowflake_inspection_budget.py @@ -10,10 +10,9 @@ import pytest +from sqlbuild._native import CompilePipelineResult, PlanReason from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.relations.constants import INSPECTION_IN_LIST_LIMIT -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.types import PlanReason from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( SnowflakeCursorBoundsBudgetTestCase, SnowflakeFreshTargetTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py index 3ac70e8357..9af062cc0b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_source_table_probes.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.pipeline.models import CompilePipelineResult +from sqlbuild._native import CompilePipelineResult from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( SourceTableProbeBudgetTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 097ddd2d38..02360d80eb 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -14,9 +14,26 @@ import sqlbuild._native as _native from sqlbuild._native import ( + ConstantDeclaration, + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, + DeclarationResolutionContext, DeclarationScopeBuild, DeclarationScopeResolver, + DeclarationVisibility, + DiscoveredProjectInputs, DiscoveryError, + EnumDeclaration, + LoadedMacro, + ResourceIdentity, + ResourceKind, + ScopeIndex, + ScopeKind, + ScopeLookup, + SqlLexicalSyntax, + VisibilityReason, + VisibilityRecord, build_declaration_scope, discover_project_inputs, resolve_declaration_context, @@ -26,28 +43,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - EnumDeclaration, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibility, - ResourceIdentity, - ScopeIndex, - ScopeLookup, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, - VisibilityReason, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry CURRENT_PATH_COMPONENT: str = "." diff --git a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py index c0bdd44c72..f97b7170c4 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/test_native_scope_parity.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild._native import ScopeIndex from tests.integration.src.sqlbuild.compiler.scopes._test_types import ( GeneratedScopeParityTestCase, ScopeCommandParityTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index d236fd3411..54f83d4884 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -13,10 +13,14 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import build_project_graph, discover_project_inputs, project_with +from sqlbuild._native import ( + CompilerDiagnostic, + build_project_graph, + discover_project_inputs, + project_with, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( generated_analysis_files, diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py index 488db3daf8..b2dc24b6db 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import SourceFreshnessIdentity -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord +from sqlbuild._native import SourceFreshnessIdentity, SourceFreshnessRecord @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py index 73e5d059a0..22fe766871 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py @@ -7,7 +7,9 @@ import pytest from sqlbuild._native import ( + SOURCE_FRESHNESS_TABLE_NAME, SourceFreshnessIdentity, + SourceFreshnessRecord, SourceFreshnessRenderers, SourceFreshnessSet, read_latest_source_freshness, @@ -15,8 +17,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.integration.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessLatestResolutionTestCase, SourceFreshnessPruneHistoryTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index ee3e91c7ba..c835c7305b 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -14,15 +14,13 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + CompiledSqlTest, NativeSqlTestAssembly, NativeSqlTestFailure, build_project_graph, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry diff --git a/tests/integration/src/sqlbuild/executor/build/_test_types.py b/tests/integration/src/sqlbuild/executor/build/_test_types.py index 8e9ab894a8..b12e53944a 100644 --- a/tests/integration/src/sqlbuild/executor/build/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/_test_types.py @@ -1,8 +1,8 @@ from dataclasses import dataclass, field +from sqlbuild._native import MicrobatchLimitAction from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index b4a63cf3c8..3426afc360 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -7,12 +7,16 @@ from pathlib import Path from typing import Any, cast -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + PlanOutput, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCustomizations, @@ -211,7 +215,7 @@ def extract_upstream_model_deps( ) -> dict[str, frozenset[str]]: """Extract upstream model dependency names for each model in the plan.""" - from sqlbuild.compiler.compile.models import CompiledObjectKey + from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType dependencies_by_resource_type: dict[object, dict[str, frozenset[str]]] = { diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py index 1748ca2566..a29f568635 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_ordering_invariant.py @@ -9,12 +9,16 @@ import pytest -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + PlanOutput, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py b/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py index cd5026db34..ea47f3a373 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/_test_types.py @@ -1,9 +1,8 @@ from dataclasses import dataclass, field +from sqlbuild._native import AuditPlanEntry, PlanReason from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 3b622651e7..4982244da4 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -6,6 +6,16 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + DiscoveredHookFunction, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -13,14 +23,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import ( MaterializationContext, @@ -427,13 +430,13 @@ def run_scheduler_build( from sqlbuild._native import ( CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + PlanOutput, discover_project_inputs, run_compile_pipeline, ) from sqlbuild.adapter.contract.types import TablePromotionMode - from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.pipeline.models import CompilePipelineResult - from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCustomizations, diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py index 06f7212128..0eab5b6f8d 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_advanced_custom_materialization.py @@ -9,12 +9,10 @@ import duckdb import pytest +from sqlbuild._native import AuditPlanEntry, CompiledRelationLocation, ModelPlanEntry, PlanReason from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult diff --git a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py index b984b2a510..7879a17227 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/test_custom_materialization.py @@ -9,12 +9,17 @@ import duckdb import pytest +from sqlbuild._native import ( + AuditPlanEntry, + CompiledRelationLocation, + DiscoveredHookFunction, + ModelPlanEntry, + PlanReason, + PythonHookEntry, + SqlHookEntry, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.custom.models import ( MaterializationContext, MaterializationResult, diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 3d0f0bc1d9..3f84e5983c 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -10,20 +10,25 @@ import duckdb -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + MicrobatchLimitAction, + PlanOutput, + TableTypePlanEntry, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildExecutionResult, BuildRuntimeParams, ) from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.build._test_types import ( BuildExecutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index 2f89e19bdc..cdce00d9fa 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -13,10 +13,14 @@ from pathlib import Path from typing import Any -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.pipeline.main.run import run_build_pipeline from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py b/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py index 5a2a93a8e8..984c8be714 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py +++ b/tests/integration/src/sqlbuild/executor/build/test_capped_microbatch_dependencies.py @@ -4,10 +4,8 @@ import pytest +from sqlbuild._native import CompileInputError, CompilePipelineResult, MicrobatchLimitAction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.build._test_types import ( CappedFilterConsumerTestCase, CappedIntermediateFilterTestCase, diff --git a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py index ba35552546..ab6df50036 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py +++ b/tests/integration/src/sqlbuild/executor/build/test_snapshot_execution.py @@ -8,11 +8,16 @@ import pytest -from sqlbuild._native import CompilePipelineOptions, discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + CompilePipelineOptions, + CompilePipelineResult, + DiscoveredProjectInputs, + ModelPlanEntry, + PlanOutput, + discover_project_inputs, + run_compile_pipeline, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index 707a8d34e2..36acabc258 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -10,6 +10,7 @@ import pytest from sqlbuild._native import ( + CompiledProject, DiffCommandRequest, QueryDiffPreparation, build_janitor_plan, @@ -18,7 +19,6 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle diff --git a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py index c588ec16b1..2ba1a0d788 100644 --- a/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -2,13 +2,16 @@ from __future__ import annotations -from sqlbuild._native import MigrationCompatibility, MigrationDiscovery +from sqlbuild._native import ( + CompiledRelationLocation, + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, + ModelMigrationPlanEntry, +) from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.types import MigrationDecision -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry from sqlbuild.executor.migrations.models import MigrationArtifactNames diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py index fc4c5bf5ea..58662db772 100644 --- a/tests/integration/src/sqlbuild/executor/run/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/helpers.py @@ -7,6 +7,16 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + DiscoveredHookFunction, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( @@ -14,18 +24,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py index baef1aa160..f0a0665f71 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/_test_types.py @@ -1,11 +1,10 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import FutureCursorsConfig, OnSchemaChange from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import FutureCursorsConfig @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py index 15d4924f49..4a0f56e9b0 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py @@ -7,32 +7,28 @@ from pathlib import Path from typing import Any, cast -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditRunScope, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, CursorBounds, CursorInputRelation, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( + DiscoveredHookFunction, IncrementalStrategy, MaterializationType, + ModelPlanEntry, OnSchemaChange, PlanAction, PlanReason, ) +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.auditing.types import ( + AuditAttachmentKind, + AuditRunScope, + AuditSeverity, +) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.incremental import execute_incremental_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py index 56b4063e9f..1ba5edc877 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/test_incremental_execution.py @@ -8,15 +8,20 @@ import pytest +from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + DiscoveredHookFunction, + FutureCursorAction, + FutureCursorsConfig, + OnSchemaChange, + PythonHookEntry, + SqlHookEntry, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.integration.src.sqlbuild.executor.run.incremental._test_types import ( IncrementalFailureTestCase, IncrementalSuccessTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py b/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py index 5135161793..02106223b4 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/_test_types.py @@ -2,12 +2,10 @@ from dataclasses import dataclass, field from datetime import datetime +from sqlbuild._native import FutureCursorsConfig, MicrobatchLimitAction, OnSchemaChange from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import MicrobatchLimitAction @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 85b5114203..7a19c2c803 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -7,34 +7,28 @@ from pathlib import Path from typing import Any, cast -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.constants import ( +from sqlbuild._native import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, -) -from sqlbuild.compiler.planner.models import ( AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, CursorBounds, CursorInputRelation, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( + DiscoveredHookFunction, IncrementalMode, IncrementalStrategy, MaterializationType, + ModelPlanEntry, PlanAction, PlanReason, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.auditing.types import ( + AuditAttachmentKind, + AuditSeverity, +) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.microbatch import execute_microbatch_entry from sqlbuild.executor.run._helpers.validation.cursor_bounds import resolve_runtime_cursor_bounds from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py index e2961d27d3..78215eeda5 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_execution.py @@ -10,19 +10,22 @@ import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.constants import ( +from sqlbuild._native import ( MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, + DiscoveredHookFunction, + FutureCursorAction, + FutureCursorsConfig, + MicrobatchLimitAction, + OnSchemaChange, + PythonHookEntry, + SqlHookEntry, ) -from sqlbuild.compiler.planner.types import OnSchemaChange +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.executor.run._helpers.materializations import microbatch as microbatch_module from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.run.microbatch._test_types import ( MicrobatchFailureTestCase, MicrobatchSuccessTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py index afc61aa22c..ec6c1320c7 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_observability.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import MicrobatchLimitAction from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run._helpers.materializations import microbatch as microbatch_module from sqlbuild.executor.run.models import BatchWindow @@ -18,7 +19,6 @@ resource_attempt_scope, run_scope, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.run.microbatch._test_types import ( MicrobatchFailureTestCase, MicrobatchSuccessTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index e779e11b4a..548b644586 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -8,22 +8,19 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, + CursorBounds, CursorWatermarkMode, + IntegerValue, + MicrobatchLimitAction, + ModelCursorSnapshot, + ModelPlanEntry, compute_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner.models import ( - CursorBounds, - ModelCursorSnapshot, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) -from sqlbuild.cursor_algebra.models import IntegerValue from sqlbuild.executor.run._helpers.materializations.microbatch import ( _clamp_intervals_to_model_domain, _count_unaccounted_intervals, @@ -50,7 +47,6 @@ MicrobatchScope, MicrobatchWriteResult, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.integration.src.sqlbuild.executor.run.microbatch._test_types import ( MicrobatchBehaviorTestCase, MicrobatchReconciliationChunkTestCase, diff --git a/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py b/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py index ea9d0433b9..798a903a6d 100644 --- a/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/test_audit_helpers.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry from tests.integration.src.sqlbuild.executor.run._test_types import AuditSqlResolutionTestCase from tests.integration.src.sqlbuild.executor.run.helpers import build_test_audit_plan_entry diff --git a/tests/integration/src/sqlbuild/executor/run/test_table_execution.py b/tests/integration/src/sqlbuild/executor/run/test_table_execution.py index 876b861819..691687feb2 100644 --- a/tests/integration/src/sqlbuild/executor/run/test_table_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/test_table_execution.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase from tests.integration.src.sqlbuild.executor.run._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py index f182f88d24..25b4a0e41c 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py @@ -6,24 +6,23 @@ from pathlib import Path from typing import Any, cast +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + DiscoveredHookFunction, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.executor.run._helpers.materializations.view import execute_view_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py b/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py index 06016f62fe..18efbc5742 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/test_audit_helpers.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry from tests.integration.src.sqlbuild.executor.run.view._test_types import ( ViewAuditSqlResolutionTestCase, ) diff --git a/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py b/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py index ae50c38012..20f5475d59 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/view/test_view_execution.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.run.types import ExecutionPhase from tests.integration.src.sqlbuild.executor.run.view._test_types import ( diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index b7d9284811..d8e679f0c1 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -4,29 +4,23 @@ from typing import Any from sqlbuild._native import ( - ScenarioGraphPlan, -) -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.models import ( + DiscoveredHookFunction, + MaterializationType, ModelPlanEntry, + PlanAction, + PlanReason, + ScenarioArtifactKind, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, + ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, ) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, - ScenarioArtifactKind, -) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import HookContext SCENARIO_NAME: str = "revenue__customer_refund" diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py index 32037dd928..fd7dd93469 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py @@ -9,9 +9,8 @@ import duckdb import pytest +from sqlbuild._native import CompiledRelationLocation, ScenarioArtifactKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import read_scenario_snapshot_manifest from sqlbuild.executor.scenario.main._capture import execute_scenario_snapshot_capture from sqlbuild.executor.scenario.models import ( diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py index 333fb27434..1c5a05a9fc 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py @@ -5,20 +5,19 @@ import pytest -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry -from sqlbuild.compiler.planner.models import ( + DiscoveredHookFunction, + PythonHookEntry, ScenarioExecutionPlan, ScenarioFixturePlan, SeedPlanEntry, ) +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 2514914c86..6fcc3ecd2c 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -6,12 +6,15 @@ from pathlib import Path from typing import Any -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import ( + ChainStep, + CompiledObjectKey, + CompilePipelineResult, + SqlTestPlanEntry, + render_sql_test_comparison_sql, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py index 10a88288a7..d58f985ab1 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_build_parity.py @@ -9,9 +9,8 @@ import pytest -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.planner.models import SqlTestPlanEntry from tests.integration.src.sqlbuild.executor.testing._test_types import ( ExpectedBooleanTestCase, SqlTestBuildParityTestCase, diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py index 5af2fe7cee..88c05db349 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py @@ -7,11 +7,15 @@ import pytest -from sqlbuild._native import render_sql_test_comparison_sql, render_sql_test_difference_sample_sql +from sqlbuild._native import ( + ChainStep, + SqlTestPlanEntry, + render_sql_test_comparison_sql, + render_sql_test_difference_sample_sql, +) from sqlbuild.adapter.contract.types import StatementSizeLimit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import ( SqlTestColumnDifference, diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index 97893a3edc..9fcebecf61 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -8,7 +8,7 @@ build_compile_reference_resolver, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver +from sqlbuild.compiler.compile.types import ExternalSqlReferenceResolver DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DuckDbAdapter(), diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 4284a9bbf6..e459462a3d 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -12,6 +12,7 @@ DbtCommandResult, DbtManifestIndex, DbtRunner, + DiscoveredProjectInputs, assemble_project, build_compile_inputs, build_dbt_combined_graph, @@ -19,7 +20,6 @@ discover_project_inputs, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtCombinedGraphTestCase, ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index 1f527821db..d7e800f628 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -9,10 +9,10 @@ DbtCliOptions, DbtCommandResult, DbtRunner, + DiscoveredProjectInputs, build_compile_inputs, discover_project_inputs, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtManifestCompileTestCase, ) diff --git a/tests/integration/src/sqlbuild/microbatches/classes/helpers.py b/tests/integration/src/sqlbuild/microbatches/classes/helpers.py index c7a59c4964..2e04e6a956 100644 --- a/tests/integration/src/sqlbuild/microbatches/classes/helpers.py +++ b/tests/integration/src/sqlbuild/microbatches/classes/helpers.py @@ -5,11 +5,9 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import deterministic_microbatch_event_id from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS -from sqlbuild.microbatches.main.deterministic_event_id import ( - deterministic_microbatch_event_id, -) from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope from sqlbuild.microbatches.types import ( MicrobatchCompletionType, diff --git a/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py b/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py index 359c5c12e5..13e0d3681b 100644 --- a/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py +++ b/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py @@ -7,11 +7,13 @@ import pytest +from sqlbuild._native import ( + deterministic_microbatch_event_id, + project_microbatch_coverage, + project_replay_requirement, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.microbatches.classes.direct_store import DirectMicrobatchEventStore -from sqlbuild.microbatches.main.deterministic_event_id import deterministic_microbatch_event_id -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index 8419916287..ddaa1811cd 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -6,10 +6,15 @@ import pytest -import sqlbuild.compiler.compile.models as compile_models -from sqlbuild._native import build_compiled_project, discover_project_inputs +import sqlbuild._native as compile_models +from sqlbuild._native import ( + CompiledModel, + CompiledProject, + Model, + build_compiled_project, + discover_project_inputs, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine import fact_replay from sqlbuild.rule_engine._helpers.engine.fact_replay import ( @@ -18,7 +23,6 @@ ) from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views -from sqlbuild.rule_engine.models import Model from sqlbuild.rule_engine.types import FactKey _TREE_ARGUMENTS: tuple[str, ...] = ("models/marts/order_totals.sql", "models") diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index 5088abeffd..de14299b49 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -5,10 +5,11 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledProject, compiled_project_record, custom_host_payload, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.types import FactKey diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index f464671cd6..91ab02ce99 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -10,8 +10,7 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import SqlRulesEvaluation -from sqlbuild.rule_engine.models import Finding +from sqlbuild._native import Finding, SqlRulesEvaluation from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( SqlRulePublicationTestCase, ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index e7ee8d4540..1486874e28 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 6798ba8333..6507e94d9c 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -6,9 +6,12 @@ from itertools import filterfalse from typing import cast -from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES, CompileProjectInputs -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import ( + STAGE_CAPTURE_OMITTED_ATTRIBUTES, + CompiledProject, + CompileProjectInputs, + DiscoveredProjectInputs, +) _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py index ba728bf872..c9dc6fe253 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py @@ -15,9 +15,12 @@ required_discovery_kinds, ) from scripts.compiler_differential.constants import DISCOVERY_NON_COLLECTION_FIELDS -from sqlbuild._native import render_stage_capture -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from sqlbuild._native import ( + DiscoveredProjectInputs, + LocalConfig, + ProjectConfig, + render_stage_capture, +) from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( AuthoredByteKindsTestCase, CaptureProblemsTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py index 4b51f34890..a8e553b9ee 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -15,10 +15,14 @@ required_render_kinds, ) from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS -from sqlbuild._native import CompileProjectInputs, render_stage_capture -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from sqlbuild._native import ( + CompileProjectInputs, + DiscoveredProjectInputs, + LocalConfig, + ProjectConfig, + SqlLexicalSyntax, + render_stage_capture, +) from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( CaptureProblemsTestCase, RenderCaptureKindsTestCase, diff --git a/tests/unit/scripts/python_boundary/main/test_check.py b/tests/unit/scripts/python_boundary/main/test_check.py index cd5071b1a3..2e41167a10 100644 --- a/tests/unit/scripts/python_boundary/main/test_check.py +++ b/tests/unit/scripts/python_boundary/main/test_check.py @@ -50,7 +50,7 @@ " return native_project_graph(project)\n" " return None\n" ) -STAYS: str = '[[stays]]\npath = "executor"\nreason = "executor"\nnote = "Runs plans."\n' +STAYS: str = '[[stays]]\npath = "executor"\nrule = 1\nnote = "Rule 1: runs plans."\n' REVIEWED: str = ( '[[handled_native_errors]]\nlocation = "compiler/keys.py::digest"\nnote = "No key."\n' ) @@ -149,9 +149,7 @@ def test_given_policy_matching_source_when_checking_then_passes( PythonBoundaryTestCase( description="a staying path escapes the source root", source_files={"compiler/plan.py": "a = 1\n"}, - policy=( - 'remaining_lines = 0\n[[stays]]\npath = ".."\nreason = "executor"\nnote = "No."\n' - ), + policy=('remaining_lines = 0\n[[stays]]\npath = ".."\nrule = 1\nnote = "No."\n'), expected_messages=("stays entry '..' must name a module under src/sqlbuild.",), ), PythonBoundaryTestCase( @@ -161,16 +159,68 @@ def test_given_policy_matching_source_when_checking_then_passes( expected_messages=("'compiler/keys.py::digest' no longer matches any handler",), ), PythonBoundaryTestCase( - description="a staying module is missing, unexplained and has an unknown reason", + description="a staying module is missing and has no rule or note", source_files={"compiler/plan.py": "a = 1\n"}, + policy='remaining_lines = 1\n[[stays]]\npath = "planner"\nnote = ""\n', + expected_messages=( + "stays entry 'planner' does not exist under src/sqlbuild.", + "stays entry 'planner' needs a rule: one of 1 = executor or adapter runtime", + "stays entry 'planner' needs a note with the evidence for its rule.", + ), + ), + PythonBoundaryTestCase( + description="a staying module has a rule outside 1 to 4", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = 5\nnote = "Runs."\n' + ), + expected_messages=("stays entry 'executor' has rule 5; use one of 1 = ",), + ), + PythonBoundaryTestCase( + description="a staying module has a non-integer rule", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = "1"\nnote = "Runs."\n' + ), + expected_messages=("stays entry 'executor' has rule '1'; use one of 1 = ",), + ), + PythonBoundaryTestCase( + description="a staying module has a boolean rule", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, policy=( - 'remaining_lines = 1\n[[stays]]\npath = "planner"\nreason = "too hard"\nnote = ""\n' + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = true\nnote = "Runs."\n' ), + expected_messages=("stays entry 'executor' has rule True; use one of 1 = ",), + ), + PythonBoundaryTestCase( + description="a staying module has a whitespace-only note", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=('remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = 1\nnote = " "\n'), expected_messages=( - "stays entry 'planner' does not exist under src/sqlbuild.", - "stays entry 'planner' has reason 'too hard'", - "stays entry 'planner' needs a note explaining why it stays.", + "stays entry 'executor' needs a note with the evidence for its rule.", + ), + ), + PythonBoundaryTestCase( + description="a staying module still uses the replaced reason key", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nreason = "executor"\n' + 'note = "Runs."\n' + ), + expected_messages=( + "stays entry 'executor' has unknown key(s) reason; " + "an entry has only path, rule and note.", + "stays entry 'executor' needs a rule: one of ", + ), + ), + PythonBoundaryTestCase( + description="a staying module's note cites a different rule", + source_files={"compiler/plan.py": "a = 1\n", "executor/run.py": "c = 3\n"}, + policy=( + 'remaining_lines = 1\n[[stays]]\npath = "executor"\nrule = 1\n' + 'note = "Rule 4 glue around a library."\n' ), + expected_messages=("stays entry 'executor' has rule 1 but its note cites rule 4.",), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py index 096b53fc6b..4015954770 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/_helpers/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py b/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py index 8fe449ef20..abd0e1226e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py +++ b/tests/unit/src/sqlbuild/adapter/contract/_helpers/test_seed_csv.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import SeedCsvSettings from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.adapter.contract._helpers._test_types import ( NormalizeSeedCsvValueTestCase, ) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py index a7d666cb56..29a0cde63e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import CompileInputError from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -11,7 +12,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.executor.testing.main._sql_length import validate_unit_test_sql_length from tests.unit.src.sqlbuild.adapter.contract.classes.base_adapter._test_types import ( StatementSizeLimitTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py index bb8a7c17b8..bc0009d664 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py +++ b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/helpers.py @@ -1,7 +1,6 @@ from __future__ import annotations -from sqlbuild._native import normalize_sql_value -from sqlbuild.sql_values.models import SqlValue +from sqlbuild._native import SqlValue, normalize_sql_value def typed_sql_value(raw_value: object) -> SqlValue: diff --git a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py index 246ce97278..7a65929f2e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py +++ b/tests/unit/src/sqlbuild/adapter/contract/typed_sql_rendering/test_typed_sql_rendering.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import SqlValue from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,7 +14,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.adapter.contract.typed_sql_rendering._test_types import ( AdapterTypedSqlRenderingTestCase, InvalidTypedArrayTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py index aa7e533133..ab35de93a2 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py +++ b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_adversarial_input_runtime.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.adapter.relations._helpers.ddl_effects import statement_metadata_effect +from sqlbuild._native import statement_metadata_effect from sqlbuild.adapters.snowflake._helpers.session_context import may_switch_session_database from tests.unit.src.sqlbuild.adapter.relations._helpers.ddl_effects._test_types import ( AdversarialStatementTestCase, diff --git a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py index ae3d75af7a..8708427e95 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py +++ b/tests/unit/src/sqlbuild/adapter/relations/_helpers/ddl_effects/test_statement_metadata_effect.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.adapter.relations._helpers.ddl_effects import ( +from sqlbuild._native import ( qualified_relation_name_key, statement_metadata_effect, ) diff --git a/tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py b/tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py deleted file mode 100644 index 7bdcd5cf85..0000000000 --- a/tests/unit/src/sqlbuild/adapter/state_sql/_test_types.py +++ /dev/null @@ -1,7 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class StateSqlGoldenTestCase: - description: str - expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py b/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py deleted file mode 100644 index 9bf68b82c8..0000000000 --- a/tests/unit/src/sqlbuild/adapter/state_sql/test_source_freshness.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Golden SQL tests for adapter-owned source freshness state.""" - -from __future__ import annotations - -from datetime import datetime, timedelta, timezone - -import pytest - -from sqlbuild.adapter.state_sql._helpers.source_freshness import ( - render_insert_source_freshness_records_sql, -) -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from tests.unit.src.sqlbuild.adapter.state_sql._test_types import StateSqlGoldenTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - StateSqlGoldenTestCase( - description="representative source freshness insert", - expected_sql=( - "INSERT INTO analytics._sqlbuild_source_freshness (source_name, " - "target_database, target_schema, target_name, run_id, strategy, value_kind, " - "data_version, data_version_hash, observed_at) VALUES ('raw.o''rders', NULL, " - "'raw', 'orders', 'run-1', 'adapter', 'timestamp', " - "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00')" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_representative_freshness_record_when_rendering_then_sql_matches_golden( - test_case: StateSqlGoldenTestCase, -) -> None: - sql: str = render_insert_source_freshness_records_sql( - database=None, - schema="analytics", - records=( - SourceFreshnessRecord( - source_name="raw.o'rders", - target_database=None, - target_schema="raw", - target_name="orders", - run_id="run-1", - strategy="adapter", - value_kind="timestamp", - data_version="2026-01-01T01:00:00+01:00", - data_version_hash="hash", - observed_at=datetime(2026, 1, 1, 1, tzinfo=timezone(timedelta(hours=1))), - ), - ), - render_qualified_name=DuckDbAdapter().render_qualified_name, - ) - - assert sql == test_case.expected_sql - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py index ce055965dc..9f110cd9d6 100644 --- a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from sqlbuild.adapter.contract.models import NormalizedType +from sqlbuild.compiler.lineage.types import InferredNullability @dataclass(frozen=True) @@ -26,3 +27,19 @@ class NumericFamilyTestCase: dialect: str | None raw_type: str expected_family: str | None + + +@dataclass(frozen=True) +class NullabilityRuleTestCase: + description: str + args: tuple[InferredNullability, ...] + expected_first_arg: InferredNullability + expected_conditional: InferredNullability + expected_safe_cast: InferredNullability + + +@dataclass(frozen=True) +class FirstArgIdentityTestCase: + description: str + argument: object + expected_same_object: bool diff --git a/tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py b/tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py new file mode 100644 index 0000000000..17c57e0ce8 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/type_system/test_nullability_rules.py @@ -0,0 +1,92 @@ +import pytest + +from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( + conditional_result_nullability, +) +from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability +from sqlbuild.adapter.type_system.main.safe_cast_nullability import safe_cast_nullability +from sqlbuild.compiler.lineage.types import InferredNullability +from tests.unit.src.sqlbuild.adapter.type_system._test_types import ( + FirstArgIdentityTestCase, + NullabilityRuleTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NullabilityRuleTestCase( + description="no arguments", + args=(), + expected_first_arg=InferredNullability.UNKNOWN, + expected_conditional=InferredNullability.UNKNOWN, + expected_safe_cast=InferredNullability.UNKNOWN, + ), + NullabilityRuleTestCase( + description="two non-null results", + args=( + InferredNullability.UNKNOWN, + InferredNullability.NON_NULL, + InferredNullability.NON_NULL, + ), + expected_first_arg=InferredNullability.UNKNOWN, + expected_conditional=InferredNullability.NON_NULL, + expected_safe_cast=InferredNullability.UNKNOWN, + ), + NullabilityRuleTestCase( + description="one nullable result", + args=( + InferredNullability.NULLABLE, + InferredNullability.NON_NULL, + InferredNullability.NULLABLE, + ), + expected_first_arg=InferredNullability.NULLABLE, + expected_conditional=InferredNullability.NULLABLE, + expected_safe_cast=InferredNullability.NULLABLE, + ), + NullabilityRuleTestCase( + description="an unproven result", + args=( + InferredNullability.NON_NULL, + InferredNullability.NON_NULL, + InferredNullability.UNKNOWN, + ), + expected_first_arg=InferredNullability.NON_NULL, + expected_conditional=InferredNullability.UNKNOWN, + expected_safe_cast=InferredNullability.UNKNOWN, + ), + ], + ids=lambda case: case.description, +) +def test_given_argument_nullabilities_when_inferred_then_each_public_rule_returns_its_member( + test_case: NullabilityRuleTestCase, +) -> None: + assert first_arg_nullability(test_case.args) is test_case.expected_first_arg + assert conditional_result_nullability(test_case.args) is test_case.expected_conditional + assert safe_cast_nullability(test_case.args) is test_case.expected_safe_cast + + +@pytest.mark.parametrize( + "test_case", + [ + FirstArgIdentityTestCase( + description="plain text", argument="nullable", expected_same_object=True + ), + FirstArgIdentityTestCase( + description="an enum member", + argument=InferredNullability.NON_NULL, + expected_same_object=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_first_argument_when_first_arg_inferred_then_the_argument_itself_returns( + test_case: FirstArgIdentityTestCase, +) -> None: + returned: object = first_arg_nullability((test_case.argument,)) # ty: ignore[invalid-argument-type] + + assert (returned is test_case.argument) is test_case.expected_same_object + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py index b580c0c654..f9cc02a30f 100644 --- a/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py +++ b/tests/unit/src/sqlbuild/adapters/sqlserver/test_migration_state_sql.py @@ -7,13 +7,14 @@ import pytest from sqlbuild._native import ( + MigrationDecision, MigrationDiscovery, + MigrationEvent, + MigrationRelation, read_migration_state_events, write_migration_state_event, ) from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.migrations.models import MigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import MigrationDecision from tests.unit.src.sqlbuild.adapters.sqlserver._test_types import ( SqlServerMigrationEventReadTestCase, SqlServerMigrationEventWriteTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py index 19d4a9294c..fffeb3ee94 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SnapshotsConfig +from sqlbuild._native import PlanOutput, SnapshotsConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index 2eccf2fc78..e8940a0343 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -2,23 +2,22 @@ from pathlib import Path -from sqlbuild._native import PlanSafetyGates -from sqlbuild.adapter.contract.models import RetentionRequest -from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, MaterializationType, + ModelPlanEntry, PlanAction, PlanReason, + PlanSafetyGates, RetentionDirection, + RetentionPlanEntry, RetentionPlanPhase, + TableTypePlanEntry, ) +from sqlbuild.adapter.contract.models import RetentionRequest +from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.compiler.compile.types import CompiledResourceType def build_snapshot_full_refresh_entry( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py index 9295e1c946..fb722bc451 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py @@ -1,7 +1,6 @@ import pytest -from sqlbuild._native import CliUserError, enforce_model_execution_limit -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig +from sqlbuild._native import CliUserError, ExecutionLimitsConfig, enforce_model_execution_limit from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( ModelExecutionLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py index cb653ebbde..754d7c88a3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py @@ -6,9 +6,13 @@ import pytest -from sqlbuild._native import CliUserError, enforce_plan_safety_policies -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig +from sqlbuild._native import ( + CliUserError, + ExecutionLimitsConfig, + PlanOutput, + SnapshotsConfig, + enforce_plan_safety_policies, +) from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( SnapshotFullRefreshPolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py index 5dfa1f0206..6a31f8b025 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py @@ -6,11 +6,12 @@ from sqlbuild._native import ( CliUserError, + ExecutionLimitsConfig, + PlanOutput, + SnapshotsConfig, dbt_interop_plan_safety_gates, enforce_plan_safety_policies, ) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( DbtPlanSafetyPolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py index 1355aec214..3f0e3b3fb9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py @@ -4,10 +4,14 @@ import pytest -from sqlbuild._native import CliUserError, enforce_plan_safety_policies -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.planner.types import RetentionDirection -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig +from sqlbuild._native import ( + CliUserError, + ExecutionLimitsConfig, + PlanOutput, + RetentionDirection, + SnapshotsConfig, + enforce_plan_safety_policies, +) from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( RetentionDecreasePolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py index cb78756d38..d163fc690c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py @@ -4,9 +4,13 @@ import pytest -from sqlbuild._native import CliUserError, enforce_plan_safety_policies -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig +from sqlbuild._native import ( + CliUserError, + ExecutionLimitsConfig, + PlanOutput, + SnapshotsConfig, + enforce_plan_safety_policies, +) from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( TableTypeDowngradePolicyTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 394064a12b..0e9b24c3ef 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -6,18 +6,22 @@ import pytest -from sqlbuild._native import CliUserError, run_check_read_side_dependencies +from sqlbuild._native import ( + CliUserError, + CompiledProject, + CompiledRelationLocation, + CompilePipelineResult, + PlanOutput, + PythonSqlRunLifecyclePlan, + SourceEntry, + run_check_read_side_dependencies, +) from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.models import PythonSqlRunLifecyclePlan from sqlbuild.executor.python_nodes.main import read_side as read_side_module from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.cli.commands._helpers.check._test_types import ( CheckPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py index 4f758851b9..1649470087 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py @@ -3,8 +3,7 @@ from pathlib import Path from typing import Any -from sqlbuild._native import BuildCostFinalization -from sqlbuild.spec.contracts.models import CostConfig +from sqlbuild._native import BuildCostFinalization, CostConfig def build_cost_finalization( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 9566ba298c..0c6990f34e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -10,11 +10,11 @@ from sqlbuild._native import ( ColumnLineageMode, CompileLineageMode, + DeclarationKind, ParsedCliInvocation, PlaygroundTemplate, parse_cli_invocation, ) -from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( AuditConcurrencyParsingTestCase, ChoiceListTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py index c69eeaf2d2..a284893347 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py @@ -3,8 +3,7 @@ from dataclasses import dataclass, field from datetime import UTC, datetime -from sqlbuild._native import SourceFreshnessIdentity -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord +from sqlbuild._native import SourceFreshnessIdentity, SourceFreshnessRecord @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py index 328765c511..592a1ef620 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/helpers.py @@ -2,19 +2,20 @@ from datetime import UTC, datetime -from sqlbuild._native import source_freshness_data_version_hash +from sqlbuild._native import ( + SourceEntry, + SourceFreshnessAgePolicy, + SourceFreshnessConfig, + SourceFreshnessRecord, + SourceFreshnessStrategy, + SourceFreshnessValueKind, + source_freshness_data_version_hash, +) from sqlbuild.adapter.contract.models import ( QueryResult, TableFreshnessMetadata, TableFreshnessRequest, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.models import ( - SourceEntry, - SourceFreshnessAgePolicy, - SourceFreshnessConfig, -) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind class FreshnessRecordingAdapter: diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index bf07ffd0a2..5c431c8e83 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -4,20 +4,22 @@ from pathlib import Path from sqlbuild._native import ( + ChainStep, + CompiledObjectKey, + CompiledRelationLocation, CostContext, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, SourceFreshnessIdentity, + SourceFreshnessRecord, + SqlTestPlanEntry, build_source_freshness_create_table_sql, ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py index f16717b056..d0312a9858 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py @@ -4,12 +4,10 @@ from pathlib import Path -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredLoaderFunction, DiscoveredProjectInputs, DiscoveredSourceFile, -) -from sqlbuild.spec.contracts.models import ( IntegrationLoaderConfig, LocalConfig, ProjectConfig, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py index b738f4483d..de12ddeff6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/test_selection.py @@ -7,12 +7,16 @@ import pytest from sqlbuild._native import ( + DiscoveredLoaderFunction, + DiscoveredProjectInputs, + LocalConfig, + ProjectConfig, + SourceEntry, + TargetConfig, resolve_loader_defaults, select_load_entries, select_load_reference_entries, ) -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry, TargetConfig from tests.unit.src.sqlbuild.cli.commands._helpers.load_selection._test_types import ( LoaderDefaultResolutionTestCase, LoaderDestinationSelectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py index f324319fe9..64497ac254 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py @@ -1,13 +1,13 @@ from dataclasses import dataclass, field -from sqlbuild._native import CliCommand -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + CliCommand, LocalConfig, LocalTargetConfig, ProjectConfig, TargetWarehousesConfig, + WarehouseGroup, ) -from sqlbuild.spec.contracts.types import WarehouseGroup @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py index bd00e4aa67..a707305d5c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/helpers.py @@ -4,8 +4,8 @@ from pathlib import Path -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + DiscoveredProjectInputs, LocalConfig, ProjectConfig, TargetConfig, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py index 4f2d58fa6b..dbd92dbfe6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_connection.py @@ -6,21 +6,19 @@ from sqlbuild import _native as native from sqlbuild._native import ( + DiscoveredProjectInputs, + LocalConfig, + LocalTargetConfig, NormalizedDbtProfileConnection, + ProjectConfig, ProjectConfigError, + TargetConfig, resolve_connection_config, resolve_project_connection_config, resolve_target_connection_config, ) from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import ( - LocalConfig, - LocalTargetConfig, - ProjectConfig, - TargetConfig, -) from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( ExplicitMemoryConnectionTestCase, MissingConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index e732a3ad32..bf38d9db07 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -9,17 +9,18 @@ from sqlbuild._native import ( CliCommand, CliUserError, + DiscoveredProjectInputs, + LocalTargetConfig, ParsedCliInvocation, ResolvedWarehouse, + TargetWarehousesConfig, + WarehouseGroup, command_warehouse_scope, current_command_warehouse_scope, describe_connection_warehouse, parse_cli_invocation, resolve_project_connection_config, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig -from sqlbuild.spec.contracts.types import WarehouseGroup from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( CommandWarehouseErrorTestCase, CommandWarehouseGroupTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py index fbc50afd1e..6d9d48d82c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/helpers.py @@ -4,17 +4,19 @@ from dataclasses import replace -from sqlbuild._native import build_scope_lookup, preview_scope_move, query_scope_report -from sqlbuild.compiler.scopes.models import ( +from sqlbuild._native import ( OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeIndex, ScopeLookup, ScopeReport, ScopeReportFilters, + build_scope_lookup, + preview_scope_move, + query_scope_report, ) -from sqlbuild.compiler.scopes.types import ResourceKind from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index 44a8306b08..2781355b6b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -10,12 +10,10 @@ CostContext, DirectSourceFreshnessPlanningResult, DirectSourceFreshnessPropagationResult, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( + PlanOutput, append_eligible_direct_source_freshness_records, ) -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py index 5f7f39c3e8..bfd5c1aa64 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py index 458f05fd04..1f72c96b35 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index 1ef2d91a94..3b87f9458a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -4,11 +4,13 @@ import pytest -from sqlbuild._native import CompileModelConfig, PlannerScope, resolve_defer_clone_selection -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompileModelConfig, + PlannerScope, + resolve_defer_clone_selection, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from tests.unit.src.sqlbuild.cli.commands.main.build._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 72c54c82d1..e0e8a179d9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -6,10 +6,26 @@ from typing import Any from sqlbuild._native import ( + AuditPlanEntry, + ChainStep, CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, FunctionArgument, + FunctionPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, PreparedCompileArtifacts, + ProjectGraph, + SeedCsvSettings, + SeedPlanEntry, + SqlTestPlanEntry, WrittenTarget, project_with, ) @@ -21,29 +37,12 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ChainStep, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - SqlTestPlanEntry, -) -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.build.models import ( BuildExecutionResult, FunctionExecutionResult, @@ -51,7 +50,6 @@ from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.cli.commands.main.dag.helpers import ( prepare_python_dag_project, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index d3ea7889a5..331c59ac23 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,9 +5,14 @@ import pytest -from sqlbuild._native import CliUserError, resolve_adapter, resolve_effective_adapter_name +from sqlbuild._native import ( + CliUserError, + LocalConfig, + ProjectConfig, + resolve_adapter, + resolve_effective_adapter_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, ResolveAdapterErrorTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 52ece4d88d..c25a082aee 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -7,19 +7,19 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, CompileLineageMode, CompileModelConfig, ProjectColumnLineage, + ProjectGraph, + SqlHookEntry, WrittenTarget, compiled_model_record, format_compile_json, format_compile_text, project_with, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, CompileJsonLargeIntegerTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 87d117101e..908f900469 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -11,7 +11,11 @@ import sqlbuild._native as model_chain_module import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( + ChainStep, + CompiledSqlTest, DiagnosticPhase, + PlanOutput, + SqlTestPlanEntry, WrittenTarget, build_manifest, compiled_model_record, @@ -21,13 +25,11 @@ write_static_compile_target, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, TargetWriterCacheTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 59eb70930a..b3fbacdb99 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -18,6 +18,7 @@ CompileCommandRequest, CompileLineageMode, CostCommandRequest, + CursorOverrides, DbtInitCommandRequest, DiffCommandRequest, FreshnessCommandRequest, @@ -35,7 +36,6 @@ from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.models import CursorOverrides from sqlbuild.executor.pipeline.exceptions import AuditExecutionError from sqlbuild.executor.pipeline.models import AuditExecutionFailure from sqlbuild.observability import EventDispatcher, ExecutionIdentity, current_execution_identity diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 62c9577aa9..ea4175c20a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -10,12 +10,12 @@ from _pytest.capture import CaptureResult from sqlbuild._native import ( + INTEGRATION_RESULT_PATH_ENV, CliNamespace, NativeProgressProjector, cli_observability_scope, current_native_progress_projector, ) -from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.observability import OperationLifecycle from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( MachineProgressRoutingCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py index 78ed6ae108..882ecb69f3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entrypoint/test_dispatch_with_output_capture.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import EventExporterCommandScope +from sqlbuild._native import INVOCATION_CONTEXT_ENV, EventExporterCommandScope from sqlbuild.observability import ExecutionIdentity -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV from sqlbuild.runtime.output_capture.main.configured_output_capture_scope import ( configured_output_capture_scope, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 50ff8083a9..18a48bbc06 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -10,13 +10,18 @@ import pytest from sqlbuild import _native as native +from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlScenario, + DiscoveredProjectInputs, + DiscoveredSqlScenarioFile, + LocalConfig, + ProjectConfig, +) from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py index b20328cb18..340918201b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py @@ -8,11 +8,11 @@ import pytest from sqlbuild._native import ( + CompilePipelineResult, NativeProgressProjector, ScenarioTestCommandRequest, run_scenario_test_command, ) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( ScenarioCompilePresentationTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index 2af8533318..4f82827a5a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -8,9 +8,8 @@ import pytest from sqlbuild import _native as native -from sqlbuild._native import ScopeCommandRequest, run_scope_command +from sqlbuild._native import ScopeCommandRequest, ScopeCompleteness, ScopeIndex, run_scope_command from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_registry -from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py index 1c03c3f7a8..969189e0a3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index 3efb0d4095..b9554f7905 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -5,17 +5,25 @@ import re from pathlib import Path -from sqlbuild._native import SqlTestAssertionStep +from sqlbuild._native import ( + ChainStep, + CompiledObjectKey, + CompiledRelationLocation, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + SqlTestAssertionStep, + SqlTestPlanEntry, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 526c540419..88223e1e7f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -11,6 +11,7 @@ BuildProgressCallbacks, NestedCommandProgressCallbacks, NestedProgressChildRow, + PlanOutput, build_test_expectation_rows, format_build_footer, resolve_test_name_width, @@ -19,7 +20,6 @@ from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.executor.build.models import ( diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index 33442e434c..e01af2c24b 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -17,3 +17,25 @@ class ProviderSettingsInputsTestCase: expected_env_file_names: tuple[str, ...] = () expected_secrets_dir_names: tuple[str, ...] = () expected_unsupported: bool = False + + +@dataclass(frozen=True) +class StartupInputsTestCase: + """Import-path entries under a temporary root and the stamps they produce there.""" + + description: str + folders: tuple[str, ...] + files: tuple[str, ...] + path_entries: tuple[str, ...] + expected_stamps: tuple[tuple[str, bool], ...] + + +@dataclass(frozen=True) +class StartupInputsErrorTestCase: + """An import-path entry whose stat fails with an error other than a missing file.""" + + description: str + files: tuple[str, ...] + path_entry: str + expected_error: type[OSError] + expected_message: str diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py new file mode 100644 index 0000000000..2d4e416b6a --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_startup_inputs.py @@ -0,0 +1,85 @@ +import site +import sys +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + StartupInputsErrorTestCase, + StartupInputsTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StartupInputsTestCase( + description="relative, normalized and missing entries with their path files", + folders=("a", "a-b", "a/b"), + files=("a/q.pth", "a/notes.txt", "a-b/.pth"), + path_entries=("a", "./a-b/", "a/b", "gone"), + expected_stamps=( + ("a", True), + ("a/b", True), + ("a/q.pth", True), + ("a-b", True), + ("a-b/.pth", True), + ("gone", False), + ("python", False), + ("pyvenv.cfg", False), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_import_path_directories_when_stamping_then_lists_absolute_paths_and_pth_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: StartupInputsTestCase +) -> None: + for folder in test_case.folders: + (tmp_path / folder).mkdir(parents=True) + for file in test_case.files: + (tmp_path / file).write_text("x\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(sys, "path", list(test_case.path_entries)) + monkeypatch.setattr(sys, "executable", str(tmp_path / "python")) + monkeypatch.setattr(sys, "prefix", str(tmp_path)) + monkeypatch.setattr(sys, "base_prefix", str(tmp_path)) + monkeypatch.setattr(site, "getsitepackages", list) + monkeypatch.setattr(site, "getusersitepackages", lambda: str(tmp_path / "gone")) + + stamps: list[tuple[str, int | None, int | None]] = startup_inputs() + listed: tuple[tuple[str, bool], ...] = tuple( + (path.removeprefix(f"{tmp_path}/"), size is not None) for path, _, size in stamps + ) + + assert listed == test_case.expected_stamps + assert stamps == sorted(stamps, key=lambda row: Path(row[0])) + + +@pytest.mark.parametrize( + "test_case", + [ + StartupInputsErrorTestCase( + description="an entry below a regular file", + files=("file.txt",), + path_entry="file.txt/sub", + expected_error=NotADirectoryError, + expected_message=r"\[Errno 20\] Not a directory", + ), + ], + ids=lambda case: case.description, +) +def test_given_import_path_entry_below_a_file_when_stamping_then_raises_the_stat_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: StartupInputsErrorTestCase +) -> None: + for file in test_case.files: + (tmp_path / file).write_text("x\n") + monkeypatch.setattr(sys, "path", [str(tmp_path / test_case.path_entry)]) + + with pytest.raises(test_case.expected_error, match=test_case.expected_message): + startup_inputs() + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py index 356455b32f..b22f33f9d5 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py @@ -9,9 +9,7 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.frontier.constants import ( - COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, -) +from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR def compile_in_fresh_process( diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index 24853fe94b..65d94956f7 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -14,7 +14,7 @@ ) from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process -_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.compile.models",) +_FULL_COMPILE_MODULES: tuple[str, ...] = ("sqlbuild.compiler.compile.types",) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index 84ae015d09..559a92b544 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -8,13 +8,22 @@ import pytest -from sqlbuild._native import format_audit_execution_json, format_parameterized_test_label -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( + CursorInputEvidence, + FutureCursorAction, + FutureCursorSafetyEvidence, + MaximumStartSafetyEvidence, MeasurementThresholdBound, MeasurementThresholds, + SqlLogicalType, + SqlTestParameterDeclaration, + SqlValue, + SqlValueKind, + format_audit_execution_json, + format_parameterized_test_label, ) +from sqlbuild.adapter.contract.models import LifeCycleEvent +from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -23,12 +32,6 @@ ThresholdOperator, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind -from sqlbuild.compiler.discovery.models import SqlTestParameterDeclaration -from sqlbuild.compiler.planner.models import ( - CursorInputEvidence, - FutureCursorSafetyEvidence, - MaximumStartSafetyEvidence, -) from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run.models import ( MicrobatchAccountingInterval, @@ -43,9 +46,6 @@ StepResult, ) from sqlbuild.executor.testing.types import SqlTestOutcome -from sqlbuild.spec.contracts.types import FutureCursorAction -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind from tests.unit.src.sqlbuild.cli.output._helpers._test_types import ( AuditExecutionProtocolTestCase, FailedModelSqlOutputTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py index 9502a17e50..07a291f365 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_integration_result_measurements.py @@ -6,8 +6,8 @@ import pytest -from sqlbuild._native import IntegrationCheckResult -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( + IntegrationCheckResult, MeasurementThresholdBound, MeasurementThresholds, ) diff --git a/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py b/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py index 6177791a4d..5f1c6f1628 100644 --- a/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py +++ b/tests/unit/src/sqlbuild/cli/output/classes/test_terminal_event_index.py @@ -13,8 +13,10 @@ from sqlbuild._native import ( ExecutionEventWriter, + FutureCursorAction, IntegrationAssetResult, IntegrationResultEnvelope, + MicrobatchLimitAction, TerminalEventClaim, TerminalEventIndex, current_terminal_event_index, @@ -25,7 +27,6 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import LifecycleEvent -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.output.classes._test_types import ( CheckSeverityDecodingTestCase, EnvelopeFieldValidationTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py index a193b0dd5b..56b15cc7ce 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py +++ b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_execution_protocol_v1.py @@ -6,16 +6,14 @@ import pytest -from sqlbuild._native import format_build_execution_json +from sqlbuild._native import MicrobatchLimitAction, PlanReason, format_build_execution_json from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.types import MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.commands.shared._helpers.helpers import build_audit_result from tests.unit.src.sqlbuild.cli.output.main.build_execution_json._test_types import ( ExecutionJsonCostTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py index a60630e85a..6aece1104c 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/_test_types.py @@ -4,11 +4,7 @@ from dataclasses import dataclass, field -from sqlbuild._native import DisplayOptions, PythonPlanEntry -from sqlbuild.compiler.planner.models import ( - CursorInputRelation, - PlanOutput, -) +from sqlbuild._native import CursorInputRelation, DisplayOptions, PlanOutput, PythonPlanEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index 11716681e2..de19758d14 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -5,39 +5,34 @@ from pathlib import Path from sqlbuild._native import ( + BackfillAction, BackfillResult, ColumnRenameHint, - PlanProviderUsage, - PlanWarning, - SchemaChangeKind, - SchemaColumnSource, - WarningSeverity, -) -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage -from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage -from sqlbuild.compiler.planner.models import ( CursorBounds, CursorInputRelation, + DiscoveredProviderUsage, FunctionPlanEntry, + MaterializationType, + MicrobatchLimitAction, ModelPlanEntry, + PlanAction, PlanOutput, + PlanProviderUsage, + PlanReason, + PlanWarning, + SchemaChangeKind, + SchemaColumnSource, SchemaFinding, + SeedCsvSettings, SeedPlanEntry, SourceLoadPlanEntry, + SourceWriteStrategy, TableTypePlanEntry, + WarningSeverity, ) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) -from sqlbuild.spec.contracts.models import SeedCsvSettings -from sqlbuild.spec.contracts.types import MicrobatchLimitAction, SourceWriteStrategy +from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage def build_model_entry( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 21cf8aca85..01747e9e02 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -7,30 +7,26 @@ import pytest from sqlbuild._native import ( + BackfillAction, ColumnRenameHint, + CursorBounds, + CursorInputRelation, DisplayOptions, GraphResourceKind, + MaterializationType, + PlanAction, + PlanOutput, + PlanReason, PythonIdentityStatus, PythonPlanEntry, PythonRunPhase, SchemaChangeKind, + SourceWriteStrategy, WarningSeverity, format_plan, ) from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.cli.output.main.plan._test_types import ( FormatPlanColorTestCase, FormatPlanTestCase, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py index 22ea29272f..4561411ef2 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild._native import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, PythonPlanEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index b714af570a..19192c4d82 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -8,28 +8,25 @@ import pytest from sqlbuild._native import ( - PythonIdentityStatus, - PythonPlanEntry, - PythonRunPhase, - WarningSeverity, - format_plan_json, -) -from sqlbuild.compiler.planner.models import ( + BackfillAction, CursorBounds, CursorInputEvidence, CursorInputRelation, + FutureCursorAction, FutureCursorSafetyEvidence, - MaximumStartSafetyEvidence, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, MaterializationType, + MaximumStartSafetyEvidence, + MicrobatchLimitAction, PlanAction, + PlanOutput, PlanReason, + PythonIdentityStatus, + PythonPlanEntry, + PythonRunPhase, + WarningSeverity, + format_plan_json, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import ( build_discovered_provider_usage, build_model_entry, diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index 35846a539d..ba2277d229 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -9,13 +9,15 @@ from sqlbuild._native import ( BuildProgressCallbacks, + ChainStep, + CompiledObjectKey, NativeProgressProjector, + PlanOutput, + SqlTestPlanEntry, TerminalEventIndex, terminal_event_index_scope, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult diff --git a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py index e4d9c27550..e11cecf15f 100644 --- a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import write_runtime_target -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, write_runtime_target from sqlbuild.executor.build.models import BuildExecutionResult from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( build_runtime_target_execution_result, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/helpers.py index 3288d3173e..ac8a127551 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/helpers.py @@ -1,6 +1,6 @@ """Test helpers for auditing domain models.""" -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound +from sqlbuild._native import MeasurementThresholdBound from sqlbuild.compiler.auditing.types import ThresholdOperator diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py index ae1f3dd4ca..9014dc445c 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.auditing.models import MeasurementThresholds +from sqlbuild._native import MeasurementThresholds from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditOutcome, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py index 79346aabf4..5d7f7b34ca 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py @@ -7,7 +7,14 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import parse_audit_instances +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + SchemaAuditInstance, + SourceEntry, + parse_audit_instances, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.discovery.main.builtins import builtin_adapter_classes from sqlbuild.compiler.auditing.types import ( @@ -15,10 +22,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceEntry def build_render_adapter(adapter_name: str | None) -> BaseAdapter | None: diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py index 8b2dfc7135..5ad5c96978 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_evaluate_measurement.py @@ -2,11 +2,7 @@ import pytest -from sqlbuild._native import evaluate_measurement -from sqlbuild.compiler.auditing.models import ( - MeasurementThresholdBound, - MeasurementThresholds, -) +from sqlbuild._native import MeasurementThresholdBound, MeasurementThresholds, evaluate_measurement from sqlbuild.compiler.auditing.types import AuditOutcome, ThresholdOperator from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( EvaluateMeasurementTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py index c9867d00a4..5d359f92c4 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_identity.py @@ -6,12 +6,13 @@ import pytest -from sqlbuild._native import build_audit_gate_identity -from sqlbuild.compiler.auditing.models import ( +from sqlbuild._native import ( AuditGateIdentity, AuditIdentity, + AuditPlanEntry, MeasurementThresholdBound, MeasurementThresholds, + build_audit_gate_identity, ) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -20,7 +21,6 @@ AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.planner.models import AuditPlanEntry from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( AuditGateAggregateIdentityTestCase, AuditGateIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py index e8731420e6..2ff4dfb4f9 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instance.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import SchemaAuditInstance from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator -from sqlbuild.spec.contracts.models import SchemaAuditInstance from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( MeasurementPolicyParsingTestCase, ParseAuditInstanceErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py index f9476e57a5..3466972905 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_parse_audit_instances.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import parse_audit_instances -from sqlbuild.spec.contracts.models import SchemaAuditInstance +from sqlbuild._native import SchemaAuditInstance, parse_audit_instances from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( ParseAuditInstancesErrorTestCase, ParseAuditInstancesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py b/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py index 35e91797fe..5abc5566aa 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/test_rendering.py @@ -4,10 +4,8 @@ import pytest -from sqlbuild._native import render_audit_sql +from sqlbuild._native import CompiledRelationLocation, SourceEntry, render_audit_sql from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.auditing.main._test_types import ( RenderAuditSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/auditing/test_models.py b/tests/unit/src/sqlbuild/compiler/auditing/test_models.py index 6b698c908b..a3fa5deb10 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/test_models.py @@ -2,12 +2,8 @@ import pytest +from sqlbuild._native import MeasurementContract, MeasurementThresholdBound, MeasurementThresholds from sqlbuild.audits import below -from sqlbuild.compiler.auditing.models import ( - MeasurementContract, - MeasurementThresholdBound, - MeasurementThresholds, -) from sqlbuild.compiler.auditing.types import ThresholdOperator from tests.unit.src.sqlbuild.compiler.auditing._test_types import ( BooleanThresholdTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 459cc67b16..d3d7488e73 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -5,18 +5,24 @@ from pathlib import Path from typing import TYPE_CHECKING -from sqlbuild._native import CompiledLineageColumnFact, ExpansionSpan +from sqlbuild._native import ( + CompiledLineageColumnFact, + CompiledObjectKey, + CompileSqlReference, + ExpansionSpan, + InferredColumn, + ResolvedTimeTravelRetention, + SourceLocation, + SqlLexicalSyntax, + SqlReferenceKind, + SqlValue, + UsageRecord, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileSqlReference, InferredColumn from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.compiler.lineage.types import ( InferredNullability, ) -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.scopes.models import UsageRecord -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, SourceLocation -from sqlbuild.sql_values.models import SqlValue if TYPE_CHECKING: from sqlbuild._native import CompileInputsView diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index 547af16fd6..8b870dd850 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -5,10 +5,10 @@ from sqlbuild._native import ( DynamicColumnContractProof, NativePivotTables, + SchemaDynamicColumnFamily, prove_native_dynamic_contracts, ) from sqlbuild._native import dynamic_contract_proof as contract_proof -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.compile._helpers.analysis._test_types import ( DynamicPivotAnalysisTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py index abcfe1084f..2d518305b9 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import MaterializationDefaultsConfig, TargetConfig -from sqlbuild.spec.contracts.types import TableType, TableTypeSource +from sqlbuild._native import MaterializationDefaultsConfig, TableType, TableTypeSource, TargetConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 0412e70802..e22b83aaab 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -8,23 +8,23 @@ import sqlbuild._native as _native from sqlbuild._native import ( CompileModelConfig, - NativeModelConfigInputs, - NativeModelConfigSession, - build_compile_inputs, - discover_project_inputs, - native_model_config_session, - record_template_reads, -) -from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, + NativeModelConfigInputs, + NativeModelConfigSession, ProjectConfig, ResolvedTableType, ResolvedTimeTravelRetention, SchemaColumn, + TableType, + TableTypeSource, TargetConfig, + TimeTravelRetentionSource, + build_compile_inputs, + discover_project_inputs, + native_model_config_session, + record_template_reads, ) -from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py index dec868dd0b..7b1e157a0c 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/test_table_type.py @@ -2,16 +2,17 @@ import pytest -from sqlbuild._native import CompileModelConfig -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + CompileModelConfig, DefaultsConfig, MaterializationDefaultsConfig, MaterializationRetentionDefaults, ResolvedTableType, + TableType, + TableTypeSource, TargetConfig, ) -from sqlbuild.spec.contracts.types import TableType, TableTypeSource +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers.config._test_types import ( TableTypeResolutionErrorTestCase, TableTypeResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index fc68224946..904478c5df 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -14,12 +14,33 @@ CompileAdapterContext, CompiledDirectLogicSqlTestPayload, CompiledLineageColumnFact, + CompiledLineageSourceFact, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledRelationLocation, + CompiledSqlTest, + CompilerDiagnostic, + CompileSqlReference, DeclarationExpansionContext, + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, + DeclarationResolutionContext, DeclarationRuntimeProjection, DeclarationScopeResolver, + DiscoveredMacroFile, + DiscoveredProjectInputs, + LoadedMacro, + MacroMetadata, NativeColumnQuery, NativeQueryColumns, + OwnershipRoot, + ResourceIdentity, + ResourceKind, + ResourceRecord, + ScopeIndex, + ScopeKind, + ScopeLookup, assemble_project, build_compile_inputs, build_execution_upstream_deps, @@ -41,38 +62,13 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import ( - CompiledLineageSourceFact, - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlTest, - CompilerDiagnostic, - CompileSqlReference, - DeclarationResolutionContext, - LoadedMacro, - MacroContext, -) +from sqlbuild.compiler.compile.models import MacroContext from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredMacroFile, - DiscoveredProjectInputs, -) from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - MacroMetadata, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeIndex, - ScopeLookup, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind if TYPE_CHECKING: from sqlbuild._native import CompileInputsView, ModelInput diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index 24af81b145..49e2ce9725 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + InferredColumn, NativeColumnQuery, cursor_intrinsics_analysis_sql, infer_native_query_columns, @@ -11,7 +12,6 @@ ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.lineage.types import InferredNullability from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( CursorIntrinsicAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index 1c5b701ff9..cf62a2121f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -6,18 +6,21 @@ import pytest -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, + DiscoveredProjectInputs, + SqlHookEntry, + assemble_project, + build_compile_inputs, + discover_project_inputs, ) +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, CompiledResourceType, CompiledSeedView, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AssembleCompiledProjectEffectiveTargetTestCase, AssembleCompiledProjectTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 3404d677c3..8f541715e3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -8,9 +8,13 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + SchemaModelEntry, + SourceLocation, + build_compile_inputs, + discover_project_inputs, +) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AuditFactoryAttachmentTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index 10d1a9b0e6..beef2dd785 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -10,6 +10,16 @@ import pytest from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, + DiscoveredProjectInputs, + DiscoveredSourceFile, + ResourceIdentity, + ResourceKind, + SourceEntry, + SqlHookEntry, + UsageKind, + UsageRecord, assemble_project, build_compile_inputs, discover_project_inputs, @@ -17,22 +27,6 @@ ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSourceFile, - SqlHookEntry, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - UsageKind, -) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedBooleanTestCase, ExpectedModelDeclarationGrantTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index 9a08befc21..c5000b10f7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -9,7 +9,9 @@ from sqlbuild._native import ( DeclarationParseError, + DiscoveredProjectInputs, ResourceIdentityError, + SqlHookEntry, assemble_project, build_compile_inputs, build_model_version_identity_metadata_json, @@ -18,7 +20,6 @@ ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CompileDeclarationsErrorTestCase, CompileDeclarationsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py index af860d16b1..acc1df5771 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DescriptionInputClassificationCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py index 33ee290dfc..2660dfddf4 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expansion_spans.py @@ -8,11 +8,12 @@ from sqlbuild._native import ( ExpansionSpan, + LoadedMacro, MappedOffset, expand_authored_sql_with_spans, map_expanded_offset, ) -from sqlbuild.compiler.compile.models import LoadedMacro, MacroContext +from sqlbuild.compiler.compile.models import MacroContext from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ExpandWithSpansTestCase from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py index 17e5d1a51b..e9e012828f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_macros.py @@ -5,15 +5,16 @@ import pytest -from sqlbuild._native import expand_sql_macros, find_macro_call_names -from sqlbuild.compiler.compile._helpers.render import user_macros as macros_module -from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + DiscoveredMacroFile, LoadedMacro, MacroContext, + ScopeKind, + expand_sql_macros, + find_macro_call_names, ) -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile -from sqlbuild.compiler.scopes.types import ScopeKind +from sqlbuild.compiler.compile._helpers.render import user_macros as macros_module +from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandSqlMacrosErrorTestCase, ExpandSqlMacrosTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py index 7e93361a45..ea02a03e59 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_microbatch_redesign.py @@ -6,13 +6,16 @@ from sqlbuild._native import ( CompileModelConfig, + CursorBounds, + CursorInputRelation, + DateValue, + ModelCursorSnapshot, + TimestampValue, compute_cursor_bounds, resolve_effective_timestamp_grain, watermark_type_is_compatible, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.models import CursorBounds, CursorInputRelation, ModelCursorSnapshot -from sqlbuild.cursor_algebra.models import DateValue, TimestampValue from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( resolve_effective_timestamp_grain as resolve_runtime_effective_timestamp_grain, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index c0948d8dfe..17b779ab5a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -6,10 +6,13 @@ import pytest -from sqlbuild._native import CompileModelConfig, ConfigValueTypeError +from sqlbuild._native import ( + CompileModelConfig, + ConfigValueTypeError, + ResolvedTimeTravelRetention, + TimeTravelRetentionSource, +) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ChangePolicyHelpTestCase, ContractConfigErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 7440b70bba..bc73d41f34 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -8,6 +8,11 @@ import pytest from sqlbuild._native import ( + CompilerDiagnostic, + DiscoveredProjectInputs, + SchemaAuditInstance, + SchemaModelEntry, + SourceLocation, assemble_project, build_compile_inputs, build_model_version_identity_metadata_json, @@ -16,12 +21,7 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, -) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, ExpectedCountTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 1a3b2a3110..8ce9866e21 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -7,21 +7,23 @@ import pytest -from sqlbuild._native import assemble_project, scope_overlaps -from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, CompiledSqlTest, + SqlLogicalType, + SqlValue, + SqlValueKind, + assemble_project, + scope_overlaps, ) +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, TypedSqlValueRenderer, ) -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ParameterizedSqlTestAdapterRenderingTestCase, ParameterizedSqlTestCompilationErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 28a9954852..edf999d9d1 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -8,17 +8,17 @@ import pytest from sqlbuild._native import ( + DeclarationIdentity, + DeclarationKind, DeclarationScopeBuild, + DiscoveredProjectInputs, + LoadedMacro, + SqlLexicalSyntax, build_declaration_scope, discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.user_macros import load_project_macros -from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index f07166bac9..7c3ffd22c2 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -4,10 +4,13 @@ import pytest -from sqlbuild._native import analyze_resolved_column_reads -from sqlbuild.compiler.compile.models import CompiledLineageSourceFact, CompileSqlReference +from sqlbuild._native import ( + CompiledLineageSourceFact, + CompileSqlReference, + SqlReferenceKind, + analyze_resolved_column_reads, +) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( LongUnionRequiredColumnsTestCase, RequiredExternalColumnsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py index 0d074171cf..0f5f16a290 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py @@ -7,8 +7,8 @@ import pytest +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.scopes.types import DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( NamedDeclarationAcceptedTestCase, NamedDeclarationErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 0f73e79bee..4ef9a72518 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -6,20 +6,18 @@ ColumnLineageConfidence, ColumnTransformKind, CompiledLineageColumnFact, + CompiledLineageSourceFact, + CompileSqlReference, + InferredColumn, NativeColumnQuery, NativeQueryColumns, + SqlReferenceKind, infer_native_query_columns, substitute_placeholder_defaults, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompiledLineageSourceFact, - CompileSqlReference, - InferredColumn, -) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( InferColumnsTestCase, PolyglotAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py index ab3eaf9d83..9db54e9764 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_validation.py @@ -5,7 +5,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ValidateHookSqlSyntaxTestCase, ValidateSqlSyntaxTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index fcb2625574..1e4c1265e5 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -6,16 +6,19 @@ import pytest -from sqlbuild._native import collect_compile_diagnostics, extract_sql_references +from sqlbuild._native import ( + CompileInputError, + CompileSqlReference, + SqlLexicalSyntax, + SqlReferenceKind, + collect_compile_diagnostics, + extract_sql_references, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CollectedReferenceCallSyntaxTestCase, DialectSqlScanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index d070fba082..06d1cc8571 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import substitute_sql_vars_with_spans -from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild._native import CompileInputError, substitute_sql_vars_with_spans from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( SubstituteSqlVarsErrorTestCase, SubstituteSqlVarsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py index 8830420024..05f0680f65 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_templating.py @@ -3,11 +3,11 @@ import pytest from sqlbuild._native import ( + CompileInputError, UnicodeEnvironment, expand_effective_vars, expand_template_data, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpandTemplateDataErrorTestCase, ExpandTemplateDataTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index c828ed11da..fa0b2e5a9b 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild._native import CompileInputError, CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( MacroGeneratedReferenceCollectionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py index 20a9a56759..c0cc5cd6a7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_helpers.py @@ -9,7 +9,7 @@ CompileSqlTestInput, build_compile_reference_resolver, ) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver +from sqlbuild.compiler.compile.types import ExternalSqlReferenceResolver def base_repo_files() -> dict[str, str]: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_test_types.py index f1952b41bd..6039b5cc98 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_test_types.py @@ -1,11 +1,11 @@ from dataclasses import dataclass, field -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( DefaultsConfig, MaterializationDefaultsConfig, TargetConfig, + TimeTravelRetentionSource, ) -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index 00c1a2ab70..0cdc703b1f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs, build_compile_inputs, discover_project_inputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py index 6617c86aa4..120fa04c5e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_effective_runtime.py @@ -4,13 +4,14 @@ import pytest -from sqlbuild._native import build_effective_runtime_config, resolve_run_id -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + DiscoveredProjectInputs, LocalConfig, LocalTargetConfig, ProjectConfig, TargetConfig, + build_effective_runtime_config, + resolve_run_id, ) from tests.unit.src.sqlbuild.compiler.compile._test_types import ( BuildEffectiveRuntimeConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 8729c83215..b182f19296 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -9,32 +9,28 @@ import pytest from sqlbuild._native import ( - CompileModelConfig, - ResourceIdentityError, - SpecConfigError, - build_compile_inputs, - discover_project_inputs, - resolve_target_config, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, + CompileModelConfig, DefaultsConfig, + DiscoveredProjectInputs, LocalClonePolicy, LocalConfig, LocalTargetConfig, MaterializationDefaultsConfig, MaterializationRetentionDefaults, ProjectConfig, + PythonHookEntry, ResolvedTimeTravelRetention, + ResourceIdentityError, + SpecConfigError, + SqlHookEntry, TargetConfig, + TimeTravelRetentionSource, + build_compile_inputs, + discover_project_inputs, + resolve_target_config, ) -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers.config.helpers import build_config_natively from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index a5c718b2d3..aacbaf32ff 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs, build_compile_inputs, discover_project_inputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 6360d6504a..3aed31fcf7 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -1,10 +1,14 @@ import pytest -from sqlbuild._native import ContractEvidence, compare_contract_evidence, edit_contract_model +from sqlbuild._native import ( + ContractEvidence, + SchemaDynamicColumnFamily, + compare_contract_evidence, + edit_contract_model, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.contract_adoption._helpers._test_types import ( DynamicContractAdoptionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index e8f754aaa2..61471d09ce 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -3,17 +3,15 @@ from pathlib import Path from typing import cast -from sqlbuild._native import CompileModelConfig, DynamicColumnContractProof -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + ColumnContractMode, CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, + DynamicColumnContractProof, InferredColumn, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import ( SchemaAuditInstance, SchemaColumn, SchemaDynamicColumnFamily, @@ -21,7 +19,8 @@ SettingsConfig, SourceLocation, ) -from sqlbuild.spec.contracts.types import ColumnContractMode +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.lineage.types import InferredNullability def make_contract_project( diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py b/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py index 6e9d014531..f8c848a6b0 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_promotion_conflicts.py @@ -2,9 +2,8 @@ import pytest -from sqlbuild._native import promotion_conflict_diagnostics +from sqlbuild._native import CompilerDiagnostic, promotion_conflict_diagnostics from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompilerDiagnostic from tests.unit.src.sqlbuild.compiler.contracts._test_types import PromotionConflictTestCase from tests.unit.src.sqlbuild.compiler.contracts.helpers import make_promotion_project diff --git a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py index 47fb4428f7..6bb37dd1d1 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/test_validate.py @@ -8,11 +8,12 @@ ContractValidationResult, DynamicColumnContractProof, DynamicColumnFamilyProof, + SchemaDynamicColumnFamily, + SourceLocation, evaluate_model_contracts, ) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceLocation from tests.unit.src.sqlbuild.compiler.contracts._test_types import ( ContractLocationTestCase, ContractMissingDeclarationsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index 49834cbe00..f6a4f748c2 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -3,47 +3,41 @@ from pathlib import Path from sqlbuild._native import ( + CompiledAudit, CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, + CompiledSqlScenario, + CompiledSqlTest, CompileModelConfig, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredLoaderFunction, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, FunctionArgument, + ProjectGraph, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SourceColumnEntry, + SourceEntry, ) from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceColumnEntry, - SourceEntry, -) from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index ba96505a1a..14303e8a88 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -9,6 +9,9 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + DiscoveredProjectInputs, + ProjectGraph, assemble_project, build_compile_inputs, build_dag_json, @@ -16,13 +19,8 @@ project_lineage_views, project_with, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.dag.types import NodeKind -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index db5f550b6e..6c95e35e99 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -12,6 +12,7 @@ import sqlbuild._native as _native from sqlbuild._native import ( + DiscoveredProjectInputs, discover_audit_files, discover_constant_files, discover_enum_files, @@ -24,7 +25,6 @@ discover_sql_hook_files, discover_test_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import ( load_local_config, load_project_config, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 12997eba18..1b4112bbc6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -2,9 +2,13 @@ from decimal import Decimal from pathlib import Path -from sqlbuild._native import CollectionRendering -from sqlbuild.spec.contracts.models import AuthoredTimeTravelRetention, TargetWarehousesConfig -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy +from sqlbuild._native import ( + AuthoredTimeTravelRetention, + CollectionRendering, + MissingMigrationOriginPolicy, + RetentionDecreasePolicy, + TargetWarehousesConfig, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py index 0d0d63f921..8c714f268f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/helpers.py @@ -6,9 +6,13 @@ from sqlbuild._native import ( DiscoveredConstantFile, DiscoveredEnumFile, + DiscoveredMacroFile, DiscoveredPythonFunctionFile, DiscoveredSqlTestBlock, DiscoveredSqlTestCase, + LocalConfig, + ProjectConfig, + SqlTestParameterDeclaration, discover_audit_files, discover_constant_files, discover_enum_files, @@ -18,8 +22,6 @@ discover_sql_function_files, discover_sql_hook_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, SqlTestParameterDeclaration -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( LoadProjectConfigTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py index 18b7540228..d78d981546 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py @@ -7,9 +7,10 @@ import pytest -from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredPythonNodeFunctions, + PythonNodeDiscoveryError, + discover_python_node_functions, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( AuditFactoryDiscoveryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py index 48cc3d5cb7..e5bd1975e0 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_command_output_sinks.py @@ -6,16 +6,14 @@ import pytest from sqlbuild._native import ( + DiscoveredCommandOutputSink, + DiscoveredProvider, EventExporterDiscoveryError, discover_provider_classes, ) from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_functions, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredProvider, -) from sqlbuild.sinks import CommandOutputStream from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( CommandOutputSinkDiscoveryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py index 7244ce5182..419fe53c88 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_event_exporters.py @@ -6,14 +6,12 @@ import pytest from sqlbuild._native import ( + DiscoveredEventExporter, + DiscoveredProvider, EventExporterDiscoveryError, discover_event_exporter_functions, discover_provider_classes, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredEventExporter, - DiscoveredProvider, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( EventExporterDiscoveryTestCase, EventExporterSignatureErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py index ce2714758c..4b248fb85d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py @@ -7,9 +7,10 @@ import pytest -from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredPythonNodeFunctions, + PythonNodeDiscoveryError, + discover_python_node_functions, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverPythonNodeFactoriesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py index 071c411e05..89e0f856cc 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_hooks.py @@ -6,14 +6,12 @@ import pytest from sqlbuild._native import ( - PythonNodeDiscoveryError, - discover_hook_functions, - discover_provider_classes, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredHookFunction, DiscoveredProvider, DiscoveredProviderUsage, + PythonNodeDiscoveryError, + discover_hook_functions, + discover_provider_classes, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverHookFunctionsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index 74c3d57a2f..735972e909 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -6,9 +6,10 @@ import pytest -from sqlbuild._native import PythonNodeDiscoveryError, discover_python_node_functions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredLoaderFunction, + PythonNodeDiscoveryError, + discover_python_node_functions, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverLoaderFunctionsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py index b73f54fb5d..afeda43c52 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_macros.py @@ -7,9 +7,9 @@ import pytest from sqlbuild._native import ( + DiscoveredMacroFile, discover_macro_files, ) -from sqlbuild.compiler.discovery.models import DiscoveredMacroFile from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverMacroFilesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py index 6e031cbed8..4943308fea 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_materializations.py @@ -7,13 +7,11 @@ import pytest from sqlbuild._native import ( - discover_materialization_files, - discover_provider_classes, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredMaterializationFile, DiscoveredProvider, DiscoveredProviderUsage, + discover_materialization_files, + discover_provider_classes, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverMaterializationFilesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py index d599d561f7..b7f2416dfd 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_providers.py @@ -6,13 +6,11 @@ import pytest from sqlbuild._native import ( + DiscoveredProvider, ProviderDiscoveryError, ResourceIdentityError, discover_provider_classes, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProvider, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverProviderCacheIsolationTestCase, DiscoverProviderClassesErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 34dfb34ca0..b88b747bd4 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -11,17 +11,15 @@ import pytest from sqlbuild._native import ( - PythonNodeDiscoveryError, - discover_python_node_functions, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, + PythonNodeDiscoveryError, + PythonNodeIdentity, + discover_python_node_functions, ) from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverCheckFunctionsTestCase, DiscoverPythonRootImportTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py index 86778b6f9f..6058740b9c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_scoped_declarations.py @@ -12,6 +12,8 @@ DeclarationParseError, DiscoveredConstantFile, DiscoveredEnumFile, + DiscoveredMacroFile, + DiscoveredProjectInputs, discover_audit_files, discover_hook_functions, discover_macro_files, @@ -23,10 +25,6 @@ discover_sql_hook_files, discover_test_files, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredMacroFile, - DiscoveredProjectInputs, -) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverGlobalDeclarationTestCase, DiscoverGroupedDeclarationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index 2c8db4c589..a4d077b83a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -9,15 +9,16 @@ import sqlbuild._native as _native from sqlbuild._native import ( AuthoredSqlValueCall, + ConstantDeclaration, DeclarationParseError, DiscoveredConstantFile, DiscoveredEnumFile, DiscoveredModelSchemaFile, + EnumDeclaration, ModelSchemaDeclaration, ResourceIdentityError, + SqlValue, ) -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration -from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeclarationResourceIdentityErrorTestCase, ParseDeclarationFileErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 35921e30df..bf6107ed94 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -8,13 +8,15 @@ from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, + DiscoveredSqlModelFile, DiscoveryFileFault, NamedSqlHookEntry, + PythonHookEntry, + SourceLocation, + SqlHookEntry, discover_model_files, model_output_column_locations, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, PythonHookEntry, SqlHookEntry -from sqlbuild.spec.contracts.models import SourceLocation from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeferredModelOutputLocationTestCase, ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py index 9431ff703c..ac8b2b784a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_scenarios.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import SqlScenarioParseError -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile +from sqlbuild._native import DiscoveredSqlScenarioFile, SqlScenarioParseError from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSqlScenarioFileErrorTestCase, ParseSqlScenarioFileTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 35f64e200f..6aaaf361d3 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -9,13 +9,14 @@ from sqlbuild._native import ( DiscoveredSqlTestBlock, DiscoveredSqlTestCase, + DiscoveredSqlTestFile, DiscoveryFileFault, + SqlLexicalSyntax, + SqlTestParameterDeclaration, SqlTestParseError, discover_test_files, omitted_ceremonial_select, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile, SqlTestParameterDeclaration -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ExpectedCountTestCase, OmittedSelectPerformanceTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py index 1bfc803853..413b040910 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_warehouses.py @@ -6,9 +6,12 @@ import pytest -from sqlbuild._native import ProjectConfigError, discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import TargetWarehousesConfig +from sqlbuild._native import ( + DiscoveredProjectInputs, + ProjectConfigError, + TargetWarehousesConfig, + discover_project_inputs, +) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( TargetWarehousesConfigErrorTestCase, TargetWarehousesConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index a70f2a3a52..81809b20ff 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -6,16 +6,18 @@ import pytest -from sqlbuild._native import CollectionRendering, ProjectConfigError -from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( AuthoredTimeTravelRetention, + CollectionRendering, ExecutionLimitsConfig, LocalConfig, + MissingMigrationOriginPolicy, ProjectConfig, + ProjectConfigError, + RetentionDecreasePolicy, TargetConfig, ) -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy, RetentionDecreasePolicy +from sqlbuild.runtime.event_exporting.types import LifecycleEventKind from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ColumnContractModeConfigErrorTestCase, ColumnContractModeConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py index c6440d4e47..1da48302b4 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_schema.py @@ -4,12 +4,7 @@ import pytest -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SeedCsvSettings, -) +from sqlbuild._native import SchemaColumn, SchemaModelEntry, SchemaSeedEntry, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSchemaYamlErrorTestCase, ParseSchemaYamlTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 721e2e09c3..fd57f1a902 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -5,11 +5,9 @@ import pytest -from sqlbuild._native import SourceParseError +from sqlbuild._native import SourceEntry, SourceParseError, SourceWriteStrategy from sqlbuild.integrations.dlt.models import DltSourceConfig from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSourcesYamlDltTestCase, ParseSourcesYamlErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index 40699cae19..a35a152be8 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,9 +1,10 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import DiscoveredSqlTestBlock -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveredSqlScenarioFile -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( + DiscoveredSqlModelFile, + DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, SchemaModelEntry, SchemaSeedEntry, SourceEntry, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py index 853a33d646..4304bec241 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import serialize_hook_entries -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild._native import PythonHookEntry, SqlHookEntry, serialize_hook_entries from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import ( SerializeHookEntriesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 1ba30ebcf2..57fdddbf9e 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -7,12 +7,12 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, DiscoveredSqlTestBlock, ProjectConfigError, ProjectPythonPathError, discover_project_inputs, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.compiler.discovery._test_helpers import ( base_repo_files, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py index b1f5e23222..97cd46cb36 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py @@ -2,12 +2,12 @@ import pytest -from sqlbuild._native import ProjectConfigError -from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredEventExporter, DiscoveredRuntimeExtensions, + ProjectConfigError, ) +from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions from tests.unit.src.sqlbuild.compiler.discovery._test_types import ( EventExporterConfigTestCase, InvalidEventExporterConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py index e12d8992c8..18bcca05ab 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/helpers.py @@ -5,8 +5,7 @@ from types import MappingProxyType from typing import Any -from sqlbuild._native import build_read_latest_sql -from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild._native import Fingerprint, build_read_latest_sql class FakeFingerprintResult: diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py index 04bd93ed9d..92cfc7dc16 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/shared/_helpers/test_sql.py @@ -5,12 +5,16 @@ import pytest -from sqlbuild._native import build_create_table_sql, build_read_latest_sql, write_fingerprint +from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + Fingerprint, + build_create_table_sql, + build_read_latest_sql, + write_fingerprint, +) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.fingerprints.main.shared._helpers._test_types import ( BuildCreateTableSqlTestCase, BuildInsertSqlTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py index 2461ffca0f..aa2738c008 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_read.py @@ -5,9 +5,14 @@ import pytest -from sqlbuild._native import FingerprintInputError, read_latest_fingerprints, write_fingerprint +from sqlbuild._native import ( + Fingerprint, + FingerprintInputError, + FingerprintSet, + read_latest_fingerprints, + write_fingerprint, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( ReadLatestFingerprintsErrorTestCase, ReadLatestFingerprintsFilterTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py index dc537b040f..be01fb9f6e 100644 --- a/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py +++ b/tests/unit/src/sqlbuild/compiler/fingerprints/main/test_write.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import write_fingerprint, write_fingerprints +from sqlbuild._native import Fingerprint, write_fingerprint, write_fingerprints from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.fingerprints.main._test_types import ( WriteFingerprintBatchTestCase, WriteFingerprintPartialFailureTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py index 192e943531..26ada14e85 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -7,8 +7,7 @@ from pathlib import PurePosixPath from typing import Any, cast -from sqlbuild._native import BindingCatalog, DeclarationScopeBuild -from sqlbuild.compiler.compile.models import CompactLineageFacts +from sqlbuild._native import BindingCatalog, CompactLineageFacts, DeclarationScopeBuild HELPERS_MODULE: str = "tests.unit.src.sqlbuild.compiler.frontier.helpers" diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 6eab3842bf..86d7bd65ee 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -9,8 +9,12 @@ import pytest from scripts.compiler_differential.classes.capture_file import expand_capture_text -from sqlbuild._native import CompilerStage, compile_frontier, render_stage_capture -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild._native import ( + STAGE_CAPTURE_DIR_ENV_VAR, + CompilerStage, + compile_frontier, + render_stage_capture, +) from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( FrontierCaptureTestCase, OmittedFieldTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/graph/helpers.py b/tests/unit/src/sqlbuild/compiler/graph/helpers.py index 55ae3cb760..7bfa72887f 100644 --- a/tests/unit/src/sqlbuild/compiler/graph/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/graph/helpers.py @@ -4,10 +4,8 @@ from collections.abc import Iterable, Mapping -from sqlbuild._native import native_graph_from_views -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, ProjectGraph, native_graph_from_views from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline.models import ProjectGraph def project_graph_from_indexes( diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index f0e2445b1d..4d55aeb1a8 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -5,18 +5,19 @@ import sqlbuild._native as native from sqlbuild._native import ( ColumnLineageMode, - CompiledSeed, - CompileModelConfig, - DiscoveredSchemaFile, - DiscoveredSeedFile, - ProjectColumnLineage, -) -from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledSeed, + CompileModelConfig, + DiscoveredSchemaFile, + DiscoveredSeedFile, InferredColumn, + ProjectColumnLineage, + SchemaColumn, + SchemaSeedEntry, + SettingsConfig, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, @@ -24,7 +25,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SettingsConfig def make_compiled_project( diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 2c0d24efa4..3337fea7b2 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -10,11 +10,11 @@ ColumnLineageMode, ColumnTransformKind, CompiledLineageColumnFact, + CompiledLineageSourceFact, ModelColumnLineage, ProjectColumnLineage, compiled_model_record, ) -from sqlbuild.compiler.compile.models import CompiledLineageSourceFact from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py index 9d68bb05c8..441d58fe03 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from pathlib import Path -from sqlbuild.compiler.compile.models import DeclarationResolutionContext, MacroContext +from sqlbuild._native import DeclarationResolutionContext, MacroContext @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py index acb50cbbe0..cfbc6d9a38 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/helpers.py @@ -14,14 +14,19 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import DeclarationResolutionContext, LoadedMacro, MacroContext -from sqlbuild.compiler.discovery.models import ConstantDeclaration +from sqlbuild._native import ( + ConstantDeclaration, + DeclarationResolutionContext, + LoadedMacro, + MacroContext, + SqlLogicalType, + SqlValue, + SqlValueKind, +) from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( project_fingerprint, store_environment, ) -from sqlbuild.sql_values.models import SqlLogicalType, SqlValue -from sqlbuild.sql_values.types import SqlValueKind BASE_CONTEXT: MacroContext = MacroContext( adapter_name="duckdb", diff --git a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py index 1974575b1c..de632b3402 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py @@ -1,10 +1,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) +from sqlbuild._native import CompiledObjectKey, ModelPlanEntry, PlanOutput from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 2d66988996..5d68f33411 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -7,12 +7,32 @@ from typing import Any, cast from sqlbuild._native import ( + AuditPlanEntry, + ChainStep, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, CompileModelConfig, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, DiscoveredSqlHookFile, + LoadedMacro, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SeedPlanEntry, + SourceColumnEntry, + SourceEntry, + SqlTestPlanEntry, build_manifest, ) from sqlbuild.compiler.auditing.types import ( @@ -20,13 +40,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - LoadedMacro, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, @@ -34,27 +47,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ChainStep, - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - SqlTestPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, -) -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SourceColumnEntry, - SourceEntry, -) def model_key(name: str) -> CompiledObjectKey: diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 022f77edd1..4c58d21fcc 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -9,31 +9,33 @@ import sqlbuild._native as _native from sqlbuild._native import ( + AuditPlanEntry, + ChainStep, + CompiledObjectKey, + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DiscoveredSqlHookFile, + LoadedMacro, + MacroMetadata, + OwnershipRoot, + PlanOutput, + ResourceKind, + SchemaColumn, + ScopeIndex, + ScopeKind, + SourceColumnEntry, + SqlHookEntry, + SqlTestPlanEntry, build_manifest, project_with, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - LoadedMacro, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - MacroMetadata, - OwnershipRoot, - ScopeIndex, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind -from sqlbuild.spec.contracts.models import SchemaColumn, SourceColumnEntry from tests.unit.src.sqlbuild.compiler.manifest._test_types import ( FqnTestCase, ManifestAuditNodeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py index 28e1acf3db..48a52f2a35 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/_test_types.py @@ -4,8 +4,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewDropReason, OldNameViewEventType +from sqlbuild._native import MigrationRelation, OldNameViewDropReason, OldNameViewEventType @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py index 89b08c4e20..0667c33bbb 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/helpers.py @@ -5,21 +5,17 @@ from datetime import UTC, datetime, timedelta from sqlbuild._native import ( - MigrationDiscovery, - deterministic_column_migration_event_id, - deterministic_migration_event_id, - deterministic_old_name_view_event_id, -) -from sqlbuild.compiler.migrations.models import ( + ColumnMigrationDecision, ColumnMigrationEvent, + MigrationDecision, + MigrationDiscovery, MigrationEvent, MigrationRelation, OldNameViewEvent, -) -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, - MigrationDecision, OldNameViewEventType, + deterministic_column_migration_event_id, + deterministic_migration_event_id, + deterministic_old_name_view_event_id, ) _STARTED_AT: datetime = datetime(2026, 1, 1, tzinfo=UTC) diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py index 916b3c6873..813e179bff 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_column_migration_events.py @@ -6,12 +6,12 @@ import pytest from sqlbuild._native import ( + ColumnMigrationEvent, newest_column_migration_event_mentioning, read_migration_state_events, write_migration_state_event, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( ColumnEventStorageTestCase, NewestColumnEventTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py index 7c1e77718d..3ba92ab6bb 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_migration_events.py @@ -6,12 +6,12 @@ import pytest from sqlbuild._native import ( + MigrationEvent, newest_migration_event_mentioning, read_migration_state_events, write_migration_state_event, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import MigrationEvent from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( NewestEventTestCase, RepeatedEventWriteTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py index eb25c85b74..7c2bb93826 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_old_name_view_events.py @@ -8,19 +8,17 @@ import pytest from sqlbuild._native import ( + MigrationEvent, MigrationStateInspection, + OldNameViewEvent, + OldNameViewEventType, + OldNameViewHistory, project_old_name_views, read_migration_state_events, stored_old_name_view_columns, write_migration_state_event, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ( - MigrationEvent, - OldNameViewEvent, - OldNameViewHistory, -) -from sqlbuild.compiler.migrations.types import OldNameViewEventType from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( OldNameViewAbsentColumnsTestCase, OldNameViewProjectionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py index e873a56e4a..8626cdb977 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_pinned_event_ids.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + MigrationRelation, + OldNameViewEventType, deterministic_column_migration_event_id, deterministic_migration_event_id, deterministic_old_name_view_event_id, source_freshness_data_version_hash, ) -from sqlbuild.compiler.migrations.models import MigrationRelation -from sqlbuild.compiler.migrations.types import OldNameViewEventType from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( PinnedColumnEventIdTestCase, PinnedFreshnessHashTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py index 0b18656d46..70d8a7107e 100644 --- a/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py +++ b/tests/unit/src/sqlbuild/compiler/migrations/main/test_stored_event_rows.py @@ -7,10 +7,14 @@ import duckdb import pytest -from sqlbuild._native import read_migration_state_events, stored_old_name_view_columns +from sqlbuild._native import ( + ColumnMigrationEvent, + OldNameViewDropReason, + OldNameViewEvent, + read_migration_state_events, + stored_old_name_view_columns, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, OldNameViewEvent -from sqlbuild.compiler.migrations.types import OldNameViewDropReason from tests.unit.src.sqlbuild.compiler.migrations.main._test_types import ( StoredColumnRowTestCase, StoredOldNameRowTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index b40947306c..0271fefc1c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -5,19 +5,19 @@ from datetime import datetime from pathlib import Path -from sqlbuild._native import CompileModelConfig -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, + Fingerprint, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint def build_single_model_project( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py index 85af573fe4..30fbe439c9 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/_test_types.py @@ -1,8 +1,7 @@ from dataclasses import dataclass +from sqlbuild._native import CompiledRelationLocation, LocalConfig, ProjectConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index b52740a712..85eb362f4b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -5,14 +5,14 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompileModelConfig -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index 4620e2c515..c7a0cdfd5e 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -5,6 +5,14 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, + DiscoveredProjectInputs, + DiscoveredSourceFile, + LocalConfig, + LocalTargetConfig, + ProjectConfig, + SourceEntry, + TargetConfig, validate_managed_loader_target_isolation, validate_managed_write_schemas, validate_named_target_schema_strategy, @@ -18,18 +26,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile -from sqlbuild.spec.contracts.models import ( - LocalConfig, - LocalTargetConfig, - ProjectConfig, - SourceEntry, - TargetConfig, -) from tests.unit.src.sqlbuild.compiler.pipeline._helpers.target_validation._test_types import ( ValidateManagedLoaderConnectionTestCase, ValidateManagedWriteSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py index 34b1eacb9d..e6c36f232e 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_clone.py @@ -6,11 +6,13 @@ ClonePipelineConnection, ClonePipelineOptions, ClonePipelineResult, + DiscoveredProjectInputs, + LocalConfig, + ProjectConfig, + TargetConfig, run_clone_pipeline_with_options, ) from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( CloneConnectionInjectionTestCase, CloneTargetSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index 2bead4698d..ed79bf8e16 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -4,14 +4,10 @@ import pytest -from sqlbuild._native import build_deferred_locations +from sqlbuild._native import CompiledRelationLocation, TargetConfig, build_deferred_locations from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.spec.contracts.models import TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( DeferredTargetTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py index 368bc2777b..ec6bb7e0b5 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_python_plan_entries.py @@ -8,15 +8,17 @@ import pytest from sqlbuild._native import ( + Fingerprint, + FingerprintSet, PythonIdentityStatus, + PythonNodeGraph, + PythonNodeIdentity, PythonPlanEntry, PythonSqlRunSelection, build_python_plan_entries, build_python_sql_run_lifecycle, read_latest_fingerprints, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.fingerprints.main.helpers import ( FakeFingerprintExecute, render_qualified_name, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py index 794f94504b..1c39fe8ed9 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py @@ -1,7 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, PythonPlanEntry from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 73fef6bf95..4437f1965b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -3,18 +3,21 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompiledSource -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledSource, + DiscoveredSourceFile, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + SourceEntry, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.spec.contracts.models import SourceEntry class RelationTargetTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 0da2b02e2e..0a41ca55d2 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -11,15 +11,15 @@ import sqlbuild._native as _native from sqlbuild._native import ( BuildPhaseTimingTracker, + CompiledProject, CompilePipelineOptions, + CompilePipelineResult, PartialBuildPhaseTimings, + PlanOutput, RulesRunResult, ) -from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py index af5e7ec7e8..ef0abd1472 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_plan_work.py @@ -3,12 +3,12 @@ import pytest from sqlbuild._native import ( + PlanOutput, PythonIdentityStatus, PythonPlanEntry, PythonRunPhase, plan_has_executable_work, ) -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import PythonNodeKind from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PlanWorkTestCase from tests.unit.src.sqlbuild.compiler.pipeline.main.helpers import build_plan_output_with_model diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py index 5779d71d6c..5f4ae4e612 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py @@ -4,14 +4,17 @@ import pytest -from sqlbuild._native import build_python_relation_targets +from sqlbuild._native import ( + DiscoveredPythonNode, + PlanOutput, + PythonNodeGraph, + SourceEntry, + build_python_relation_targets, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.refs import model, source -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import ( PythonRelationTargetScopeTestCase, PythonRelationTargetsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 65b24a5dce..be95ea7350 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -3,43 +3,37 @@ from pathlib import Path from sqlbuild._native import ( + BackfillAction, ChangeKind, + CompiledObjectKey, + CompileSqlReference, + CursorBounds, + Fingerprint, + InferredColumn, + MaterializationType, MissingUpstream, + OnSchemaChange, ParsedSelector, PathSelector, + PlanAction, + PlanReason, PlanWarning, ScenarioArtifactIdentity, ScenarioGraphPlan, + ScenarioRelationMap, SchemaAction, + SchemaFinding, + SourceEntry, + SourceLoadPlanEntry, + SqlLexicalSyntax, + SqlReferenceKind, WarningSeverity, ) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompileSqlReference, - InferredColumn, -) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledProjectView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ( - CursorBounds, - ScenarioRelationMap, - SchemaFinding, - SourceLoadPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - OnSchemaChange, - PlanAction, - PlanReason, -) -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase as PlanTestChainTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 4c8bf2aeff..b8e04c4b3a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -10,21 +10,22 @@ from sqlbuild._native import ( ChangeKind, CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, + Fingerprint, FunctionArgument, PlannerScope, + SchemaColumn, + SchemaModelEntry, WarehouseFingerprints, WarehouseSnapshot, build_version_identity_metadata_json, compute_query_hash, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, @@ -32,8 +33,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectModelChangesTestCase, DetectModelMetadataTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index f84a55b035..1e6be88af8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -1,21 +1,17 @@ from dataclasses import dataclass, field from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeKind, - ReferenceRename, -) -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import ( - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, + InferredColumn, PlanAction, PlanReason, + ReferenceRename, + SchemaDynamicColumnFamily, + SchemaFinding, ) -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily +from sqlbuild.adapter.contract.models import ColumnInfo @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 42e61cb32f..966b99d644 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -6,12 +6,18 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, + Fingerprint, + InferredColumn, + PlanAction, PlannerChangeResults, PlannerScope, + PlanReason, PlanWarning, + SqlHookEntry, WarehouseFingerprints, WarehouseSnapshot, WarningSeverity, @@ -23,15 +29,7 @@ get_materialization_type, resolve_model_plan_action, ) -from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( build_function_caller_metadata_case, build_metadata_json_with_audit_gate, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py index 955b73c549..89d0a6e12a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import ( + Fingerprint, ReferenceRename, compute_query_hash, origin_reference_names, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( OriginReferenceNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index 4b9ad9fa4c..b24c1e522a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -3,14 +3,14 @@ import pytest from sqlbuild._native import ( + InferredColumn, SchemaChangeKind, SchemaColumnSource, + SchemaDynamicColumnFamily, + SchemaFinding, detect_schema_changes, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import SchemaFinding -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectSchemaChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py index c4efc9f391..0a6994b40e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py @@ -3,13 +3,11 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, build_downstream_deps, build_execution_upstream_deps, project_lineage_views, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_test_project, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 9a281f15c5..7e8ae4ca2b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -13,24 +13,52 @@ import sqlbuild._native as _native from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, + CompiledAudit, CompiledDirectLogicSqlTestPayload, CompiledFunction, + CompiledModel, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSeed, CompiledSource, + CompiledSqlScenario, + CompiledSqlTest, CompileModelConfig, + CompilePipelineResult, + CompileSqlReference, CompileSqlScenarioCte, CompileSqlTestCte, + DefaultsConfig, DiscoveredAuditBlock, DiscoveredAuditFile, + DiscoveredLoaderFunction, DiscoveredSchemaFile, DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + Fingerprint, FunctionArgument, PlannerChangeResults, + PlanOutput, + PlanReason, ScenarioArtifactIdentity, + ScenarioRelationMap, + SchemaColumn, + SchemaModelEntry, + SchemaSeedEntry, + SeedCsvSettings, + SourceColumnEntry, + SourceEntry, + SourceWriteStrategy, + SqlLexicalSyntax, + SqlReferenceKind, WarehouseSnapshot, build_downstream_deps, build_scenario_relation_map, @@ -43,16 +71,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, - CompiledSqlTest, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledAuditView, @@ -64,36 +82,6 @@ CompiledSourceView, SqlTestMode, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ( - PlanOutput, - ScenarioRelationMap, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanReason, -) -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ( - DefaultsConfig, - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SeedCsvSettings, - SourceColumnEntry, - SourceEntry, -) -from sqlbuild.spec.contracts.types import ( - SourceWriteStrategy, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, IncrementalStrategyErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py index fbb009e40f..7991d7f842 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py @@ -3,10 +3,8 @@ from dataclasses import dataclass from sqlbuild._native import ( - ChangeKind, -) -from sqlbuild.compiler.planner.types import ( BackfillAction, + ChangeKind, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py index 3ac9251cf6..a7df66545e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py @@ -3,15 +3,13 @@ from __future__ import annotations from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, PlannerResolvedActions, ResolvedModelAction, ) -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) MERGE_MODEL_NAME: str = "orders" MERGE_RESOLVED_HASH: str = "resolved-hash" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py index 8eba85f9f6..dd6e604a02 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_function_local_hashes, model_definition_hash -from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild._native import PlannerInputError, build_function_local_hashes, model_definition_hash from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( DefinitionHashFailureTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index 0918c63812..b4e7f84f74 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, @@ -11,9 +12,6 @@ ResolvedModelAction, merge_recomputed_model_changes, ) -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( MergeRecomputedModelChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 5a1712b79f..10daace3fc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -7,15 +7,13 @@ from typing import TYPE_CHECKING from sqlbuild._native import ( + CompiledRelationLocation, OldNameViewAction, + OldNameViewPlanEntry, QueryShape, build_migration_fingerprint, parse_query_shape, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - OldNameViewPlanEntry, -) if TYPE_CHECKING: from sqlbuild._native import InputColumns diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index 0dea21e99b..cd0c4d1278 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -9,8 +9,14 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, DeferralInputs, + Fingerprint, + PlannerInputError, PlannerOverrides, PlannerRuntime, PlannerScope, @@ -19,15 +25,7 @@ plan_column_migrations, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index 5b420844ba..dbb864f001 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -4,10 +4,12 @@ from sqlbuild._native import ( ChangeDetectionResult, + RetentionDirection, + RetentionPlanPhase, + TableType, + TableTypeDowngradePolicy, ) from sqlbuild.adapter.contract.models import RetentionState -from sqlbuild.compiler.planner.types import RetentionDirection, RetentionPlanPhase -from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 16d6999afb..aa4ed448c9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -5,37 +5,31 @@ from unittest.mock import Mock from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, + LocalConfig, PlannerRelationsContext, PlannerRuntime, PlannerScope, PlannerWarehouseState, + ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, + TableTypeDowngradePolicy, + TargetConfig, + TimeTravelRetentionSource, WarehouseSnapshot, ) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, - TargetConfig, -) -from sqlbuild.spec.contracts.types import ( - TableTypeDowngradePolicy, - TimeTravelRetentionSource, -) def build_retention_planner_inputs( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py index 56d3145fc6..d2dcff5f74 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py @@ -5,16 +5,14 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, ChangeDetectionResult, ChangeKind, + PlannerInputError, check_full_rebuild_allowed, describe_full_rebuild_cause, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import ( - BackfillAction, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( FullRebuildCheckTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 7871c50b6d..02f7d206a5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -7,9 +7,23 @@ import pytest from sqlbuild._native import ( + Fingerprint, + LocalConfig, + MaterializationType, + PlannerInputError, PlannerRuntime, PlannerScope, PlannerWarehouseState, + ProjectConfig, + ResolvedTableType, + RetentionDirection, + RetentionPlanEntry, + RetentionPlanPhase, + TableType, + TableTypeDowngradePolicy, + TableTypePlanEntry, + TableTypeSource, + TargetConfig, WarehouseFingerprints, plan_retention, plan_table_types, @@ -25,21 +39,6 @@ RetentionScope, ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import RetentionPlanEntry, TableTypePlanEntry -from sqlbuild.compiler.planner.types import ( - MaterializationType, - RetentionDirection, - RetentionPlanPhase, -) -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - ResolvedTableType, - TargetConfig, -) -from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy, TableTypeSource from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( ForeignSnapshotRelationTestCase, RetentionPlanningErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py index e0cd13ede3..5793219d9f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/_test_types.py @@ -5,9 +5,8 @@ from dataclasses import dataclass, field from datetime import date, datetime +from sqlbuild._native import CursorBounds, CursorOverrides, ModelCursorSnapshot, SourceEntry from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.models import CursorBounds, CursorOverrides, ModelCursorSnapshot -from sqlbuild.spec.contracts.models import SourceEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 82f2acdab3..88f507dd64 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -5,15 +5,13 @@ from pathlib import Path from sqlbuild._native import ( - CompileModelConfig, - ModelPlanContext, -) -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompileModelConfig, + ModelPlanContext, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 1ad561525f..bac7d15b57 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -8,23 +8,22 @@ import pytest from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + AlignedInterval, + CursorBounds, + CursorGrain, + CursorType, CursorWatermarkMode, + DateValue, + ModelCursorSnapshot, + TimestampValue, apply_typed_cursor_replay_policy, compute_cursor_bounds, ) from sqlbuild._native import cursor_observed_partition as observed_partition from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.constants import ( - MICROBATCH_END_SENTINEL, - MICROBATCH_START_SENTINEL, -) -from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, -) -from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorBoundsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 30324d6023..5ab6298183 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -3,18 +3,16 @@ import pytest from sqlbuild._native import ( + BackfillAction, BackfillResult, CursorOverridePair, + ModelCursorSnapshot, + PlannerInputError, WarehouseSnapshot, resolve_model_sql, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ModelCursorSnapshot, -) -from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( CursorIntrinsicResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py index 19522990df..3c5a26a036 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_override_types.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild._native import type_cursor_overrides -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild._native import CursorOverrides, PlannerInputError, type_cursor_overrides from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( UntypedCursorOverrideErrorTestCase, UntypedCursorOverrideTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index 142a004783..c00449ebb0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -8,12 +8,13 @@ from sqlbuild._native import ( ModelPlanContext, + PythonHookEntry, + SourceEntry, + SqlHookEntry, resolve_model_hook_entries, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( HookSqlResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 9a83879c95..28b25f0c39 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -5,6 +5,9 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + CursorBounds, DbtCompileReferenceResolver, DbtManifestIndex, apply_deferred_locations, @@ -16,12 +19,7 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import CursorBounds from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, RefResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py index 62a2c40c5c..5a8031c023 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_sources.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild._native import resolve_source_references +from sqlbuild._native import CursorBounds, SourceColumnEntry, SourceEntry, resolve_source_references from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -12,8 +12,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( AdapterSourceResolutionTestCase, ReservedSourceColumnTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py index 1858c9a743..e7f3904331 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py @@ -2,7 +2,7 @@ from types import SimpleNamespace from typing import cast -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild._native import DiscoveredProjectInputs def build_selector_discovered_inputs( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index bff9a49c97..5fc6d33ba1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, + PlannerInputError, SqlTestSelection, UnitTestSelectorSplit, split_project_unit_test_selectors, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 7d13ab9bc9..cfe2db5d8b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -9,12 +9,22 @@ from sqlbuild._native import ( CompiledFunction, + CompiledModel, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + LoadedMacro, NativeSqlTestPlan, PlanWarning, + SqlLexicalSyntax, + SqlReferenceKind, + SqlTestPlanEntry, SqlTestPlanResult, expand_sql_macros, extract_sql_references, @@ -27,28 +37,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlTest, - LoadedMacro, - MacroContext, -) +from sqlbuild.compiler.compile.models import MacroContext from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.planner.models import ( - SqlTestPlanEntry, -) -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 8299d95df0..b79cb1a6ec 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,16 +6,26 @@ import pytest from sqlbuild._native import ( + ChainStep, CompiledDirectLogicSqlTestPayload, CompiledFunction, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, NativeSqlTestPlan, + PlannerInputError, PlannerScope, PlannerSelection, + PlanOutput, PlanWarning, + SqlTestPlanEntry, SqlTestPlanResult, + SqlTestReferenceError, WarningSeverity, build_test_command_plan, plan_compiled_sql_test_artifacts, @@ -28,17 +38,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlTest, -) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index a371b8a5d6..5af837bb96 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -2,17 +2,17 @@ import pytest -from sqlbuild._native import plan_audit -from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, CompileSqlReference, + SourceEntry, + SqlReferenceKind, + plan_audit, ) +from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanAttachedAuditTestCase, PlanAuditTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index 4264d405d9..5378bf5e98 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -5,6 +5,10 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompileSqlReference, + MaterializationType, + SqlReferenceKind, reads_outside_target_lineage, resolve_attachment_kind, resolve_effective_run_scope, @@ -13,16 +17,10 @@ AuditAttachmentKind, AuditRunScope, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledAuditView, ) -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ResolveAttachmentErrorTestCase, ResolveAttachmentTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py index 7ad94b1446..f17b1c62db 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_cursor_overrides +from sqlbuild._native import CursorOverrides, resolve_cursor_overrides from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorOverrideResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py index 96b5261e91..360696a9c8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_overrides_validation.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import CursorOverrides +from sqlbuild._native import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorOverridesValidationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py index 351f03d67d..f2b8d5ca95 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_start.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import compute_cursor_bounds +from sqlbuild._native import CursorBounds, ModelCursorSnapshot, compute_cursor_bounds from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.planner.models import CursorBounds, ModelCursorSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorStartBoundsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index a025b8a56d..5da157a363 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -8,29 +8,27 @@ import sqlbuild._native as sql_test_assembly from sqlbuild import _native as fixture_completion from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlTest, + CompileSqlReference, FixtureColumnMetadata, FixtureRelationMetadata, PlanWarning, RelationFixtureCompletion, RelationFixturePlanningContext, + SettingsConfig, + SqlReferenceKind, + SqlTestPlanEntry, SqlTestSelection, build_selected_test_entries, compiled_model_record, project_with, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, - CompileSqlReference, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import SqlTestPlanEntry -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SettingsConfig from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteFixturePlanningTestCase, FixturePlanningWorkTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index e46a99578a..d2d620d768 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import PlanReason from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( DetectFunctionChangeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py index 85a57d11bb..af86317720 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, topologically_order_keys, transitive_closure, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildDownstreamDepsTestCase, BuildUpstreamDepsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 0fd1466f3b..de3d39f92b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -5,30 +5,24 @@ import pytest from sqlbuild._native import ( - PlanEntryBuildInputs, - PlanWarning, - apply_configured_microbatch_limits, - apply_microbatch_limit, -) -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( + CompileInputError, CursorBounds, CursorInputRelation, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( IncrementalMode, MaterializationType, + MicrobatchLimitAction, + ModelPlanEntry, PlanAction, + PlanEntryBuildInputs, PlanReason, + PlanWarning, + apply_configured_microbatch_limits, + apply_microbatch_limit, ) -from sqlbuild.spec.contracts.types import MicrobatchLimitAction +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, EffectiveMicrobatchLimitPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py index 1a38e56bda..23d4d51949 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_microbatch_lookback +from sqlbuild._native import IncrementalStrategy, resolve_microbatch_lookback from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.types import IncrementalStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( MicrobatchLookbackTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index 923a00170c..f78652a500 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import build_downstream_model_name_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, build_downstream_model_name_closure from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index 3d3f1430d8..878a03936b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -8,26 +8,23 @@ import pytest from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + BackfillAction, BackfillResult, - CompileModelConfig, - WarehouseSnapshot, - compute_microbatch_range, - compute_plan_cursor_bounds, -) -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL -from sqlbuild.compiler.planner.models import ( + CompileModelConfig, CursorBounds, CursorInputRelation, ModelCursorSnapshot, + WarehouseSnapshot, + compute_microbatch_range, + compute_plan_cursor_bounds, ) -from sqlbuild.compiler.planner.types import BackfillAction +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( AuthoritativeCursorOverrideTestCase, MicrobatchCursorEndPlanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index f874eeed7d..ab899d223f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -4,8 +4,10 @@ from sqlbuild._native import ( ParsedScenarioArtifactName, + PlannerInputError, ScenarioArtifactIdentity, ScenarioArtifactName, + ScenarioRelationMap, build_scenario_artifact_name, build_scenario_hash_index, build_scenario_relation_map, @@ -13,10 +15,6 @@ is_scenario_artifact_physical_name, parse_scenario_artifact_physical_name, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ScenarioRelationMap, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioArtifactNameRecognitionTestCase, ScenarioArtifactNameTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index 8b9ef461f2..d96a4723c5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -6,15 +6,13 @@ import pytest from sqlbuild._native import ( + CompiledSqlScenario, + CompilePipelineResult, ScenarioArtifactName, - build_cli_scenario_plan, -) -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, + SqlLexicalSyntax, + build_cli_scenario_plan, ) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCliPlanIdentifierLimitTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index aa955bd78a..060193c5c8 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -5,14 +5,12 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledSqlScenario, ScenarioGraphPlan, WarningSeverity, plan_scenario_graph, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlScenario, -) from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanScenarioGraphErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index e15464501e..8f824c76bf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,8 +7,15 @@ import pytest from sqlbuild._native import ( + CompiledSqlScenario, CompileSqlScenarioCte, + DiscoveredHookFunction, + PlannerInputError, + ScenarioExecutionPlan, + ScenarioFixturePlan, ScenarioGraphPlan, + ScenarioRelationPlan, + SqlLexicalSyntax, build_scenario_execution_plan, build_scenario_fixture_plans, build_scenario_relation_plan, @@ -21,16 +28,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ScenarioExecutionPlan, - ScenarioFixturePlan, - ScenarioRelationPlan, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCheckSqlResolutionErrorTestCase, ScenarioCheckSqlResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py index 3f34e9ed29..a8163a1205 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import build_seed_identity -from sqlbuild.spec.contracts.models import SeedCsvSettings +from sqlbuild._native import SeedCsvSettings, build_seed_identity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( SeedIdentityCsvConfigTestCase, SeedIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index ed26cc5d80..b727f99bde 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -6,10 +6,14 @@ import pytest from sqlbuild._native import ( + NODE_TYPE_SEED, ChangeDetectionResult, ChangeKind, + CompiledObjectKey, + CompiledProject, DirectModelVersionIdentities, DirectSourceFreshnessPlanningResult, + Fingerprint, PlannerChangeResults, PlannerPolicies, PlannerScope, @@ -23,14 +27,11 @@ compiled_model_record, resolve_planner_scopes, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, PlannerStaleWarningScopeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index a190291669..aee3733314 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -3,6 +3,7 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, ParsedSelector, PathSelector, SelectorKind, @@ -11,9 +12,6 @@ project_lineage_views, resolve_project_selectors, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, -) from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildPathIndexTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index d3f480cabb..3c1c5c141e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -7,8 +7,14 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledProject, + CursorInputRelation, + PlannerInputError, PlannerRelationsContext, PlannerScope, + SourceEntry, + SqlReferenceKind, build_cursor_input_relations, build_planner_relations_context, gather_source_columns, @@ -18,20 +24,12 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, -) from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import CursorInputRelation -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorRuntimeOwnershipTestCase, KnownSourceColumnsReuseTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py index 20cd18c124..47e077aed9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_freshness.py @@ -5,15 +5,17 @@ import pytest from sqlbuild._native import ( + CompiledProject, DirectSourceFreshnessPlanningResult, PlannerRelationsContext, PlannerScope, + SourceEntry, + SourceFreshnessConfig, + SourceFreshnessStrategy, + SourceFreshnessValueKind, build_planner_source_freshness_result, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlannerSourceFreshnessReadMapTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 54bbd30a4f..fa112e1b48 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -8,7 +8,15 @@ import pytest from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, + DiscoveredLoaderFunction, + SourceEntry, + SourceLoadPlanEntry, + SourceWriteStrategy, build_intermediate_source_map, build_source_load_entries, build_source_load_map, @@ -17,22 +25,12 @@ project_with, upstream_loader_dependency_names, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import SourceLoadPlanEntry from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( LoaderDagExpansionTestCase, LoaderDependencyNamesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index 63b5df9306..f22f3e2ec2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -3,21 +3,17 @@ import pytest from sqlbuild._native import ( + BackfillAction, ChangeDetectionResult, ChangeKind, + PlanAction, + PlanReason, SchemaChangeKind, SchemaColumnSource, + SchemaFinding, resolve_model_plan_action, ) from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.models import ( - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - PlanAction, - PlanReason, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( IncrementalStrategyErrorTestCase, ResolveModelPlanActionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py index 296125ebf7..c1c7835558 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py @@ -3,17 +3,13 @@ import pytest from sqlbuild._native import ( + OnSchemaChange, SchemaAction, SchemaActionKind, SchemaChangeKind, SchemaColumnSource, - resolve_schema_actions, -) -from sqlbuild.compiler.planner.models import ( SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - OnSchemaChange, + resolve_schema_actions, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ResolveSchemaActionsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index 16f393b55d..6a9e995c3b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -3,24 +3,20 @@ import pytest from sqlbuild._native import ( + BackfillAction, ChangeDetectionResult, ChangeKind, + MaterializationType, + OnSchemaChange, PlanWarning, SchemaAction, SchemaActionKind, SchemaChangeKind, SchemaColumnSource, + SchemaFinding, WarningSeverity, build_model_warnings, ) -from sqlbuild.compiler.planner.models import ( - SchemaFinding, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - OnSchemaChange, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py index a9a2f1a7f9..e21ef80d4f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild._native import SourceEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 7cd8253437..9172b003b0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -7,17 +7,18 @@ from typing import cast from sqlbuild._native import ( + CompiledObjectKey, + CompiledProject, CompiledSource, + DiscoveredSourceFile, PlannerRelationsContext, PlannerScope, + SourceEntry, check_selected_source_tables_exist, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile -from sqlbuild.spec.contracts.models import SourceEntry def column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 973e792d11..f82650e67c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -11,6 +11,7 @@ import pytest from sqlbuild._native import ( + SourceEntry, build_compiled_project, discover_project_inputs, get_semantic_source_columns, @@ -20,7 +21,6 @@ from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( FakeRelation, build_offline_snowflake, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py index 574fbf0e21..f2918437ce 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_source_tables.py @@ -2,9 +2,8 @@ import pytest +from sqlbuild._native import PlannerInputError, SourceEntry from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.adapter.contract.main.helpers import ( ADAPTERS_BY_NAME, BinderException, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py index 5563adc22d..1e9c357578 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy +from sqlbuild._native import MissingMigrationOriginPolicy @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index fbe40f6ebf..e58cae2ad0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -7,15 +7,15 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + CompiledObjectKey, PlannedSqlTests, + PlannerInputError, PlanWarning, SqlTestSelection, WarningSeverity, start_sql_test_planning, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index a1686d952e..fc73db0258 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -39,7 +39,7 @@ def slow_plan_selected_sql_tests(**arguments): _FAILED_BUILD: str = dedent( """ from sqlbuild.cli.entry.main.entry import main - from sqlbuild.compiler.planner.exceptions import PlannerInputError + from sqlbuild._native import PlannerInputError def failing_reconciliation(**arguments): diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index 9aebae157b..502519ebf2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -2,13 +2,11 @@ from datetime import datetime from sqlbuild._native import ( + CursorBounds, + FutureCursorsConfig, SelectionStalenessGraph, SelectionStalenessWarning, ) -from sqlbuild.compiler.planner.models import ( - CursorBounds, -) -from sqlbuild.spec.contracts.models import FutureCursorsConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py index af8416541e..926910e148 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import PlanAction, PlanReason +from sqlbuild._native import PlanAction, PlanReason @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 43420693fe..0b5ca170fb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -6,16 +6,20 @@ import pytest -from sqlbuild._native import CompileModelConfig, build_display_only_sqlbuild_plan, format_plan -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, + build_display_only_sqlbuild_plan, + format_plan, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import PlanAction, PlanReason from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( DisplayFullRefreshTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 7bd5efe351..1af69db657 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -6,11 +6,19 @@ from typing import Any from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompileModelConfig, DeferralInputs, + Fingerprint, PlannerOverrides, PlannerPolicies, PlannerSelection, + PlanOutput, + SchemaColumn, + SchemaModelEntry, SelectionStalenessGraph, SelectionStalenessNodeKey, build_execution_plan, @@ -19,20 +27,11 @@ write_fingerprint, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry def model_definition_hash(project: CompiledProjectView, name: str) -> str: diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 0e14fac65d..41b5526012 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -5,28 +5,27 @@ from sqlbuild._native import ( CompiledFunction, + CompiledModel, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, DirectModelVersionIdentities, FunctionArgument, FunctionReturnColumn, PlannerScope, + SchemaColumn, + SeedCsvSettings, build_direct_model_version_identities, build_downstream_deps, build_model_version_identity_metadata_json, build_version_identity_metadata_json, compiled_model_record, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledResourceType, FunctionLanguage, ) -from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index 065326faee..3cc6ca8e30 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -6,11 +6,11 @@ from sqlbuild._native import ( CompiledFunction, + CompiledObjectKey, DirectModelVersionIdentities, FunctionReturnColumn, build_function_local_hashes, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py index b84f7ff474..0b107bd78a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_hooks.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild._native import PythonHookEntry, SqlHookEntry from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( HookIdentityPayloadTestCase, HookIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py index e4e1d8f073..8c1a0a302c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/_test_types.py @@ -1,7 +1,11 @@ from dataclasses import dataclass -from sqlbuild._native import MigrationCompatibility, MigrationDiscovery -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision +from sqlbuild._native import ( + ColumnMigrationDecision, + MigrationCompatibility, + MigrationDecision, + MigrationDiscovery, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py index dcf6d63594..d3322ed84d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/helpers.py @@ -3,23 +3,19 @@ from __future__ import annotations from sqlbuild._native import ( + ColumnMigrationDecision, + ColumnMigrationPlanEntry, + CompiledRelationLocation, DisplayOptions, MigrationCompatibility, - MigrationDiscovery, - format_pre_build_work, -) -from sqlbuild.adapter.contract.types import MigrationTransfer -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.types import ( - ColumnMigrationDecision, MigrationDecision, + MigrationDiscovery, MigrationPromotion, -) -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, ModelMigrationPlanEntry, PlanOutput, + format_pre_build_work, ) +from sqlbuild.adapter.contract.types import MigrationTransfer def migration_entry( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index 45345c47cb..f3c1786018 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -5,14 +5,15 @@ import pytest from sqlbuild._native import ( + ColumnMigrationDecision, DisplayOptions, MigrationCompatibility, + MigrationDecision, MigrationDiscovery, + PlanOutput, format_plan, format_pre_build_work, ) -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision -from sqlbuild.compiler.planner.models import PlanOutput from tests.unit.src.sqlbuild.compiler.planner.main.pre_build._test_types import ( ColumnMigrationPlainTextTestCase, ColumnMigrationStyleTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py index e2c255b594..e341ef01d5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_advance_cursor_end.py @@ -2,11 +2,11 @@ import pytest +from sqlbuild._native import CursorGrain, CursorType from sqlbuild._native import cursor_exclusive_to_inclusive as exclusive_to_inclusive from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.types import CursorScalar from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( AdvanceCursorEndTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py index a21c19ef1b..93eccb9dee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_cursor_bound_display.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import cursor_bound_display -from sqlbuild.compiler.planner.types import CursorGrain, CursorType +from sqlbuild._native import CursorGrain, CursorType, cursor_bound_display from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( CursorBoundDisplayTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py index f2bb357ec8..e5b58e756d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_effective_microbatch_batch_size.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import resolve_effective_microbatch_batch_size -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild._native import CursorGrain, resolve_effective_microbatch_batch_size from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( EffectiveMicrobatchBatchSizeTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 890873ae51..fadd90f751 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -6,21 +6,17 @@ import pytest from sqlbuild._native import ( - project_with, -) -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( + CompiledProject, + DiscoveredHookFunction, ModelPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( PlanAction, + PlannerInputError, + PlanOutput, PlanReason, + project_with, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( DirectSourceFreshnessPlanOutputTestCase, ExternalBlockedPlanOutputTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py index 62d897ec76..69a5bbe4fb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_future_cursor_safety.py @@ -4,14 +4,17 @@ import pytest +from sqlbuild._native import ( + CursorBounds, + CursorGrain, + CursorType, + FutureCursorAction, + FutureCursorSafetyError, + FutureCursorsConfig, + future_cursor_cap_warning, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import future_cursor_cap_warning from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety -from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.spec.contracts.models import FutureCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.compiler.planner.main._test_types import FutureCursorSafetyTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py index 279f4c809e..f8dfc16cb0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_inclusive_cursor_end.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import inclusive_cursor_end -from sqlbuild.compiler.planner.types import CursorGrain, CursorType +from sqlbuild._native import CursorGrain, CursorType, inclusive_cursor_end from tests.unit.src.sqlbuild.compiler.planner.main._test_types import InclusiveCursorEndTestCase diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py index 6c10df7d30..4585925d8f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_maximum_start_safety.py @@ -5,17 +5,18 @@ import pytest -from sqlbuild._native import compute_cursor_bounds -from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety -from sqlbuild.compiler.planner.exceptions import MaximumAutomaticStartError -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( CursorBounds, + FutureCursorAction, + FutureCursorsConfig, + MaximumAutomaticStartError, MaximumStartPolicyInputs, ModelCursorSnapshot, + StartCursorsConfig, + compute_cursor_bounds, ) -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction +from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token +from sqlbuild._native import resolve_future_cursor_safety as apply_future_cursor_safety from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( MaximumStartCapTestCase, MaximumStartErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 50a5f7cfac..cd9d9916e4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -12,9 +12,13 @@ from sqlbuild._native import ( CompileModelConfig, DiscoveredSeedFile, + Fingerprint, PlannerRelationsContext, PlannerScope, PlannerSelection, + PlanOutput, + PlanReason, + SeedPlanEntry, build_audit_command_plan, build_seed_command_plan, build_test_command_plan, @@ -25,9 +29,6 @@ resolve_static_relation_context, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import PlanOutput, SeedPlanEntry -from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( PlannerTestAdapter, build_test_project, diff --git a/tests/unit/src/sqlbuild/compiler/planner/test_models.py b/tests/unit/src/sqlbuild/compiler/planner/test_models.py index 847e0b6039..c763b0560c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/planner/test_models.py @@ -6,9 +6,11 @@ import pytest -from sqlbuild.compiler.migrations.types import MigrationDecision -from sqlbuild.compiler.planner.models import ModelMigrationPlanEntry -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy +from sqlbuild._native import ( + MigrationDecision, + MissingMigrationOriginPolicy, + ModelMigrationPlanEntry, +) from tests.unit.src.sqlbuild.compiler.planner._test_types import MissingOriginBlockTestCase from tests.unit.src.sqlbuild.compiler.planner.main.pre_build.helpers import migration_entry diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 883331fe33..fb2b8b4c07 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -11,46 +11,40 @@ from typing import Any, cast from sqlbuild._native import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, CompiledSource, CompileModelConfig, + DiscoveredAssetFunction, + DiscoveredCheckFunction, + DiscoveredLoaderFunction, + DiscoveredProjectInputs, + DiscoveredSourceFile, + DiscoveredTaskFunction, + LoaderConnectionMode, + LocalConfig, + ProjectConfig, + ProjectGraph, + PythonNodeGraph, PythonSqlRunSelection, + SourceColumnEntry, + SourceEntry, build_python_node_graph, compiled_graph_keys, resolve_python_sql_run_selection, ) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAssetFunction, - DiscoveredCheckFunction, - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredSourceFile, - DiscoveredTaskFunction, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.providers import Provider from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.refs import model, source -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SourceColumnEntry, - SourceEntry, -) from sqlbuild.tasks import task from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py index 62a7809ca1..0f68b1caee 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py @@ -8,8 +8,8 @@ import pytest +from sqlbuild._native import PythonNodeIdentity from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeIdentityChangeTestCase, PythonNodeIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index e25150d0a8..4d6878bf34 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -4,30 +4,31 @@ import json from collections import defaultdict +from dataclasses import asdict from pathlib import Path from typing import cast import pytest -from sqlbuild._native import build_python_node_graph -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredLoaderFunction, DiscoveredProjectInputs, DiscoveredProvider, - DiscoveredTaskFunction, -) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, + DiscoveredTaskFunction, + LoaderConnectionMode, + LocalConfig, + ProjectConfig, PythonNodeGraph, PythonNodeIdentity, + SourceColumnEntry, + build_python_node_graph, ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind from sqlbuild.python_nodes.models import RetryPolicy from sqlbuild.python_nodes.types import PythonCheckSeverity -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeGraphInventoryTestCase, PythonNodeIdentityConfigTestCase, @@ -213,7 +214,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node assert tuple(usage.annotation_module for usage in notify_node.provider_usages) == ( test_case.expected_provider_usage_annotation_modules ) - assert tuple(vars(usage) for usage in notify_node.provider_usages) == ( + assert tuple(asdict(usage) for usage in notify_node.provider_usages) == ( { "provider_name": "slack_provider", "parameter_name": "slack_provider", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 9af5d91594..4b7d565d33 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -4,10 +4,14 @@ import pytest -from sqlbuild._native import PythonSqlRunSelection, build_python_sql_run_lifecycle -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import ( + CompiledObjectKey, + PythonNodeGraph, + PythonSqlRunLifecyclePlan, + PythonSqlRunSelection, + build_python_sql_run_lifecycle, +) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlRunLifecycleTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index 452fbbdd0e..c36f6956be 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild._native import PythonSqlRunSelection -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import ProjectGraph, PythonNodeGraph, PythonSqlRunSelection from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, PythonSqlSelectorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 5bdff1d5fb..6d3c8e1827 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -6,11 +6,11 @@ from sqlbuild._native import ( ParsedSelector, + PythonNodeGraph, SelectorKind, parse_project_selector, resolve_python_node_selection, ) -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonNodeParseSelectorTestCase, PythonNodeSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index aef7c08cb9..8dd3cc597c 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -4,10 +4,13 @@ import pytest -from sqlbuild._native import PythonSqlRunSelection, resolve_project_selectors -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import ( + CompiledObjectKey, + ProjectGraph, + PythonNodeGraph, + PythonSqlRunSelection, + resolve_project_selectors, +) from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlSelectorErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/references/_test_types.py b/tests/unit/src/sqlbuild/compiler/references/_test_types.py index 30174e00c8..f5ce399907 100644 --- a/tests/unit/src/sqlbuild/compiler/references/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/references/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import Literal -from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild._native import SqlReferenceKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/references/test_types.py b/tests/unit/src/sqlbuild/compiler/references/test_types.py index 71421fc6f6..9980caed45 100644 --- a/tests/unit/src/sqlbuild/compiler/references/test_types.py +++ b/tests/unit/src/sqlbuild/compiler/references/test_types.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild._native import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.references._test_types import ( SqlReferenceKindExampleCallTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py index 96ae930421..d58860beb8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_helpers.py @@ -9,34 +9,30 @@ from pathlib import PurePath from sqlbuild._native import ( - build_scope_lookup, - normalize_scope_path, - query_scope_target, - resolve_scope_declaration_visibility, - resolve_scope_visibility, -) -from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, + DeclarationKind, DeclarationRecord, DeclarationVisibility, + GrantKind, GrantRecord, + InaccessibleReason, InaccessibleRecord, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeIndex, + ScopeKind, ScopeLookup, ScopeTargetQuery, + VisibilityReason, VisibilityRecord, VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - GrantKind, - InaccessibleReason, - ResourceKind, - ScopeKind, - VisibilityReason, + build_scope_lookup, + normalize_scope_path, + query_scope_target, + resolve_scope_declaration_visibility, + resolve_scope_visibility, ) _FOLDERS: tuple[str, ...] = ("sales", "sale", "finance", "daily", "orders") diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index 458ff145d2..5737c253f0 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -5,16 +5,17 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity -from sqlbuild.compiler.scopes.types import ( +from sqlbuild._native import ( + DeclarationIdentity, DeclarationKind, - DiagnosticSeverity, InaccessibleReason, + ResourceIdentity, ResourceKind, ScopeDiagnosticCode, ScopeKind, VisibilityReason, ) +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py index 36c6d58db8..f5fc0720a4 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/helpers.py @@ -8,37 +8,32 @@ from typing import cast from sqlbuild._native import ( - build_scope_lookup, - get_placement_validated_scope_index, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, -) -from sqlbuild.compiler.scopes.models import ( ConstantMetadata, DeclarationIdentity, + DeclarationKind, DeclarationRecord, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, + EnumDeclaration, EnumMemberMetadata, EnumMetadata, GrantRecord, + LocalConfig, MacroMetadata, OwnershipRoot, + OwnershipRootKind, + ProjectConfig, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeCompleteness, ScopeIndex, + ScopeKind, ScopeLookup, UsageRecord, + build_scope_lookup, + get_placement_validated_scope_index, ) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - OwnershipRootKind, - ResourceKind, - ScopeKind, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig SCOPE_CACHE_PROJECT: str = 'name = "demo"\nadapter = "duckdb"\n' SCOPE_CACHE_MODEL: str = "MODEL(description 'Test model.');\nSELECT 1 AS id\n" diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py index 9fbe192b5e..5817aefeb3 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_builder_and_visibility.py @@ -7,51 +7,45 @@ import pytest from sqlbuild._native import ( + ConstantDeclaration, + DeclarationIdentity, + DeclarationKind, + DeclarationRecord, DiscoveredAuditBlock, DiscoveredAuditFile, DiscoveredConstantFile, DiscoveredEnumFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveredSqlTestBlock, - EnumMember, - ScopeValidationError, - build_scope_index, - build_scope_lookup, - normalize_sql_value, - query_scope_target, - resolve_scope_visibility, - validate_scope_index, -) -from sqlbuild.compiler.compile.models import LoadedMacro -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, DiscoveredMacroFile, DiscoveredSourceFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, DiscoveredSqlTestFile, EnumDeclaration, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, + EnumMember, + InaccessibleReason, + LoadedMacro, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, + ScopeDiagnosticCode, ScopeIndex, + ScopeKind, ScopeLookup, ScopeTargetQuery, - VisibilityResolution, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - InaccessibleReason, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, + ScopeValidationError, + SourceEntry, VisibilityReason, + VisibilityResolution, + build_scope_index, + build_scope_lookup, + normalize_sql_value, + query_scope_target, + resolve_scope_visibility, + validate_scope_index, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, ResourceExpectation, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index 696d092b9c..915a558a8e 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -5,19 +5,17 @@ import pytest from sqlbuild._native import ( - InvalidQualifiedIdentityError, - build_scope_lookup, - explain_scope_declaration, - query_scope_report, -) -from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, + DeclarationKind, + InvalidQualifiedIdentityError, ScopeDiagnostic, ScopeLookup, ScopeReport, + build_scope_lookup, + explain_scope_declaration, + query_scope_report, ) -from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( IdentityReportCase, InvalidIdentityCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py index 700a8bf189..af0ae86914 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py @@ -8,20 +8,19 @@ import pytest -from sqlbuild._native import build_scope_lookup, scope_metadata_projection -from sqlbuild.compiler.scopes.models import ( +from sqlbuild._native import ( DeclarationIdentity, + DeclarationKind, DeclarationRecord, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeIndex, - ScopeLookup, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, ScopeKind, + ScopeLookup, + build_scope_lookup, + scope_metadata_projection, ) from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( DeclarationContainerProjectionCase, @@ -133,7 +132,7 @@ def test_given_frozen_scope_record_when_mutating_field_then_record_is_immutable( record: DeclarationRecord = scope_index().declarations[0] with pytest.raises(test_case.expected_error): - record.path = "changed.sql" # ty: ignore[invalid-assignment] + record.path = "changed.sql" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index dcf794f89a..3931dadda7 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -10,18 +10,17 @@ import pytest from sqlbuild._native import ( - compiler_cache_directory, - load_or_build_scope_index, - scope_index_fingerprint, -) -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ScopeIndex -from sqlbuild.compiler.scopes.types import ( + DeclarationIdentity, DeclarationKind, - DiagnosticSeverity, GrantKind, ResourceKind, ScopeDiagnosticCode, + ScopeIndex, + compiler_cache_directory, + load_or_build_scope_index, + scope_index_fingerprint, ) +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( CacheFaultCase, FingerprintMutationCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py index 3d64124671..fb19bc09fb 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_placement.py @@ -5,28 +5,24 @@ import pytest from sqlbuild._native import ( - ScopeValidationError, - get_placement_validated_scope_index, - validate_scope_index, -) -from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, + DeclarationKind, DeclarationRecord, OwnershipRoot, ResourceIdentity, + ResourceKind, ResourceRecord, ScopeCompleteness, ScopeDiagnostic, - ScopeIndex, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - DiagnosticSeverity, - ResourceKind, ScopeDiagnosticCode, + ScopeIndex, ScopeKind, + ScopeValidationError, + UsageRecord, + get_placement_validated_scope_index, + validate_scope_index, ) +from sqlbuild._native import ScopeDiagnosticSeverity as DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( DeclarationChainPlacementCase, DiamondLadderPlacementCase, diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py index 0a508e3933..7fc3ac05bd 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_relocation.py @@ -5,13 +5,11 @@ import pytest from sqlbuild._native import ( - relocate_declarations_for_move, -) -from sqlbuild.compiler.scopes.models import ( DeclarationRecord, ResourceIdentity, + ResourceKind, + relocate_declarations_for_move, ) -from sqlbuild.compiler.scopes.types import ResourceKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import RelocationCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import relocation_index diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py index 79ea620d89..8bad2dd96b 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_reports.py @@ -10,28 +10,24 @@ import pytest from sqlbuild._native import ( - browse_scope_folders, - build_scope_lookup, - explain_scope_declaration, - list_scope_declarations, - preview_scope_move, - query_scope_report, - serialize_scope_report, -) -from sqlbuild.compiler.scopes.models import ( + DeclarationKind, OwnershipRoot, + ResourceKind, ResourceRecord, ScopeBrowseResult, + ScopeDiagnosticCode, ScopeIndex, ScopeListResult, ScopeLookup, ScopeReport, ScopeReportFilters, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeDiagnosticCode, + browse_scope_folders, + build_scope_lookup, + explain_scope_declaration, + list_scope_declarations, + preview_scope_move, + query_scope_report, + serialize_scope_report, ) from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( ExpectedBooleanCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py index cadbc02b3b..bc760126dd 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/_helpers/test_age_policy.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import evaluate_source_freshness_age_policy -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy +from sqlbuild._native import SourceFreshnessAgePolicy, evaluate_source_freshness_age_policy from tests.unit.src.sqlbuild.compiler.source_freshness._helpers._test_types import ( SourceFreshnessAgePolicyDurationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py index 10db904107..d7b594c0b5 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py @@ -2,9 +2,8 @@ from datetime import UTC, datetime from typing import Any -from sqlbuild._native import SourceFreshnessUnknownReason +from sqlbuild._native import SourceFreshnessUnknownReason, SourceFreshnessValueKind from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py index bfde40ce0b..98a069d2cb 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind +from sqlbuild._native import SourceFreshnessValueKind @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index 60a11450b1..558e7417bb 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -6,15 +6,17 @@ import pytest from sqlbuild._native import ( + SourceEntry, + SourceFreshnessConfig, SourceFreshnessObservation, SourceFreshnessObservationError, + SourceFreshnessStrategy, + SourceFreshnessValueKind, normalize_source_freshness_data_version, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main.configured_observation._test_types import ( SourceFreshnessObservationErrorTestCase, SourceFreshnessObservationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index 37cd5b2a76..10ff692c35 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -6,8 +6,15 @@ from typing import Any from sqlbuild._native import ( + SOURCE_FRESHNESS_TABLE_NAME, + CompiledObjectKey, + SourceEntry, + SourceFreshnessConfig, SourceFreshnessIdentity, + SourceFreshnessRecord, SourceFreshnessRenderers, + SourceFreshnessStrategy, + SourceFreshnessValueKind, build_source_freshness_read_latest_sql, source_freshness_data_version_hash, write_source_freshness_records, @@ -20,12 +27,7 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind def state_table_exists_map( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py index c49624b344..08f851d558 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/shared/_helpers/test_sql.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + SOURCE_FRESHNESS_TABLE_NAME, build_source_freshness_create_table_sql, build_source_freshness_read_latest_sql, ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from tests.unit.src.sqlbuild.compiler.source_freshness.main.shared._helpers._test_types import ( BuildSourceFreshnessSqlTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index f9f2dd9bfc..5097f6fddc 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -8,6 +8,8 @@ from sqlbuild._native import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, + SourceFreshnessConfig, + SourceFreshnessStrategy, SourceFreshnessUnknownReason, build_direct_source_freshness_planning_result, observe_adapter_sources_freshness, @@ -19,8 +21,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( AdapterBatchIsolationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py index 9d80b24ac6..4adc4cfcc3 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py @@ -4,8 +4,11 @@ import pytest -from sqlbuild._native import SourceFreshnessAgeStatus, evaluate_source_freshness_age_policy -from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy +from sqlbuild._native import ( + SourceFreshnessAgePolicy, + SourceFreshnessAgeStatus, + evaluate_source_freshness_age_policy, +) from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessAgeEvaluationTestCase, SourceFreshnessNaiveDatetimeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py index 2373169f04..e9d8628b6f 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py @@ -4,9 +4,12 @@ import pytest -from sqlbuild._native import SourceFreshnessObservation -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind +from sqlbuild._native import ( + SourceFreshnessObservation, + SourceFreshnessRecord, + SourceFreshnessStrategy, + SourceFreshnessValueKind, +) from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessDatetimeNormalizationTestCase, SourceFreshnessNaiveDatetimeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py index 728bc319c4..49003234d5 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py @@ -6,7 +6,11 @@ import pytest from sqlbuild._native import ( + SourceEntry, + SourceFreshnessConfig, SourceFreshnessObservation, + SourceFreshnessStrategy, + SourceFreshnessValueKind, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata @@ -17,8 +21,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SharedSourceFreshnessColumnSqlTestCase, SharedSourceFreshnessExpressionSubqueryTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index d5b889758c..38faa71844 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -8,7 +8,13 @@ from sqlbuild._native import ( DirectSourceFreshnessPlanningResult, + SourceEntry, + SourceFreshnessAgePolicy, + SourceFreshnessConfig, + SourceFreshnessRecord, SourceFreshnessRenderers, + SourceFreshnessStrategy, + SourceFreshnessValueKind, build_direct_source_freshness_planning_result, source_freshness_data_version_hash, write_source_freshness_records, @@ -16,13 +22,6 @@ from sqlbuild.adapter.contract.models import TableFreshnessMetadata, TableFreshnessRequest from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.models import ( - SourceEntry, - SourceFreshnessAgePolicy, - SourceFreshnessConfig, -) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( DirectSourceFreshnessAdapterDefaultTestCase, DirectSourceFreshnessAgePolicyTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py index b619eb4fed..fbc2e35032 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py @@ -7,13 +7,13 @@ from sqlbuild._native import ( SourceFreshnessIdentity, SourceFreshnessInputError, + SourceFreshnessRecord, SourceFreshnessRenderers, SourceFreshnessSet, read_latest_source_freshness, write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( ReadLatestSourceFreshnessErrorTestCase, ReadLatestSourceFreshnessRendererTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py index 793d2035ba..50a1a49c3a 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_state.py @@ -5,12 +5,13 @@ import pytest from sqlbuild._native import ( + SourceFreshnessRecord, + SourceFreshnessStrategy, + SourceFreshnessValueKind, normalize_source_freshness_data_version, source_freshness_data_version_hash, source_freshness_records_equivalent, ) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SharedSourceFreshnessHashTestCase, SourceFreshnessStateCompatibilityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py index 64026852d6..5948b906a5 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import SqlLexicalSyntax @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py index 79b8981a2d..f00fa74306 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/helpers.py @@ -3,7 +3,7 @@ from __future__ import annotations import sqlbuild._native as _native -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import SqlLexicalSyntax def generic_scanner(*, sql: str, context: str = "SQL") -> _native.SqlCodeScanner: diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index 850e3434e6..426131f98c 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -3,8 +3,7 @@ import pytest import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import CompileInputError, SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( DialectMatchingParenTestCase, DialectNonCodeErrorTestCase, diff --git a/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py b/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py index d16e5d8aa8..1180e853b4 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/main/_test_types.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import datetime -from sqlbuild.compiler.planner.types import CursorGrain +from sqlbuild._native import CursorGrain @dataclass(frozen=True, kw_only=True) diff --git a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py index 9968b12aef..382015ae9e 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/main/test_cursor_algebra.py @@ -4,6 +4,7 @@ import pytest +from sqlbuild._native import AlignedInterval, CursorGrain, CursorType, IntegerValue, TimestampValue from sqlbuild._native import cursor_cap_from_end as cap_from_end from sqlbuild._native import cursor_cap_from_start as cap_from_start from sqlbuild._native import cursor_clamp as clamp @@ -24,8 +25,6 @@ from sqlbuild._native import cursor_sentinel_from_token as sentinel_from_token from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild._native import cursor_split_into_batches as split_into_batches -from sqlbuild.compiler.planner.types import CursorGrain, CursorType -from sqlbuild.cursor_algebra.models import AlignedInterval, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import BoundSentinel, CursorScalar from tests.unit.src.sqlbuild.cursor_algebra.main._test_types import ( CursorAlgebraMatrixCase, diff --git a/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py b/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py index 8056cfdafe..8b4e8473d9 100644 --- a/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py +++ b/tests/unit/src/sqlbuild/cursor_algebra/test_duration.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cursor_algebra.models import Duration +from sqlbuild._native import Duration from tests.unit.src.sqlbuild.cursor_algebra._test_types import ( DurationParseNoneTestCase, DurationParseTestCase, diff --git a/tests/unit/src/sqlbuild/diagnostics/_test_types.py b/tests/unit/src/sqlbuild/diagnostics/_test_types.py index f7ab2e6602..eb51dbd1fe 100644 --- a/tests/unit/src/sqlbuild/diagnostics/_test_types.py +++ b/tests/unit/src/sqlbuild/diagnostics/_test_types.py @@ -29,3 +29,11 @@ class ProcessResourceTestCase: expected_user_cpu_seconds: float expected_system_cpu_seconds: float expected_max_rss_bytes: int | None + + +@dataclass(frozen=True) +class ConsoleDurationTestCase: + description: str + subject: object + duration_ms: object + expected_line: str diff --git a/tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py b/tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py new file mode 100644 index 0000000000..4a08a8eaaf --- /dev/null +++ b/tests/unit/src/sqlbuild/diagnostics/test_console_formatter.py @@ -0,0 +1,78 @@ +"""Tests for the diagnostics console formatter's duration token.""" + +from __future__ import annotations + +import logging + +import pytest + +from sqlbuild.diagnostics.classes.diagnostics_console_formatter import ( + DiagnosticsConsoleFormatter, +) +from tests.unit.src.sqlbuild.diagnostics._test_types import ConsoleDurationTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + ConsoleDurationTestCase( + description="small duration rounds like Python", + subject="model", + duration_ms=1005, + expected_line="[info] model duration=1.00s done", + ), + ConsoleDurationTestCase( + description="duration above 2**63 formats like Python", + subject="model", + duration_ms=2**70, + expected_line="[info] model duration=1180591620717411328.00s done", + ), + ConsoleDurationTestCase( + description="negative duration below -2**63 formats like Python", + subject="model", + duration_ms=-(2**80), + expected_line="[info] model duration=-1208925819614629199872.00s done", + ), + ConsoleDurationTestCase( + description="bool duration is an int", + subject="model", + duration_ms=True, + expected_line="[info] model duration=0.00s done", + ), + ConsoleDurationTestCase( + description="non-integer duration is ignored", + subject="model", + duration_ms=1.5, + expected_line="[info] model done", + ), + ConsoleDurationTestCase( + description="non-string subject ignores an oversized duration", + subject=5, + duration_ms=2**70, + expected_line="[info] exec done", + ), + ConsoleDurationTestCase( + description="duration too large for a float is omitted instead of raising", + subject="model", + duration_ms=10**400, + expected_line="[info] model done", + ), + ], + ids=lambda case: case.description, +) +def test_given_duration_field_when_formatting_console_line_then_matches_python_rendering( + test_case: ConsoleDurationTestCase, +) -> None: + record: logging.LogRecord = logging.makeLogRecord( + { + "name": "sqlbuild.exec", + "levelname": "INFO", + "msg": "done", + "sqlbuild_subject": test_case.subject, + "sqlbuild_duration_ms": test_case.duration_ms, + } + ) + + line: str = DiagnosticsConsoleFormatter().format(record) + + assert line == test_case.expected_line, test_case.description diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py index 438cd53137..9820c3f510 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py @@ -6,7 +6,12 @@ from types import SimpleNamespace from typing import Any, cast -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + MeasurementThresholdBound, + MeasurementThresholds, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditEvaluationMode, @@ -15,9 +20,7 @@ AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py index fe2b5fb92e..ea7d76af3d 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_execute_reporting_errors.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import AuditPlanEntry from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity -from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from tests.unit.src.sqlbuild.executor.auditing.main._test_types import ( AuditInterruptCase, diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py index 304ed6fc8f..36fd549e5c 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py @@ -9,9 +9,8 @@ import pytest import sqlbuild.executor.auditing._helpers.result_projection as projection_module +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.models import AuditResultRecord from sqlbuild.executor.auditing.main._project_results import project_audit_result_batch diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py index 7507f93334..e30da1baae 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/_test_types.py @@ -1,15 +1,15 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.adapter.contract.types import RetentionChangePhase -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.types import ( +from sqlbuild._native import ( + DiscoveredLoaderFunction, MaterializationType, PlanAction, RetentionDirection, RetentionPlanPhase, ) +from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.runtime.contracts.types import ExecutionResourceKind diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index d453bfd9a6..c5144c16ce 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -6,33 +6,31 @@ from typing import Any from unittest.mock import Mock -from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin -from sqlbuild.compiler.auditing.types import ( - AuditAttachmentKind, - AuditOutcome, - AuditRunScope, - AuditSeverity, -) -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.planner.models import ( + DiscoveredLoaderFunction, + MaterializationType, ModelPlanEntry, + PlanAction, PlanOutput, + PlanReason, + SeedCsvSettings, SeedPlanEntry, + SourceColumnEntry, + SourceEntry, SourceLoadPlanEntry, + SourceWriteStrategy, ) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, +from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin +from sqlbuild.compiler.auditing.types import ( + AuditAttachmentKind, + AuditOutcome, + AuditRunScope, + AuditSeverity, ) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.spec.contracts.models import SeedCsvSettings, SourceColumnEntry, SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy def build_audit_result( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py index e8bf977cb7..385244587c 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_blocking.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.executor.build._helpers.blocking import downstream_blocked_keys from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( DownstreamBlockedKeysTestCase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index b194c12040..d50712544f 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -6,11 +6,15 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import ( + CompiledObjectKey, + DiscoveredLoaderFunction, + LoaderConnectionMode, + PlanOutput, + SourceEntry, + SourceLoadPlanEntry, +) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode -from sqlbuild.compiler.planner.models import PlanOutput, SourceLoadPlanEntry from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections, ) @@ -28,7 +32,6 @@ invocation_scope, ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( ExternalBuildSourceLoadTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py index aaba5c6571..fb682d78a3 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py @@ -2,14 +2,13 @@ import pytest +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.build._helpers.indexes import build_execution_indexes from sqlbuild.executor.build.models import BuildIndexes from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py index 5c153f528b..349623b741 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_retention.py @@ -5,6 +5,18 @@ import pytest +from sqlbuild._native import ( + CompiledRelationLocation, + IncrementalMode, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + RetentionDirection, + RetentionPlanEntry, + RetentionPlanPhase, + TableTypePlanEntry, +) from sqlbuild.adapter.contract.models import ( RelationInfo, RenderedRetentionChange, @@ -12,20 +24,6 @@ RetentionState, ) from sqlbuild.adapter.contract.types import RetentionChangePhase, RetentionScope -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, - RetentionPlanEntry, - TableTypePlanEntry, -) -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - MaterializationType, - PlanAction, - RetentionDirection, - RetentionPlanPhase, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.build._helpers.retention import ( apply_retention_phase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index bc41d529fa..45fa8be653 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -9,18 +9,23 @@ import pytest -from sqlbuild._native import NativeProgressProjector -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( + CompiledObjectKey, DiscoveredHookFunction, DiscoveredLoaderFunction, + ModelPlanEntry, + NativeProgressProjector, + PlanAction, + PlanOutput, + PlanReason, PythonHookEntry, + SourceEntry, + SourceLoadPlanEntry, + SourceWriteStrategy, ) -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput, SourceLoadPlanEntry -from sqlbuild.compiler.planner.types import PlanAction, PlanReason +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, @@ -38,8 +43,6 @@ invocation_scope, ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( BuildSchedulerModelHookTestCase, BuildSchedulerPlannedSkipTestCase, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 192e3161cf..669a32dc4c 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -6,10 +6,9 @@ import pytest +from sqlbuild._native import CompiledObjectKey, PlanOutput from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py index bc2f4770a1..dae1f86f97 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import BuildCallbacks, SchedulerState diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index 0b9f8fe5b4..c2c9626b84 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -9,7 +9,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import NodeCompletion diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py index 909aa90453..fdd2c9506c 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild._native import CompiledObjectKey, PlanOutput, SqlTestPlanEntry from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import PlanOutput, SqlTestPlanEntry from sqlbuild.executor.build.classes.build_scheduler import _test_result_authored_order from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py index 7312f01590..963278a303 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import RetentionPlanPhase +from sqlbuild._native import RetentionPlanPhase from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 46b1b95b02..18a2a6aa22 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -5,14 +5,21 @@ import pytest +from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + CompiledObjectKey, + CompiledRelationLocation, + Fingerprint, + FingerprintSet, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + SeedCsvSettings, + SeedPlanEntry, +) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.spec.contracts.models import SeedCsvSettings class CloneFingerprintAdapter: diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py index c4220e6bdd..52e78e64a3 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/test_fingerprinting.py @@ -5,11 +5,16 @@ import pytest +from sqlbuild._native import ( + NODE_TYPE_MODEL, + NODE_TYPE_SEED, + Fingerprint, + FingerprintSet, + MaterializationType, + ModelPlanEntry, + SeedPlanEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_MODEL, NODE_TYPE_SEED -from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone._helpers.fingerprinting import copy_clone_fingerprints from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index 985dd7a269..cf53cd8e3b 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -3,24 +3,27 @@ from pathlib import Path from typing import Any, ClassVar -from sqlbuild._native import CompileModelConfig -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, + FunctionPlanEntry, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + ResolvedTimeTravelRetention, + TimeTravelRetentionSource, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.models import FunctionPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention -from sqlbuild.spec.contracts.types import TimeTravelRetentionSource class FakeCloneAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py index f15a4301e8..775fa6e004 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_operations.py @@ -2,10 +2,10 @@ import pytest +from sqlbuild._native import ModelPlanEntry from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.executor.clone._helpers.operations import clone_relation from sqlbuild.executor.clone.models import CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py index b6d9f37732..cfbdde3ade 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/test_retention.py @@ -4,13 +4,13 @@ import pytest +from sqlbuild._native import RetentionPlanPhase from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, RetentionRequest, RetentionState, ) from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionChangePhase, RetentionScope -from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.clone._helpers.retention import ( apply_clone_namespace_retention_phase, diff --git a/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py b/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py index fb15f5b492..6ed72ccf14 100644 --- a/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/clone/main/test_execute.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild.compiler.planner.models import FunctionPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild._native import FunctionPlanEntry, MaterializationType, ModelPlanEntry from sqlbuild.executor.clone.main.execute import execute_clone from sqlbuild.executor.clone.models import ( CloneExecutionInput, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 43417dd2fc..ff95e704a1 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -9,7 +9,20 @@ from pathlib import Path from typing import Any, ClassVar, cast -from sqlbuild._native import CompiledSeed, CompiledSource, CompileModelConfig +from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSeed, + CompiledSource, + CompileModelConfig, + SchemaSeedEntry, + SeedCsvSettings, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ( @@ -21,22 +34,13 @@ SchemaDiffResult, ) from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SchemaSeedEntry, SeedCsvSettings, SourceEntry UNQUOTED_ADDRESSING_REASON: str = ( "relation name is not a plain lowercase identifier and may require quoting; " diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index fbfd2166cc..4f85fd29c9 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + SOURCE_FRESHNESS_TABLE_NAME, build_janitor_plan, build_scenario_artifact_name, compute_scenario_hash_prefix, ) -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.models import ( JanitorExecutionResult, diff --git a/tests/unit/src/sqlbuild/executor/load/_test_types.py b/tests/unit/src/sqlbuild/executor/load/_test_types.py index d7883ac3d9..20f6a9fc0a 100644 --- a/tests/unit/src/sqlbuild/executor/load/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/load/_test_types.py @@ -6,12 +6,11 @@ from datetime import date, datetime from typing import Any +from sqlbuild._native import LoaderConnectionMode, SourceColumnEntry from sqlbuild.adapter.contract.types import CursorKind -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceColumnEntry @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/load/test_cursors.py b/tests/unit/src/sqlbuild/executor/load/test_cursors.py index 12e9540275..249c33ddd3 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_cursors.py +++ b/tests/unit/src/sqlbuild/executor/load/test_cursors.py @@ -7,10 +7,10 @@ import pytest +from sqlbuild._native import DateValue, IntegerValue, TimestampValue from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.cursors import _parse_loader_cursor, infer_cursor_kind diff --git a/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py b/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py index cabeea0117..d06f82daf1 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py +++ b/tests/unit/src/sqlbuild/executor/load/test_dag_runtime.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import SourceEntry from sqlbuild.executor.load._helpers.dag_runtime import ( build_load_dag_state, complete_dag_source, @@ -19,7 +20,6 @@ LoadRuntimeParams, ) from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.load._test_types import ( LoadDagStateSchedulingTestCase, LoadDagWorkerFailureTestCase, diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index 9e5203a0d5..4b277f089a 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -9,10 +9,14 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import ( + CostContext, + DiscoveredLoaderFunction, + LoaderConnectionMode, + SourceEntry, + SourceWriteStrategy, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.load.main._execute import execute_source_load from sqlbuild.executor.load.models import ( @@ -31,8 +35,6 @@ from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.executor.load._test_types import ( ExternalLoaderContractTestCase, LoaderOperationLifecycleTestCase, diff --git a/tests/unit/src/sqlbuild/executor/load/test_rows.py b/tests/unit/src/sqlbuild/executor/load/test_rows.py index afcc0b9c14..b4b006dfaf 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_rows.py +++ b/tests/unit/src/sqlbuild/executor/load/test_rows.py @@ -8,6 +8,7 @@ import duckdb import pytest +from sqlbuild._native import SourceColumnEntry from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.rows import ( @@ -17,7 +18,6 @@ update_loader_rows_schema, ) from sqlbuild.executor.load.models import LoaderRowsSchema -from sqlbuild.spec.contracts.models import SourceColumnEntry from tests.unit.src.sqlbuild.executor.load._test_types import ( LoaderRowsBatchTestCase, LoaderRowsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index ecab9d43a5..1d5bc8a8e9 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -8,9 +8,13 @@ import pytest -from sqlbuild._native import LoadProgressReporter, NativeProgressProjector -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.compiler.discovery.types import LoaderConnectionMode +from sqlbuild._native import ( + DiscoveredLoaderFunction, + LoaderConnectionMode, + LoadProgressReporter, + NativeProgressProjector, + SourceEntry, +) from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.main.run import run_load_pipeline from sqlbuild.executor.load.models import ( @@ -26,7 +30,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.load._test_types import ( ConcurrentLoadProgressTestCase, ExternalLoadPipelineTestCase, diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py index 8188fa6641..d73f04aaf8 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/helpers.py @@ -9,16 +9,18 @@ import duckdb from sqlbuild._native import ( + ColumnMigrationDecision, + ColumnMigrationEvent, + ColumnMigrationPlanEntry, + CompiledRelationLocation, MigrationDiscovery, + MigrationRelation, OldNameViewAction, + OldNameViewPlanEntry, deterministic_column_migration_event_id, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent, MigrationRelation -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision -from sqlbuild.compiler.planner.models import ColumnMigrationPlanEntry, OldNameViewPlanEntry RELATION: str = "main.fct_orders" diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py index 5df4303783..f67bdd91a1 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_columns.py @@ -5,9 +5,8 @@ import duckdb import pytest -from sqlbuild._native import read_migration_state_events +from sqlbuild._native import ColumnMigrationEvent, read_migration_state_events from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.executor.migrations._helpers.columns import apply_model_column_renames from tests.unit.src.sqlbuild.executor.migrations._helpers._test_types import ( OverlappingColumnRenameTestCase, diff --git a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py index f385f71556..3f8ead34bd 100644 --- a/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py +++ b/tests/unit/src/sqlbuild/executor/migrations/_helpers/test_old_name_views.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild._native import ColumnMigrationEvent from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationGrant from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,7 +14,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.executor.migrations._helpers.old_name_views import ( compose_column_aliases, reader_access_warning, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index a680ad3f23..1ed9f9368a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -9,7 +9,21 @@ from unittest.mock import MagicMock from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledProject, + CompiledRelationLocation, + CompiledSqlScenario, + CompilePipelineResult, + DiscoveredSqlScenarioFile, + PlanOutput, + ScenarioExecutionPlan, ScenarioGraphPlan, + ScenarioRelationMap, + ScenarioRelationPlan, + SeedCsvSettings, + SeedPlanEntry, + SqlLexicalSyntax, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder @@ -20,24 +34,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSqlScenario, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - PlanOutput, - ScenarioExecutionPlan, - ScenarioRelationMap, - ScenarioRelationPlan, - SeedPlanEntry, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_LOCAL_JSONL_INVALID @@ -47,7 +44,6 @@ ) from sqlbuild.observability import LifecycleEvent from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.executor.pipeline._helpers._test_types import ( ScenarioLocalPipelineTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index c7aefb0ff7..09d1b648da 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -19,8 +19,11 @@ import pytest from sqlbuild._native import ( + AuditPlanEntry, AuditProgressReporter, + CompiledObjectKey, CostContext, + PlanOutput, StatementLedgerEntry, audit_resource_id, read_statement_ledger, @@ -34,9 +37,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType -from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py index 94721e1ba7..efb84cb39d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_connections.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import BuildCallbacks from sqlbuild.executor.pipeline._helpers import connections as connections_module from sqlbuild.executor.pipeline._helpers.connections import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index 02d472bc5b..75f64d2ecc 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -7,10 +7,12 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import ( + CompiledSqlScenario, + CompilePipelineResult, + ScenarioExecutionPlan, + SqlLexicalSyntax, +) from sqlbuild.executor.pipeline._helpers import scenario as scenario_pipeline from sqlbuild.executor.scenario._helpers.local import execution as local_scenario from sqlbuild.executor.scenario.constants import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py index a7d77ae537..28c6ee74a1 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_seeding.py @@ -8,8 +8,7 @@ import pytest -from sqlbuild._native import TerminalEventIndex, format_seed_execution_json -from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild._native import PlanOutput, TerminalEventIndex, format_seed_execution_json from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.pipeline._helpers.seeding import run_seed_pipeline from sqlbuild.observability import ( diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py index 6401a444f4..f3eb21e27d 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py @@ -7,13 +7,15 @@ import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( + ChainStep, CompiledObjectKey, CompiledRelationLocation, + PlanOutput, + SqlTestPlanEntry, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.pipeline._helpers import testing as testing_pipeline from sqlbuild.executor.pipeline._helpers.testing import run_test_pipeline from sqlbuild.executor.pipeline.models import TestPipelineCallbacks as PipelineCallbacks diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py index b3d0b38db2..aa1c4745d4 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py @@ -3,22 +3,25 @@ from pathlib import Path from typing import ClassVar -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, FunctionPlanEntry, + MaterializationType, ModelPlanEntry, + PlanAction, PlanOutput, + PlanReason, SeedPlanEntry, + SourceEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SourceEntry class BuildSchemaPreflightAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py index bcf529cd61..21afaeaa3c 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_identity.py @@ -7,9 +7,8 @@ import pytest import sqlbuild.executor.pipeline.main.run as run_module -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, PlanOutput, SettingsConfig from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams from sqlbuild.executor.build.types import BuildStatus @@ -21,7 +20,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SettingsConfig from tests.unit.src.sqlbuild.executor.pipeline.main._test_types import ( PipelineIdentityTestCase, RunTerminalTestCase, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py index 4d1a4dfa06..1118c704ba 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py @@ -4,10 +4,16 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, + PlanReason, +) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.pipeline._helpers.graph_width import runnable_graph_width from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from tests.unit.src.sqlbuild.executor.pipeline.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index cbdb2539f1..d775f4e8ce 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -13,26 +13,25 @@ from sqlbuild._native import ( CostContext, - PythonSqlRunSelection, - build_python_node_graph, - build_python_sql_run_lifecycle, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.assets import AssetContext, asset -from sqlbuild.checks import CheckContext -from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, DiscoveredLoaderFunction, DiscoveredProjectInputs, - DiscoveredTaskFunction, -) -from sqlbuild.compiler.python_nodes.models import ( DiscoveredPythonNode, + DiscoveredTaskFunction, + LocalConfig, + ProjectConfig, PythonNodeGraph, PythonSqlRunLifecyclePlan, + PythonSqlRunSelection, + SourceEntry, + build_python_node_graph, + build_python_sql_run_lifecycle, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.assets import AssetContext, asset +from sqlbuild.checks import CheckContext from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError @@ -53,7 +52,6 @@ from sqlbuild.providers import Provider from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.refs import model -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry from sqlbuild.tasks import TaskContext, task from tests.unit.src.sqlbuild.compiler.python_nodes._helpers.helpers import ( build_intermediate_loader_asset_dependency_python_node_graph, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index e50de69cc8..41439907dd 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DiscoveredAssetFunction, DiscoveredTaskFunction from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.discovery.models import DiscoveredAssetFunction, DiscoveredTaskFunction from sqlbuild.compiler.python_nodes.types import PythonNodeStatus from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.classes.direct_store import DirectNodeResultStore diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py index 24230a7762..f81724f4f0 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity +from sqlbuild._native import PythonNodeIdentity from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py index 14e3f59a08..573a3696a3 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph +from sqlbuild._native import DiscoveredPythonNode, PythonNodeGraph, SourceEntry from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.executor.load.main._resource_kind import load_resource_kind from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( @@ -29,7 +29,6 @@ dispatcher_scope, invocation_scope, ) -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( IngressSchedulerSkipLifecycleTestCase, PythonIngressLoaderExecutorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py index 98819bdafa..29ed327271 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_lifecycle_nodes.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan +from sqlbuild._native import PythonNodeGraph, PythonSqlRunLifecyclePlan from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import build_ingress_lifecycle_nodes from sqlbuild.executor.scheduling.models import LifecycleExecutionNode from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py index dc5da97064..c735ce8329 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_python_checks.py @@ -10,13 +10,13 @@ from sqlbuild._native import ( CostContext, + DiscoveredCheckFunction, ExecutionEventWriter, + PythonNodeGraph, TerminalEventIndex, format_check_json, terminal_event_index_scope, ) -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus, SkipMode from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.node_results.models import NodeResultEnvelope diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py index be3f30a40b..a40b19a781 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_read_side_execution.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph +from sqlbuild._native import PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeStatus from sqlbuild.executor.python_nodes.classes.read_side_python_execution_tracker import ( ReadSidePythonExecutionTracker, diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py index 89b16ada8b..a6f0961b84 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind +from sqlbuild._native import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py index fb6847b0d3..d178f0f5bd 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/classes/test_runtime_relation_guard.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind +from sqlbuild._native import HardCodedRelationOwnerKind from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.refs import model, source diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index da728214e4..481ec6c633 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field +from sqlbuild._native import PlanReason, SchemaDynamicColumnFamily from sqlbuild.adapter.contract.models import ColumnInfo, LifeCycleEvent -from sqlbuild.compiler.planner.types import PlanReason from sqlbuild.executor.run.types import ( AuditGateReuseReason, AuditGateStatus, @@ -9,7 +9,6 @@ RuntimeCursorWatermarkMode, ) from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 4c6844f2fe..37a09440f3 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -2,6 +2,19 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledRelationLocation, + DiscoveredHookFunction, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + PythonHookEntry, + SchemaDynamicColumnFamily, + SourceEntry, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -15,20 +28,12 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, -) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext from sqlbuild.executor.run.types import HookPhase from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceEntry def insert_snapshot_hook_log(ctx: HookContext, phase: str) -> None: diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py index 63cefeb15a..32e1292e35 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_contracts.py @@ -2,11 +2,10 @@ import pytest +from sqlbuild._native import ModelPlanEntry, SchemaDynamicColumnFamily from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( RuntimeContractValidationTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 2fd1c8da80..34da88d5e4 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -9,29 +9,29 @@ import pytest from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + CursorBounds, + CursorGrain, + CursorInputRelation, + CursorType, CursorWatermarkMode, + FutureCursorAction, + FutureCursorSafetyError, + FutureCursorsConfig, + MaterializationType, + ModelCursorSnapshot, + ModelPlanEntry, + PlanAction, + PlanReason, + StartCursorsConfig, normalize_cursor_snapshot_grain, ) from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.exceptions import FutureCursorSafetyError -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - ModelCursorSnapshot, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, - MaterializationType, - PlanAction, - PlanReason, -) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, @@ -41,8 +41,6 @@ from sqlbuild.executor.run.exceptions import EmptyCursorInputsError from sqlbuild.executor.run.models import RuntimeCursorInputRelation, RuntimeCursorSpec from sqlbuild.executor.run.types import RuntimeCursorWatermarkMode -from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig -from sqlbuild.spec.contracts.types import FutureCursorAction from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( AuthoritativeRuntimeCursorOverrideTestCase, CursorSentinelSubstitutionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py index 2942843573..36a5418f99 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_custom_query_change.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.types import PlanReason +from sqlbuild._native import PlanReason from sqlbuild.executor.run._helpers.materializations.custom import reports_query_change from tests.unit.src.sqlbuild.executor.run._helpers._test_types import CustomQueryChangeTestCase diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py index 1132488185..74ce5ac758 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_fingerprint_metadata.py @@ -9,10 +9,8 @@ import pytest +from sqlbuild._native import AUDIT_GATE_METADATA_KEY, AuditPlanEntry, Fingerprint, ModelPlanEntry from sqlbuild.compiler.auditing.types import AuditOutcome, AuditSeverity -from sqlbuild.compiler.fingerprints.constants import AUDIT_GATE_METADATA_KEY -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import AuditPlanEntry, ModelPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse import fingerprinting from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py index 934ce4aedc..970a227712 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_hooks.py @@ -7,12 +7,16 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import ( + CompiledRelationLocation, + CostContext, + DiscoveredHookFunction, + ModelPlanEntry, + PythonHookEntry, + SqlHookEntry, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.cost.models import CostResourceContext from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks, render_hooks diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py index 4d865e97ca..a997e8f6ce 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_inclusive_end_batch_windows.py @@ -9,11 +9,11 @@ import pytest +from sqlbuild._native import CursorGrain, CursorType from sqlbuild._native import cursor_compare as compare from sqlbuild._native import cursor_inclusive_to_exclusive as inclusive_to_exclusive from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render -from sqlbuild.compiler.planner.types import CursorGrain, CursorType from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.executor.run._helpers.materializations.microbatch import compute_batch_windows from sqlbuild.executor.run.models import BatchWindow diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py index 0bf29e1032..de21a651c1 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py @@ -4,10 +4,10 @@ import pytest +from sqlbuild._native import OnSchemaChange from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import PromotionStrategy -from sqlbuild.compiler.planner.types import OnSchemaChange from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.execution.promotion import promote_relation_to_destination from sqlbuild.executor.run._helpers.execution.staging import create_staging_relation diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py index d5553fd303..1988180e60 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_reuse_audit.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.planner.models import AuditPlanEntry +from sqlbuild._native import AuditPlanEntry from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.reuse.audit import reused_final_audit_results_by_binding_key from sqlbuild.executor.run._helpers.reuse.fingerprint_metadata import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py index 165c3aa037..d9c4caf2ae 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py @@ -8,16 +8,17 @@ import pytest -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, HistoricalInput, MaterializationType, + ModelPlanEntry, PlanAction, PlanReason, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py index aee7401e98..9217840fbd 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_lifecycle.py @@ -7,13 +7,19 @@ import pytest +from sqlbuild._native import ( + FINGERPRINT_TABLE_NAME, + NODE_TYPE_MODEL, + DiscoveredHookFunction, + ModelPlanEntry, + PythonHookEntry, + SnapshotsConfig, + SqlHookEntry, +) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_MODEL -from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.materializations.snapshot import ( _apply_snapshot_schema_change, @@ -22,7 +28,6 @@ from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.run.types import ExecutionPhase from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SnapshotsConfig from tests.unit.src.sqlbuild.executor.run._helpers._test_types import ( SnapshotHookFailureTestCase, SnapshotLifecycleTestCase, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py index 1abc8b0deb..63e0cc8ae0 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from pathlib import Path -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCapturePlan, ScenarioSnapshotFileStats, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py index c9ef94e5bb..8d88deaef8 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py @@ -5,29 +5,23 @@ from typing import cast from sqlbuild._native import ( - ScenarioGraphPlan, -) -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( + MaterializationType, ModelPlanEntry, + PlanAction, + PlanReason, + ScenarioArtifactKind, ScenarioAssertionExpectationPlan, ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, + ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, - ScenarioArtifactKind, -) +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCaptureRunResult, ScenarioSnapshotColumn, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py index 352c0adcb5..ef2f111d58 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture.py @@ -5,8 +5,7 @@ import pytest -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan, SeedPlanEntry -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan, SeedPlanEntry from sqlbuild.executor.scenario._helpers.capture import core as capture_core from sqlbuild.executor.scenario._helpers.capture.core import execute_scenario_snapshot_capture_steps from sqlbuild.executor.scenario.models import ( diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py index 540d94ae32..a5e3afef91 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_failures.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.lifecycle.failures import first_failure_details from sqlbuild.executor.scenario.models import ( ScenarioFailureDetails, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py index a6b6cb685e..175f1f0695 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_snapshots.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_capture_plan, build_scenario_snapshot_input_fingerprint, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 8001e4cf88..16e0263698 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -5,32 +5,26 @@ from typing import Any, ClassVar from sqlbuild._native import ( - ScenarioGraphPlan, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult, RelationInfo -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( + MaterializationType, ModelPlanEntry, + PlanAction, + PlanReason, + ScenarioArtifactKind, ScenarioExecutionPlan, ScenarioFixturePlan, + ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, + SeedCsvSettings, SeedPlanEntry, ) -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanAction, - PlanReason, - ScenarioArtifactKind, -) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult, RelationInfo +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS -from sqlbuild.spec.contracts.models import SeedCsvSettings class ScenarioFixtureTestAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py index 591bf67964..da2be93081 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py @@ -5,10 +5,9 @@ import pytest +from sqlbuild._native import ScenarioArtifactKind, ScenarioExecutionPlan from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.snapshots.core import ( build_scenario_snapshot_capture_plan, build_scenario_snapshot_manifest_shell, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py index 3503c7dcf8..805788c1eb 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_cleanup.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import MaterializationType from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.scenario.main._cleanup import execute_scenario_cleanup from sqlbuild.executor.scenario.main._run import execute_scenario_run from sqlbuild.executor.scenario.models import ScenarioCleanupExecutionResult, ScenarioRunOptions diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py index 17aaa5106a..bc9d6bc935 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_fixtures.py @@ -2,9 +2,8 @@ import pytest +from sqlbuild._native import ScenarioArtifactKind, ScenarioFixturePlan from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.planner.models import ScenarioFixturePlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind from sqlbuild.executor.scenario._helpers.lifecycle.fixtures import execute_scenario_fixtures from sqlbuild.executor.scenario.main._fixtures import execute_scenario_fixture from sqlbuild.executor.scenario.models import ScenarioFixtureExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py index 1f46373255..082f42e4a7 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_models.py @@ -2,9 +2,8 @@ import pytest +from sqlbuild._native import MaterializationType, ModelPlanEntry, PlanAction, ScenarioExecutionPlan from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.planner.models import ModelPlanEntry, ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models from sqlbuild.executor.scenario.main._execute import execute_scenario_model diff --git a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py index d1e9753df4..1f2a173ddf 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py @@ -4,11 +4,14 @@ from pathlib import Path +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + SeedCsvSettings, + SeedPlanEntry, +) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import SeedPlanEntry -from sqlbuild.spec.contracts.models import SeedCsvSettings def build_seed_plan_entry(*, seed_name: str, file_path: Path) -> SeedPlanEntry: diff --git a/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py b/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py index b61f5a5ef2..88731ae8f5 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/test_execute.py @@ -7,9 +7,9 @@ import pytest +from sqlbuild._native import FINGERPRINT_TABLE_NAME from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.main._execute import execute_seed diff --git a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py index 85e5494de1..2dc0788dde 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py @@ -1,12 +1,10 @@ -from sqlbuild._native import SqlTestAssertionStep +from sqlbuild._native import ChainStep, CompiledObjectKey, SqlTestAssertionStep, SqlTestPlanEntry from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry def build_comparison_test_entry(*, sql_analysis_enabled: bool = True) -> SqlTestPlanEntry: diff --git a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py index 6f7a19a455..8b85a2b55f 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import render_sql_test_comparison_sql +from sqlbuild._native import ChainStep, SqlTestPlanEntry, render_sql_test_comparison_sql from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry from tests.unit.src.sqlbuild.executor.testing.main._test_types import ( BuildComparisonSqlTestCase, ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py b/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py index ce9886ddbd..1d5876813d 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/_test_types.py @@ -124,6 +124,20 @@ class DagsterAssetCheckFilterTestCase: unexpected_check_names: tuple[str, ...] +@dataclass(frozen=True) +class DagsterCliArgumentTypeTestCase: + description: str + args: tuple[object, ...] + expected_args: tuple[str, ...] + + +@dataclass(frozen=True) +class DagsterCliArgumentErrorTestCase: + description: str + args: tuple[object, ...] + expected_error: str + + @dataclass(frozen=True) class DagsterCliInvocationTestCase: description: str @@ -304,3 +318,11 @@ class DagsterUnitTestSelectorTestCase: select: str expected_code: str expected_message: str + + +@dataclass(frozen=True) +class DagsterCheckTranslatorCallTestCase: + description: str + include_scenarios: bool + expected_is_asset_check_ids: tuple[str, ...] + expected_checked_node_last_key: str diff --git a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py index 4da70ff215..d97ed5225c 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py @@ -7,6 +7,9 @@ from typing import Any from sqlbuild._native import IntegrationResultEnvelope +from sqlbuild.integrations.dagster.classes.sqlbuild_dagster_translator import ( + SqlBuildDagsterTranslator, +) def assert_exhaustive_enum_partition[EnumT: StrEnum]( @@ -621,3 +624,19 @@ def build_dagster_selection_graph_dag() -> Mapping[str, Any]: ], "checks": [], } + + +class RecordingDagsterTranslator(SqlBuildDagsterTranslator): + """Default Dagster translator that records the checks and nodes it is asked about.""" + + def __init__(self) -> None: + self.asset_checks: list[Mapping[str, Any]] = [] + self.asset_nodes: list[Mapping[str, Any]] = [] + + def is_asset_node(self, node: Mapping[str, Any]) -> bool: + self.asset_nodes.append(node) + return super().is_asset_node(node) + + def is_asset_check(self, check: Mapping[str, Any]) -> bool: + self.asset_checks.append(check) + return super().is_asset_check(check) diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py index eb881eb96c..31739b05c8 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_assets.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.planner.exceptions import PlannerInputError +from sqlbuild._native import PlannerInputError from sqlbuild.integrations.dagster import ( SqlBuildDagsterTranslator, build_sqlbuild_asset_selection, @@ -25,6 +25,7 @@ DagsterAssetSelectionErrorTestCase, DagsterAssetSelectionTestCase, DagsterAssetSpecTestCase, + DagsterCheckTranslatorCallTestCase, DagsterConflictingInputTestCase, DagsterDecoratorTestCase, DagsterDescriptionTestCase, @@ -35,6 +36,7 @@ DagsterUnitTestSelectorTestCase, ) from tests.unit.src.sqlbuild.integrations.dagster.helpers import ( + RecordingDagsterTranslator, build_dagster_selection_graph_dag, build_dagster_test_dag, build_python_augmented_dagster_test_dag, @@ -689,3 +691,40 @@ def assets_def() -> dg.MaterializeResult: assert raised.value.code == test_case.expected_code assert raised.value.message == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterCheckTranslatorCallTestCase( + description="excluded scenario checks are still offered to the translator", + include_scenarios=False, + expected_is_asset_check_ids=( + "audit:not_null:model:orders:order_id", + "audit:freshness:source:raw_orders:loaded_at", + "sql_scenario:orders_minimal", + "sql_scenario:customers_minimal", + ), + expected_checked_node_last_key="project_name", + ) + ], + ids=lambda case: case.description, +) +def test_given_check_filter_when_building_check_specs_then_translator_sees_every_check( + test_case: DagsterCheckTranslatorCallTestCase, +) -> None: + translator: RecordingDagsterTranslator = RecordingDagsterTranslator() + + _ = build_check_specs( + dag=build_dagster_test_dag(), + translator=translator, + include_scenarios=test_case.include_scenarios, + ) + checks: list[Mapping[str, Any]] = translator.asset_checks + checked_nodes: list[Mapping[str, Any]] = translator.asset_nodes + + assert tuple(str(check["id"]) for check in checks) == test_case.expected_is_asset_check_ids + assert checked_nodes + assert all( + tuple(node)[-1] == test_case.expected_checked_node_last_key for node in checked_nodes + ) diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py index a80d262c6a..328b01665b 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py @@ -12,8 +12,13 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import IntegrationResultEnvelope, serialize_maximum_start_safety -from sqlbuild.compiler.planner.models import MaximumStartSafetyEvidence +from sqlbuild._native import ( + FutureCursorAction, + IntegrationResultEnvelope, + MaximumStartSafetyEvidence, + MicrobatchLimitAction, + serialize_maximum_start_safety, +) from sqlbuild.integrations.dagster import ( SqlBuildCliResource, SqlBuildDagsterTranslator, @@ -23,9 +28,10 @@ _build_results_from_integration_result, ) from sqlbuild.integrations.dagster.classes.sqlbuild_cli_invocation import SqlBuildCliInvocation -from sqlbuild.spec.contracts.types import FutureCursorAction, MicrobatchLimitAction from tests.unit.src.sqlbuild.integrations.dagster._test_types import ( DagsterAuditIdentityTestCase, + DagsterCliArgumentErrorTestCase, + DagsterCliArgumentTypeTestCase, DagsterCliCloneFailureTestCase, DagsterCliCloneStreamTestCase, DagsterCliFailureTestCase, @@ -805,6 +811,62 @@ def test_given_sqlbuild_cli_resource_when_waiting_invocation_then_captures_proce assert invocation.stderr == test_case.expected_stderr +@pytest.mark.parametrize( + "test_case", + [ + DagsterCliArgumentTypeTestCase( + description="path-like arguments reach the subprocess as paths", + args=("build", "--select-file", Path("selectors/orders.txt")), + expected_args=("build", "--select-file", "selectors/orders.txt"), + ), + ], + ids=lambda case: case.description, +) +def test_given_path_like_cli_arguments_when_starting_without_context_then_paths_pass_through( + test_case: DagsterCliArgumentTypeTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + resource: SqlBuildCliResource = SqlBuildCliResource( + project_dir=str(project_dir), + sqb_command=write_fake_sqb_command(root=tmp_path, expected_args=test_case.expected_args), + ) + + invocation: SqlBuildCliInvocation = resource.cli( + cast(list[str], list(test_case.args)), raise_on_error=False + ).wait() + + assert invocation.is_successful(), invocation.stderr + assert invocation.command[-len(test_case.args) :] == test_case.expected_args + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterCliArgumentErrorTestCase( + description="integer argument fails as string methods would", + args=("build", 3), + expected_error="'int' object has no attribute 'startswith'", + ), + ], + ids=lambda case: case.description, +) +def test_given_other_non_string_cli_arguments_when_starting_then_attribute_error_is_raised( + test_case: DagsterCliArgumentErrorTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + project_dir.mkdir() + resource: SqlBuildCliResource = SqlBuildCliResource( + project_dir=str(project_dir), + sqb_command=write_fake_sqb_command(root=tmp_path), + ) + + with pytest.raises(AttributeError) as raised: + resource.cli(cast(list[str], list(test_case.args))) + + assert str(raised.value) == test_case.expected_error + + @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index ff5d42a301..0da4a83d31 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -3,8 +3,13 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import DbtCliOptions, DbtCommandResult, DbtInteropParsedArgs -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig +from sqlbuild._native import ( + DbtCliOptions, + DbtCommandResult, + DbtConfig, + DbtInteropParsedArgs, + LocalDbtConfig, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 5fbed29ce9..9d88c15421 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -9,8 +9,12 @@ from typing import cast from sqlbuild._native import ( + BackfillAction, BackfillResult, + CompiledObjectKey, + CompiledRelationLocation, CompileModelConfig, + CompileModelInput, CompileProjectInputs, DbtCliConfigOverrides, DbtCliOptions, @@ -19,6 +23,15 @@ DbtEventStream, DbtEventStreamUpdate, DbtRunner, + DiscoveredProjectInputs, + DiscoveredSqlModelFile, + LocalConfig, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, + ProjectConfig, + SqlLexicalSyntax, assemble_project, build_dbt_combined_graph, build_dbt_ls_argv, @@ -26,22 +39,7 @@ extract_sql_references, transitive_closure, ) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompileModelInput, -) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py index 644ba08b2d..3498a89fa3 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/helpers.py @@ -2,7 +2,7 @@ from pathlib import Path -from sqlbuild.spec.contracts.models import DbtConfig, ProjectConfig +from sqlbuild._native import DbtConfig, ProjectConfig def write_duckdb_dbt_workspace(root: Path) -> None: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py index 0cf38d3fce..ba5ac6adaa 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_normalize.py @@ -4,9 +4,12 @@ import pytest -from sqlbuild._native import NormalizedDbtProfileConnection, normalize_dbt_profile_connection +from sqlbuild._native import ( + NormalizedDbtProfileConnection, + ResolvedDbtProfileOutput, + normalize_dbt_profile_connection, +) from sqlbuild.integrations.dbt.exceptions import DbtProfileError -from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileNormalizeErrorTestCase, DbtProfileNormalizeTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py index 0e43ada08a..184fad2b22 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/profile/test_render.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import ResolvedDbtProfileOutput, SelectedDbtProfileOutput from sqlbuild.integrations.dbt._helpers.profile.render import render_selected_dbt_profile_output -from sqlbuild.integrations.dbt.models import ResolvedDbtProfileOutput, SelectedDbtProfileOutput from tests.unit.src.sqlbuild.integrations.dbt.profile._test_types import ( DbtProfileRenderErrorTestCase, DbtProfileRenderTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index d08b928f99..438225374a 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -5,12 +5,13 @@ import pytest from sqlbuild._native import ( + DbtConfig, DbtInteropConfigError, + LocalDbtConfig, ResolvedDbtConfig, resolve_dbt_config, resolve_dbt_vars_mapping, ) -from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtConfigErrorTestCase, DbtConfigResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 6396fd1f88..da5187c81c 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -13,6 +13,7 @@ from sqlbuild import _native as phases_module from sqlbuild import _native as plan_module from sqlbuild._native import ( + CompiledProject, DbtCliOptions, DbtCombinedGraph, DbtCommandExecutionResult, @@ -28,14 +29,13 @@ DbtManifestIndex, DbtNodeExecutionResult, DbtRunner, + DiscoveredProjectInputs, build_dbt_interop_plan, build_failed_sqlbuild_model_names, build_merged_dbt_execution_argv, execute_dbt_commands, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py index 588e28708b..ea6f0d786f 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + CompileInputError, DbtManifestIndex, DbtManifestModel, DbtManifestSource, build_dbt_manifest_index, resolve_dbt_manifest_model, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtManifestIndexErrorTestCase, DbtManifestResolutionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py index 564aa3cf45..0be18f8519 100644 --- a/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py +++ b/tests/unit/src/sqlbuild/integrations/dlt/test_dlt_loaders.py @@ -8,9 +8,7 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.discovery.types import LoaderConnectionMode +from sqlbuild._native import DiscoveredProjectInputs, LoaderConnectionMode, discover_project_inputs from tests.unit.src.sqlbuild.integrations.dlt._test_types import DltLoaderDiscoveryTestCase diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py index 591cdf8683..ac2c01efab 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass +from sqlbuild._native import SourceEntry + @dataclass(frozen=True) class IngestrCommandTestCase: @@ -13,6 +15,15 @@ class IngestrCommandTestCase: expected_command: tuple[str, ...] +@dataclass(frozen=True) +class IngestrCommandErrorTestCase: + description: str + source_entry: SourceEntry + adapter_name: str + connection_config: dict[str, object] + expected_message: str + + @dataclass(frozen=True) class IngestrLoaderDiscoveryTestCase: description: str diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py index 6aa6f7607c..bced98e655 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_command.py @@ -4,10 +4,13 @@ import pytest -from sqlbuild.integrations.ingestr._helpers.command import build_ingestr_command +from sqlbuild._native import IntegrationLoaderConfig, SourceEntry, build_ingestr_command +from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.integrations.ingestr.models import IngestrSourceConfig -from sqlbuild.spec.contracts.models import IntegrationLoaderConfig, SourceEntry -from tests.unit.src.sqlbuild.integrations.ingestr._test_types import IngestrCommandTestCase +from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( + IngestrCommandErrorTestCase, + IngestrCommandTestCase, +) @pytest.mark.parametrize( @@ -108,7 +111,7 @@ def test_given_ingestr_source_when_building_command_then_it_uses_sqlbuild_destin source_entry=source_entry, adapter_name=test_case.adapter_name, connection_config=test_case.connection_config, - destination_table="raw.orders", + destination_name="orders", vars={"token": "secret"}, environment="dev", run_id="run-1", @@ -116,3 +119,62 @@ def test_given_ingestr_source_when_building_command_then_it_uses_sqlbuild_destin ) assert command == test_case.expected_command + + +@pytest.mark.parametrize( + "test_case", + [ + IngestrCommandErrorTestCase( + description="unsupported adapter", + source_entry=SourceEntry( + name="raw_orders", + integration_loader=IntegrationLoaderConfig( + kind="ingestr", + config=IngestrSourceConfig(source_uri="csv://x", source_table="orders"), + ), + ), + adapter_name="fabric", + connection_config={}, + expected_message="Adapter 'fabric' does not support ingestr integration loaders", + ), + IngestrCommandErrorTestCase( + description="blank required connection setting", + source_entry=SourceEntry( + name="raw_orders", + integration_loader=IntegrationLoaderConfig( + kind="ingestr", + config=IngestrSourceConfig(source_uri="csv://x", source_table="orders"), + ), + ), + adapter_name="postgres", + connection_config={"user": " "}, + expected_message=( + "Adapter 'postgres' requires connection.user for ingestr integration loaders" + ), + ), + IngestrCommandErrorTestCase( + description="source without ingestr config", + source_entry=SourceEntry(name="raw_orders"), + adapter_name="duckdb", + connection_config={"database": "warehouse.duckdb"}, + expected_message="Source 'raw_orders' does not define ingestr config", + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuildable_ingestr_source_when_building_command_then_raises_integration_error( + test_case: IngestrCommandErrorTestCase, +) -> None: + with pytest.raises(IngestrIntegrationError) as raised: + _ = build_ingestr_command( + source_entry=test_case.source_entry, + adapter_name=test_case.adapter_name, + connection_config=test_case.connection_config, + destination_name="orders", + vars={}, + environment=None, + run_id="run-1", + is_reload=False, + ) + + assert str(raised.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py index 29d71abcd3..466917cf40 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_loaders.py @@ -8,9 +8,7 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.discovery.types import LoaderConnectionMode +from sqlbuild._native import DiscoveredProjectInputs, LoaderConnectionMode, discover_project_inputs from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( IngestrLoaderDiscoveryTestCase, ) diff --git a/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py b/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py index 32e5fc67ba..7fce8bdc2b 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/_test_types.py @@ -21,3 +21,11 @@ class RiversPythonArtifactCompatibilityTestCase: expected_task_group: str expected_asset_group: str expected_asset_metadata_keys: tuple[str, ...] + + +@dataclass(frozen=True) +class RiversTranslatorArgumentTestCase: + description: str + node_id: str + expected_dependency_name_count: int + expected_owned_node_keys_end: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/integrations/rivers/helpers.py b/tests/unit/src/sqlbuild/integrations/rivers/helpers.py index 18d6ff44ba..4f520eb34e 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/helpers.py @@ -1,6 +1,12 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass +from typing import Any + +from sqlbuild.integrations.rivers.classes.sqlbuild_rivers_translator import ( + SqlBuildRiversTranslator, +) @dataclass(frozen=True) @@ -25,3 +31,14 @@ def dep(name: str) -> FakeRiversAssetDep: @dataclass(frozen=True) class FakeRiversModule: AssetDef: type[FakeRiversAssetDef] = FakeRiversAssetDef + + +class RecordingRiversTranslator(SqlBuildRiversTranslator): + """Default Rivers translator that records named nodes by id and project-name presence.""" + + def __init__(self) -> None: + self.named_nodes: dict[tuple[str, bool], list[Mapping[str, Any]]] = {} + + def get_asset_name(self, node: Mapping[str, Any]) -> str: + self.named_nodes.setdefault((str(node["id"]), "project_name" in node), []).append(node) + return super().get_asset_name(node) diff --git a/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py b/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py index 5850aa6b39..755d55f294 100644 --- a/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py +++ b/tests/unit/src/sqlbuild/integrations/rivers/test_assets.py @@ -15,8 +15,12 @@ ) from tests.unit.src.sqlbuild.integrations.rivers._test_types import ( RiversPythonArtifactCompatibilityTestCase, + RiversTranslatorArgumentTestCase, +) +from tests.unit.src.sqlbuild.integrations.rivers.helpers import ( + FakeRiversModule, + RecordingRiversTranslator, ) -from tests.unit.src.sqlbuild.integrations.rivers.helpers import FakeRiversModule @pytest.mark.parametrize( @@ -77,3 +81,35 @@ def test_given_python_augmented_dag_when_building_asset_defs_then_maps_python_no assert task_def.group == test_case.expected_task_group assert python_asset_def.group == test_case.expected_asset_group assert all(key in python_asset_def.metadata for key in test_case.expected_asset_metadata_keys) + + +@pytest.mark.parametrize( + "test_case", + [ + RiversTranslatorArgumentTestCase( + description="dependencies are the artifact's own nodes without the project name", + node_id="model:orders", + expected_dependency_name_count=1, + expected_owned_node_keys_end=("materialization_type", "project_name"), + ) + ], + ids=lambda case: case.description, +) +def test_given_python_augmented_dag_when_building_asset_defs_then_translator_sees_artifact_nodes( + test_case: RiversTranslatorArgumentTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + dag: Mapping[str, Any] = build_python_augmented_dagster_test_dag() + monkeypatch.setattr(rivers_assets, "load_rivers", lambda: FakeRiversModule()) + translator: RecordingRiversTranslator = RecordingRiversTranslator() + + _ = build_asset_defs(dag=dag, translator=translator) + artifact_node: Mapping[str, Any] = {str(node["id"]): node for node in dag["nodes"]}[ + test_case.node_id + ] + dependencies: list[Mapping[str, Any]] = translator.named_nodes[(test_case.node_id, False)] + owned_nodes: list[Mapping[str, Any]] = translator.named_nodes[(test_case.node_id, True)] + + assert len(dependencies) == test_case.expected_dependency_name_count + assert all(dependency is artifact_node for dependency in dependencies) + assert [tuple(node)[-2:] for node in owned_nodes] == [test_case.expected_owned_node_keys_end] diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index 111ccd092b..ea414169a9 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -7,13 +7,13 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, LintConfig, LintRunResult, collect_lint_project_files, discover_project_inputs, run_lint, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.lint._helpers._test_types import ( CollectProjectFilesTestCase, SelectedProjectFilesTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 0ce8cb5187..67f78c8c4b 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild._native import SqlLexicalSyntax @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py index 1199b16010..c723e451f2 100644 --- a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -10,10 +10,8 @@ import sqlbuild._native as fixture_formatting_module import sqlbuild._native as native_module -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import DiscoveredProjectInputs, SqlLexicalSyntax, discover_project_inputs from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase from tests.unit.src.sqlbuild.lint.helpers import write_fixture_format_project diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 74b0cb46d6..d9f90d1c8a 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -9,6 +9,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + DiscoveredProjectInputs, LintBody, LintConfig, LintRunResult, @@ -22,7 +23,6 @@ prepare_lint_body, run_lint, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.lint._test_types import ( ExpandedLintTestCase, ExpandedTypedConstantTestCase, diff --git a/tests/unit/src/sqlbuild/microbatches/test_event_codec.py b/tests/unit/src/sqlbuild/microbatches/test_event_codec.py index db947e387c..7a70e45777 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_event_codec.py +++ b/tests/unit/src/sqlbuild/microbatches/test_event_codec.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import project_microbatch_coverage from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchEvent, diff --git a/tests/unit/src/sqlbuild/microbatches/test_projection.py b/tests/unit/src/sqlbuild/microbatches/test_projection.py index 3b7f438193..98932a28c3 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_projection.py +++ b/tests/unit/src/sqlbuild/microbatches/test_projection.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement +from sqlbuild._native import project_microbatch_coverage, project_replay_requirement from sqlbuild.microbatches.models import ( MicrobatchCoverageProjection, MicrobatchInterval, diff --git a/tests/unit/src/sqlbuild/microbatches/test_sql.py b/tests/unit/src/sqlbuild/microbatches/test_sql.py index c0a21be303..081005fdf6 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_sql.py +++ b/tests/unit/src/sqlbuild/microbatches/test_sql.py @@ -6,6 +6,13 @@ import pytest +from sqlbuild._native import ( + CursorType, + build_microbatch_create_table_sql, + build_microbatch_insert_many_sql, + build_microbatch_read_scope_sql, + deterministic_microbatch_event_id, +) from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -15,17 +22,7 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.planner.types import CursorType -from sqlbuild.microbatches._helpers.sql import ( - build_create_table_sql, - build_insert_many_sql, - build_insert_sql, - build_read_scope_sql, -) from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS, MICROBATCH_TABLE_NAME -from sqlbuild.microbatches.main.deterministic_event_id import ( - deterministic_microbatch_event_id, -) from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope from sqlbuild.microbatches.types import MicrobatchRecordType from tests.unit.src.sqlbuild.microbatches._test_types import ( @@ -115,7 +112,7 @@ def test_given_builtin_adapter_when_building_state_ddl_then_all_columns_are_decl test_case: MicrobatchDdlAdapterTestCase, ) -> None: adapter: BaseAdapter = test_case.adapter - sql: str = build_create_table_sql( + sql: str = build_microbatch_create_table_sql( database="warehouse", schema="analytics", render_qualified_name=adapter.render_qualified_name, @@ -177,8 +174,8 @@ def test_given_completion_when_building_insert_then_provenance_and_numeric_rows_ ) -> None: event: MicrobatchEvent = completion_for_sql() - sql: str = build_insert_sql( - event=event, render_qualified_name=DuckDbAdapter().render_qualified_name + sql: str = build_microbatch_insert_many_sql( + events=(event,), render_qualified_name=DuckDbAdapter().render_qualified_name ) assert "INSERT INTO analytics._sqlbuild_microbatches" in sql @@ -190,22 +187,39 @@ def test_given_completion_when_building_insert_then_provenance_and_numeric_rows_ assert sql == ( "INSERT INTO analytics._sqlbuild_microbatches (event_id, record_type, scope_kind, " "scope_key, model_name, target_database, target_schema, target_name, " - "physical_generation_id, " - "origin_run_id, origin_run_started_at, execution_run_id, execution_run_started_at, " - "run_type, completion_type, run_start, run_end, partition_start, partition_end, " - "batch_size, cursor_column, cursor_type, cursor_grain, model_version_hash, " - "definition_hash, fingerprint_status, replay_requirement_id, " - "required_model_version_hash, previous_model_version_hash, replay_policy, " - "rows_affected, completed_at, coverage_source, observed_row_count, observed_at, " - "synthetic_reason, unaccounted_policy, created_at) SELECT 'event-1', " - "'partition_completion', 'direct_logical', 'duckdb:analytics.orders', 'orders', NULL, " - "'analytics', 'orders', '*', 'origin-run', CAST(NULL AS TIMESTAMP), " - "'execution-run', CAST(NULL AS TIMESTAMP), 'replay_on_change', 'recovery', '0', '1', " - "'0', '1', '1', 'batch_id', 'integer', NULL, 'F2', 'fingerprint''definition', " - "'known', NULL, NULL, NULL, NULL, CAST(0 AS BIGINT), CAST(NULL AS TIMESTAMP), NULL, " - "CAST(NULL AS BIGINT), CAST(NULL AS TIMESTAMP), NULL, NULL, " - "CAST('2026-01-01T00:00:00+00:00' AS TIMESTAMP) WHERE NOT EXISTS (SELECT 1 FROM " - "analytics._sqlbuild_microbatches WHERE event_id = 'event-1')" + "physical_generation_id, origin_run_id, origin_run_started_at, execution_run_id, " + "execution_run_started_at, run_type, completion_type, run_start, run_end, " + "partition_start, partition_end, batch_size, cursor_column, cursor_type, " + "cursor_grain, model_version_hash, definition_hash, fingerprint_status, " + "replay_requirement_id, required_model_version_hash, previous_model_version_hash, " + "replay_policy, rows_affected, completed_at, coverage_source, observed_row_count, " + "observed_at, synthetic_reason, unaccounted_policy, created_at) SELECT event_id, " + "record_type, scope_kind, scope_key, model_name, target_database, target_schema, " + "target_name, physical_generation_id, origin_run_id, origin_run_started_at, " + "execution_run_id, execution_run_started_at, run_type, completion_type, run_start, " + "run_end, partition_start, partition_end, batch_size, cursor_column, cursor_type, " + "cursor_grain, model_version_hash, definition_hash, fingerprint_status, " + "replay_requirement_id, required_model_version_hash, previous_model_version_hash, " + "replay_policy, rows_affected, completed_at, coverage_source, observed_row_count, " + "observed_at, synthetic_reason, unaccounted_policy, created_at FROM (SELECT 'event-1'" + " AS event_id, 'partition_completion' AS record_type, 'direct_logical' AS scope_kind," + " 'duckdb:analytics.orders' AS scope_key, 'orders' AS model_name, NULL AS " + "target_database, 'analytics' AS target_schema, 'orders' AS target_name, '*' AS " + "physical_generation_id, 'origin-run' AS origin_run_id, CAST(NULL AS TIMESTAMP) AS " + "origin_run_started_at, 'execution-run' AS execution_run_id, CAST(NULL AS TIMESTAMP) " + "AS execution_run_started_at, 'replay_on_change' AS run_type, 'recovery' AS " + "completion_type, '0' AS run_start, '1' AS run_end, '0' AS partition_start, '1' AS " + "partition_end, '1' AS batch_size, 'batch_id' AS cursor_column, 'integer' AS " + "cursor_type, NULL AS cursor_grain, 'F2' AS model_version_hash, " + "'fingerprint''definition' AS definition_hash, 'known' AS fingerprint_status, NULL AS" + " replay_requirement_id, NULL AS required_model_version_hash, NULL AS " + "previous_model_version_hash, NULL AS replay_policy, CAST(0 AS BIGINT) AS " + "rows_affected, CAST(NULL AS TIMESTAMP) AS completed_at, NULL AS coverage_source, " + "CAST(NULL AS BIGINT) AS observed_row_count, CAST(NULL AS TIMESTAMP) AS observed_at, " + "NULL AS synthetic_reason, NULL AS unaccounted_policy, " + "CAST('2026-01-01T00:00:00+00:00' AS TIMESTAMP) AS created_at) AS incoming WHERE NOT " + "EXISTS (SELECT 1 FROM analytics._sqlbuild_microbatches AS existing WHERE " + "existing.event_id = incoming.event_id)" ) assert len((sql,)) == test_case.expected_statement_count @@ -249,10 +263,10 @@ def test_given_builtin_adapter_when_rendering_direct_event_inserts_then_typed_co ) -> None: event: MicrobatchEvent = completion_for_sql() - single_sql: str = build_insert_sql( - event=event, render_qualified_name=test_case.adapter.render_qualified_name + single_sql: str = build_microbatch_insert_many_sql( + events=(event,), render_qualified_name=test_case.adapter.render_qualified_name ) - bulk_sql: str = build_insert_many_sql( + bulk_sql: str = build_microbatch_insert_many_sql( events=(event, replace(event, event_id="second-event")), render_qualified_name=test_case.adapter.render_qualified_name, ) @@ -275,10 +289,10 @@ def test_given_wildcard_and_concrete_generations_when_reading_then_filter_is_opt ) -> None: scope: MicrobatchScope = completion_for_sql().scope - wildcard_sql: str = build_read_scope_sql( + wildcard_sql: str = build_microbatch_read_scope_sql( scope=scope, render_qualified_name=DuckDbAdapter().render_qualified_name ) - concrete_sql: str = build_read_scope_sql( + concrete_sql: str = build_microbatch_read_scope_sql( scope=MicrobatchScope( scope_kind=scope.scope_kind, scope_key=scope.scope_key, diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py b/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py index 991087bbb9..04fa7ff424 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_build_session.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.discovery.models import DiscoveredProvider +from sqlbuild._native import DiscoveredProvider from sqlbuild.provider._helpers.session import build_provider_session from sqlbuild.provider.classes.session import ProviderSession from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py index 8cc435a57e..7eb3747406 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/_test_types.py @@ -54,3 +54,11 @@ class FactoryDecoratorMetadataTestCase: class FactoryDecoratorErrorTestCase: description: str expected_error_fragment: str + + +@dataclass(frozen=True) +class HookReadsTestCase: + description: str + reads: object + expected_reads: tuple[str, ...] = () + expected_error: str = "" diff --git a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py index 87b8d6c06e..2b40ee7ef0 100644 --- a/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py +++ b/tests/unit/src/sqlbuild/python_nodes/decorators/test_python_decorators.py @@ -8,7 +8,9 @@ from sqlbuild.assets import asset, get_asset_definition from sqlbuild.checks import check, get_check_definition +from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.factories import factory, get_factory_definition +from sqlbuild.hooks import HookDefinition, get_hook_definition, hook from sqlbuild.python_nodes.models import ( AssetDefinition, CheckDefinition, @@ -18,12 +20,14 @@ TaskDefinition, ) from sqlbuild.python_nodes.types import PythonCheckSeverity +from sqlbuild.refs import model, seed from sqlbuild.tasks import get_task_definition, task from tests.unit.src.sqlbuild.python_nodes.decorators._test_types import ( AssetDecoratorMetadataTestCase, CheckDecoratorMetadataTestCase, FactoryDecoratorErrorTestCase, FactoryDecoratorMetadataTestCase, + HookReadsTestCase, TaskDecoratorMetadataTestCase, ) from tests.unit.src.sqlbuild.python_nodes.decorators.helpers import upstream_asset, upstream_task @@ -244,3 +248,63 @@ def test_given_factory_decorator_when_called_without_function_then_raises( factory_ref: Any = cast(Any, factory) with pytest.raises(TypeError, match=test_case.expected_error_fragment): factory_ref() + + +@pytest.mark.parametrize( + "test_case", + [ + HookReadsTestCase( + description="single reference becomes one read", + reads=model("orders"), + expected_reads=("orders",), + ), + HookReadsTestCase( + description="duplicate references keep first-seen order", + reads=[model("orders"), seed("customers"), model("orders")], + expected_reads=("orders", "customers"), + ), + HookReadsTestCase( + description="None as the first invalid item is dropped", + reads=[model("orders"), None, model("products")], + expected_reads=("orders", "products"), + ), + HookReadsTestCase( + description="items after a leading None are filtered, not rejected", + reads=[None, 5, model("orders")], + expected_reads=("orders",), + ), + ], + ids=lambda case: case.description, +) +def test_given_hook_reads_when_decorating_then_keeps_distinct_references( + test_case: HookReadsTestCase, +) -> None: + def refresh_orders() -> None: + return None + + decorated: Any = cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) + definition: HookDefinition | None = get_hook_definition(decorated) + + assert definition is not None + assert tuple(read.name for read in definition.reads) == test_case.expected_reads + + +@pytest.mark.parametrize( + "test_case", + [ + HookReadsTestCase( + description="a non-reference item is rejected", + reads=[model("orders"), "customers"], + expected_error="got str", + ), + ], + ids=lambda case: case.description, +) +def test_given_non_reference_hook_read_when_decorating_then_raises( + test_case: HookReadsTestCase, +) -> None: + def refresh_orders() -> None: + return None + + with pytest.raises(SharedInputError, match=test_case.expected_error): + cast(Any, hook(reads=cast(Any, test_case.reads)))(refresh_orders) diff --git a/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py b/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py index 841628e6d2..fb7bbf10b5 100644 --- a/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py +++ b/tests/unit/src/sqlbuild/python_nodes/main/_test_types.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from sqlbuild.compiler.auditing.types import ThresholdOperator +from sqlbuild.python_nodes.models import RetryPolicy @dataclass(frozen=True) @@ -25,3 +26,21 @@ class InvalidAuditCaseNameTestCase: description: str name: str expected_error_fragment: str + + +@dataclass(frozen=True) +class RetryDelayTestCase: + description: str + policy: RetryPolicy + retry_index: int + expected_delay: float + expected_type: type + + +@dataclass(frozen=True) +class JitteredRetryDelayTestCase: + description: str + policy: RetryPolicy + retry_index: int + seed: int + expected_upper_bound: float diff --git a/tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py b/tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py new file mode 100644 index 0000000000..040bec869d --- /dev/null +++ b/tests/unit/src/sqlbuild/python_nodes/main/test_calculate_retry_delay.py @@ -0,0 +1,93 @@ +import random + +import pytest + +from sqlbuild.python_nodes.main.calculate_retry_delay import calculate_retry_delay +from sqlbuild.python_nodes.models import RetryPolicy +from tests.unit.src.sqlbuild.python_nodes.main._test_types import ( + JitteredRetryDelayTestCase, + RetryDelayTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RetryDelayTestCase( + description="first retry waits the initial delay", + policy=RetryPolicy(jitter=False), + retry_index=0, + expected_delay=1.0, + expected_type=float, + ), + RetryDelayTestCase( + description="later retries grow by the multiplier", + policy=RetryPolicy(jitter=False, initial_delay_seconds=0.5, backoff_multiplier=3.0), + retry_index=2, + expected_delay=4.5, + expected_type=float, + ), + RetryDelayTestCase( + description="the delay is capped", + policy=RetryPolicy(jitter=False, max_delay_seconds=7.5), + retry_index=10, + expected_delay=7.5, + expected_type=float, + ), + RetryDelayTestCase( + description="integer settings keep an integer delay", + policy=RetryPolicy( + jitter=False, initial_delay_seconds=1, backoff_multiplier=2, max_delay_seconds=None + ), + retry_index=3, + expected_delay=8, + expected_type=int, + ), + ], + ids=lambda case: case.description, +) +def test_given_retry_policy_when_calculating_delay_then_returns_backoff_value_and_type( + test_case: RetryDelayTestCase, +) -> None: + delay: float = calculate_retry_delay( + retry_policy=test_case.policy, retry_index=test_case.retry_index + ) + + assert delay == test_case.expected_delay + assert type(delay) is test_case.expected_type + + +@pytest.mark.parametrize( + "test_case", + [ + JitteredRetryDelayTestCase( + description="uncapped jitter", + policy=RetryPolicy(max_delay_seconds=None, initial_delay_seconds=2.0), + retry_index=1, + seed=1234, + expected_upper_bound=4.0, + ), + JitteredRetryDelayTestCase( + description="capped jitter", + policy=RetryPolicy(max_delay_seconds=3.0), + retry_index=5, + seed=7, + expected_upper_bound=3.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_jittered_policy_when_calculating_delay_then_draws_from_python_random( + test_case: JitteredRetryDelayTestCase, +) -> None: + random.seed(test_case.seed) + delay: float = calculate_retry_delay( + retry_policy=test_case.policy, retry_index=test_case.retry_index + ) + random.seed(test_case.seed) + + assert delay == random.uniform(0, test_case.expected_upper_bound) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 2359be6808..0b72c88ed7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -11,29 +11,33 @@ import sqlbuild._native as native from sqlbuild._native import ( + MIN_CUSTOM_RULE_TEST_CASES, CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, + CompiledObjectKey, + CompiledSqlTest, CompiledSqlTestResource, CompileSqlTestCte, CustomRulesOutcome, DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, + Finding, NativeRulesEvaluation, NativeRulesRequest, + Rule, RuleExemption, RuleIgnore, RulesCacheConfig, + RulesConfig, + RulesResult, + ScopeIndex, build_rule_catalogue, install_rules_skills, project_with, ) from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestFile -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index d82902f8a9..81a9a6ec02 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -5,7 +5,10 @@ import pytest from sqlbuild._native import ( + Rule, RulesCacheConfig, + RulesConfig, + RulesResult, SqlTestRulesConfig, ThresholdOverride, build_rule_catalogue, @@ -14,7 +17,6 @@ select_catalogue_rules, ) from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( PolicyGuidanceTestCase, PolicySelectionTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 38255e913c..adc5dd4395 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -4,9 +4,14 @@ import pytest -from sqlbuild._native import SqlTestRulesConfig, load_rules_config -from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError -from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig +from sqlbuild._native import ( + LayoutConfig, + RulesConfig, + RulesConfigError, + RulesError, + SqlTestRulesConfig, + load_rules_config, +) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( AllowExceptionsConfigTestCase, PolicyLayoutConfigTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index e8bd7e90fe..1a66c58c6f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -5,14 +5,16 @@ import pytest from sqlbuild._native import ( + Rule, + RulesConfig, + RulesError, + RulesResult, build_rule_catalogue, custom_rule_implementation_fingerprint, custom_rule_test_evidence, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleEvidenceTestCase, CustomRuleTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 4f65dae842..a32df2bf26 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -6,16 +6,17 @@ import pytest from sqlbuild._native import ( + CompactLineageFacts, RuleExemption, RuleIgnore, + RulesConfig, + RulesError, + RulesResult, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import CompactLineageFacts from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleCacheDependencyTestCase, CustomRuleSuppressionTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py index bedcf29e3e..8bf3473c5a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_definition.py @@ -4,12 +4,11 @@ import pytest +from sqlbuild._native import Rule, RulesError from sqlbuild.rule_engine._helpers.engine.definition import ( resolve_rule_signature, rule_from_value, ) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Rule from sqlbuild.rules import Finding, Model, Project, RuleContext, rule from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( RuleCodeTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py index 35ccfe213f..deee74da56 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_hermeticity.py @@ -4,9 +4,7 @@ import pytest -from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine.exceptions import NonHermeticRuleError -from sqlbuild.rule_engine.models import Rule +from sqlbuild._native import NonHermeticRuleError, Rule, verify_custom_rules from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleImportTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py index 40c3583455..5e03cc34d8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_incremental_native_rules.py @@ -3,8 +3,7 @@ import pytest -from sqlbuild._native import RuleExemption, RuleIgnore -from sqlbuild.rule_engine.models import RulesResult +from sqlbuild._native import RuleExemption, RuleIgnore, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( IncrementalCacheCountsTestCase, IncrementalRulesModeTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index fab374960c..92cac5b2f2 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -4,17 +4,13 @@ import pytest from sqlbuild._native import ( - project_with, -) -from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlScenario, CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, + project_with, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( NativeFactPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py index e66b18c738..8662e1d417 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.rule_engine.models import RulesResult +from sqlbuild._native import RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import NativeMemoTestCase from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py index c35037745b..50a7f3336b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_scope_facts.py @@ -5,8 +5,12 @@ import pytest -from sqlbuild._native import scope_metadata_projection -from sqlbuild.compiler.scopes.models import DeclarationRecord, ScopeCompleteness, ScopeIndex +from sqlbuild._native import ( + DeclarationRecord, + ScopeCompleteness, + ScopeIndex, + scope_metadata_projection, +) from tests.unit.src.sqlbuild.compiler.scopes.helpers import scope_index from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ScopePayloadTestCase from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index 8719db8a69..28a64c9b6b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -4,10 +4,10 @@ import pytest from sqlbuild._native import ( + ConstantDeclaration, declaration_rows, normalize_sql_value, ) -from sqlbuild.compiler.discovery.models import ConstantDeclaration from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( TypedConstantPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index bbdb0ebc62..6b9654ef5c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -18,7 +18,10 @@ from sqlbuild._native import ( CustomHostObserver, CustomRulesOutcome, + OpaqueModuleStateError, + Rule, RulesCacheConfig, + RulesConfig, build_rule_catalogue, project_with, ) @@ -26,8 +29,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import module_state_token, rule_namespaces from sqlbuild.rule_engine.classes.module_state_monitor import ModuleStateMonitor -from sqlbuild.rule_engine.exceptions import OpaqueModuleStateError -from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project _MODEL_RULE: str = "XSQBRT101" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 528da9bb3f..7f2dd7196c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -5,11 +5,15 @@ import pytest -from sqlbuild._native import CustomHostObserver, CustomRulesOutcome, rules_bulk_cache_path +from sqlbuild._native import ( + CustomHostObserver, + CustomRulesOutcome, + HostCancelledError, + RulesError, + rules_bulk_cache_path, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host -from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE -from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from tests.unit.src.sqlbuild.rule_engine._helpers.host._test_types import ( AwaitInputsTestCase, AwaitPublishedInputsTestCase, @@ -78,9 +82,7 @@ def test_given_partitioned_hosts_when_evaluating_then_findings_and_cache_match_o detects_duplicates=test_case.detects_duplicates, ) project: CompiledProjectView = orders_project(model_count=test_case.model_count) - cache_path: Path = rules_bulk_cache_path( - project_dir=tmp_path, file_name=CUSTOM_RULES_CACHE_FILE - ) + cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="custom-rules.json") cache_glob: str = cache_path.relative_to(tmp_path).as_posix() cache_enabled: bool = test_case.cache_enabled diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py index 6036adc7b3..2718919e37 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_module_state.py @@ -48,6 +48,18 @@ model_count=4, expected_stateful=_STATELESS, ), + ModuleStateCase( + description="module-level rule option record", + files=state_case_files( + header=( + "from sqlbuild.rules import RuleOption\n" + "LIMIT = RuleOption.integer(name='limit', default=3, description='Limit.')\n" + ), + body="_ = LIMIT.default", + ), + model_count=4, + expected_stateful=_STATELESS, + ), ModuleStateCase( description="dict mutation", files=state_case_files( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py index 03ec9b4be7..3e1d2306a4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -5,10 +5,8 @@ import pytest import sqlbuild._native as native -from sqlbuild._native import RulesCacheConfig +from sqlbuild._native import RulesCacheConfig, RulesConfig, RulesError, RulesResult from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( NativeRowsEncodeErrorTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py index 52e4b5abfa..d3ce2e7515 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py @@ -1,7 +1,11 @@ from pathlib import Path -from sqlbuild._native import DiscoveredAuditBlock, DiscoveredAuditFile -from sqlbuild.compiler.compile.models import CompiledAudit, CompiledObjectKey +from sqlbuild._native import ( + CompiledAudit, + CompiledObjectKey, + DiscoveredAuditBlock, + DiscoveredAuditFile, +) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py index f48e681c8e..633f04db9d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py @@ -5,11 +5,11 @@ import pytest from sqlbuild._native import ( + Model, project_with, ) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts -from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine.classes.audit_facts._test_types import ( AuditFactsTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py index 2db8ccba43..ade7e7839a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py @@ -6,12 +6,12 @@ import pytest from sqlbuild._native import ( + CompiledSqlTest, + Model, project_with, ) -from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.rule_engine.classes.model_test_facts import TestFacts as ModelTestFacts -from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( direct_sql_test, model_sql_test, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py index 99914f928d..16a26c7e9f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/_test_types.py @@ -7,3 +7,19 @@ class ReadTextTestCase: path: str expected_text: str | None = None expected_error: str = "" + + +@dataclass(frozen=True) +class TreeQueryTestCase: + description: str + argument: str + expected_values: tuple[str, ...] + + +@dataclass(frozen=True) +class RelativePartsTestCase: + description: str + path: str + under: str + expected_parts: tuple[str, ...] | None = None + expected_error: str = "" diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py new file mode 100644 index 0000000000..d841596fc6 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_queries.py @@ -0,0 +1,152 @@ +"""Project-tree queries select discovered paths exactly as pathlib and fnmatch would.""" + +from pathlib import Path + +import pytest + +from sqlbuild._native import RuleUsageError +from sqlbuild.rule_engine.classes.project_tree import ProjectTree +from tests.unit.src.sqlbuild.rule_engine.classes.project_tree._test_types import ( + RelativePartsTestCase, + TreeQueryTestCase, +) +from tests.unit.src.sqlbuild.rule_engine.classes.project_tree.helpers import linked_project_tree + + +@pytest.mark.parametrize( + "test_case", + ( + TreeQueryTestCase("project root", "", ("models", "rules")), + TreeQueryTestCase("current directory", ".", ("models", "rules")), + TreeQueryTestCase( + "folder with trailing slash", "models/", ("models/notes.txt", "models/orders.sql") + ), + TreeQueryTestCase("nested folder", "./rules//inputs", ("rules/inputs/limits.yaml",)), + TreeQueryTestCase("missing folder", "missing", ()), + ), + ids=lambda case: case.description, +) +def test_given_directory_when_listing_children_then_returns_direct_entries( + tmp_path: Path, test_case: TreeQueryTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + values: tuple[str, ...] = tuple(node.value for node in tree.children(test_case.argument)) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + ( + TreeQueryTestCase( + "everything", + "", + ( + "models", + "models/notes.txt", + "models/orders.sql", + "rules", + "rules/inputs", + "rules/inputs/limits.yaml", + ), + ), + TreeQueryTestCase("folder", "rules/", ("rules/inputs", "rules/inputs/limits.yaml")), + TreeQueryTestCase("current directory spelled out", ".", ()), + ), + ids=lambda case: case.description, +) +def test_given_directory_when_listing_descendants_then_returns_entries_below_it( + tmp_path: Path, test_case: TreeQueryTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + values: tuple[str, ...] = tuple(node.value for node in tree.descendants(test_case.argument)) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + ( + TreeQueryTestCase("star crosses folders", "*.sql", ("models/orders.sql",)), + TreeQueryTestCase("single character", "rule?", ("rules",)), + TreeQueryTestCase("negated set", "[!m]*s", ("rules", "rules/inputs")), + TreeQueryTestCase("no match", "*.py", ()), + ), + ids=lambda case: case.description, +) +def test_given_pattern_when_globbing_then_returns_fnmatch_matches( + tmp_path: Path, test_case: TreeQueryTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + values: tuple[str, ...] = tuple(node.value for node in tree.glob(test_case.argument)) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + ( + RelativePartsTestCase("relative path", "models/orders.sql", "", ("models", "orders.sql")), + RelativePartsTestCase("beneath a folder", "models/orders.sql", "models/", ("orders.sql",)), + ), + ids=lambda case: case.description, +) +def test_given_path_when_reading_relative_parts_then_returns_project_relative_parts( + tmp_path: Path, test_case: RelativePartsTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + parts: tuple[str, ...] = tree.relative_parts(path=test_case.path, under=test_case.under) + + assert parts == test_case.expected_parts + + +@pytest.mark.parametrize( + "test_case", + ( + RelativePartsTestCase( + "through a linked folder", + "orders/linked_models/orders.sql", + "", + ("models", "orders.sql"), + ), + RelativePartsTestCase( + "beneath a folder", "orders/models/orders.sql", "models", ("orders.sql",) + ), + ), + ids=lambda case: case.description, +) +def test_given_absolute_path_inside_project_when_reading_relative_parts_then_strips_the_root( + tmp_path: Path, test_case: RelativePartsTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + parts: tuple[str, ...] = tree.relative_parts( + path=tmp_path / test_case.path, under=test_case.under + ) + + assert parts == test_case.expected_parts + + +@pytest.mark.parametrize( + "test_case", + ( + RelativePartsTestCase("not beneath", "models/orders.sql", "rules", None, "is not beneath"), + RelativePartsTestCase("absolute outside", "/", "", None, "escapes the project"), + ), + ids=lambda case: case.description, +) +def test_given_path_outside_scope_when_reading_relative_parts_then_raises_usage_error( + tmp_path: Path, test_case: RelativePartsTestCase +) -> None: + tree: ProjectTree = linked_project_tree(tmp_path) + + with pytest.raises(RuleUsageError, match=test_case.expected_error): + tree.relative_parts(path=test_case.path, under=test_case.under) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py index addaa83265..df5918535e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/test_read_text.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import RuleUsageError from sqlbuild.rule_engine.classes.project_tree import ProjectTree -from sqlbuild.rule_engine.exceptions import RuleUsageError from tests.unit.src.sqlbuild.rule_engine.classes.project_tree._test_types import ReadTextTestCase from tests.unit.src.sqlbuild.rule_engine.classes.project_tree.helpers import ( LIMITS_YAML, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index ed377026b0..ab577d63c1 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -1,18 +1,18 @@ from pathlib import Path from sqlbuild._native import ( + CompiledObjectKey, CompiledSource, + DiscoveredSourceFile, + SourceEntry, project_with, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views -from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project SOURCES_CONTENTS: str = "sources:\n - name: raw_orders\n - name: raw_customers\n" diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py index dda4c72d3f..7383671c22 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/_test_types.py @@ -2,8 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.rule_engine.models import RulesConfig +from sqlbuild._native import CompileSqlReference, RulesConfig @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index 2d882a3330..4c888d539f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -2,20 +2,21 @@ from pathlib import Path -from sqlbuild._native import CompileModelConfig, EnumMember -from sqlbuild.compiler.compile.models import ( +from sqlbuild._native import ( CompiledModel, CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompileModelConfig, CompileSqlReference, + EnumDeclaration, + EnumMember, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import EnumDeclaration _ENUM_DECLARATION: EnumDeclaration = EnumDeclaration( name="status", diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index 350169e5c8..0142313abe 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -8,13 +8,14 @@ from sqlbuild._native import ( RuleExemption, RuleIgnore, + RulesConfig, + RulesError, + RulesResult, compiled_model_record, project_with, ) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import PolicyBehaviorTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py index 15e0370529..eae4595e5b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_evaluate.py @@ -5,10 +5,8 @@ import pytest -from sqlbuild._native import SelectStarAllow +from sqlbuild._native import CompileSqlReference, Finding, RulesConfig, RulesResult, SelectStarAllow from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.rule_engine.models import Finding, RulesConfig, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.evaluate._test_types import ( PolicyEvaluationTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index 897f0cf4cc..7b5140b457 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -6,12 +6,15 @@ import pytest -from sqlbuild._native import RulesCacheConfig +from sqlbuild._native import ( + MIN_CUSTOM_RULE_TEST_CASES, + RulesCacheConfig, + RulesConfig, + RulesError, + RulesResult, +) from sqlbuild._native import evaluate_project_rules as evaluate from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesConfig, RulesResult from sqlbuild.rule_engine.types import RuleOptionValue from sqlbuild.rules.testing import RuleCase, RuleResult, evaluate_rule from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py index 45254a34f8..339e9ae891 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/render_result/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.rule_engine.models import RulesResult +from sqlbuild._native import RulesResult @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py index e2ac06ade3..c526a87aa8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import Finding, RulesResult from sqlbuild._native import format_rules_result as format_result -from sqlbuild.rule_engine.models import Finding, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.render_result._test_types import ( RenderResultParityTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 090b25dd4f..12b78fb7ac 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, + ProjectGraph, RulesRunResult, build_project_graph, discover_project_inputs, @@ -13,8 +15,6 @@ run_rules, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.rule_engine.main.run_rules._test_types import ( TypeProofRuleSkipTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index 33f78a0237..bdccd3cc09 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -5,10 +5,8 @@ import pytest -from sqlbuild._native import RuleExemption +from sqlbuild._native import RuleExemption, RulesConfig, RulesError from sqlbuild._native import install_rules_skills as install_skills -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( SkillDivergenceTestCase, SkillFreshnessTestCase, diff --git a/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py b/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py index f35e26dba5..38fc3c29a7 100644 --- a/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py +++ b/tests/unit/src/sqlbuild/runtime/compute_logs/classes/test_scoped_compute_log_capture.py @@ -10,14 +10,13 @@ import pytest -from sqlbuild._native import LocalFilesystemComputeLogStorage +from sqlbuild._native import SQL_DIGEST_FIELD, LocalFilesystemComputeLogStorage from sqlbuild.compute_logs import ( CaptureMetadata, CaptureStateError, ComputeLogStream, FinalCaptureMetadata, ) -from sqlbuild.diagnostics.constants import SQL_DIGEST_FIELD from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.diagnostics.models import DiagnosticRoutingOptions diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py index 6ce4b02087..af644e7136 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py @@ -3,11 +3,12 @@ from threading import Event from typing import ClassVar -from sqlbuild._native import BoundEventExporter, EventExporterDispatcher -from sqlbuild.compiler.discovery.models import ( +from sqlbuild._native import ( + BoundEventExporter, DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, + EventExporterDispatcher, ) from sqlbuild.observability import LifecycleEvent from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py index d509b8f65a..4f40adb060 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py @@ -8,15 +8,13 @@ import pytest from sqlbuild._native import ( - EventExporterCommandScope, - EventExporterDispatcher, - EventExportSummary, -) -from sqlbuild.compiler.discovery.models import ( DiscoveredCommandOutputSink, DiscoveredEventExporter, DiscoveredProvider, DiscoveredProviderUsage, + EventExporterCommandScope, + EventExporterDispatcher, + EventExportSummary, ) from sqlbuild.providers import Provider from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index d46ed5d96d..0135132e6e 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -9,17 +9,17 @@ import pytest from sqlbuild._native import ( + COMMAND_OUTPUT_LOSS_RECORD_TYPE, BoundEventExporter, EventExporterCounts, EventExporterDispatcher, + EventExportSeverity, EventExportSummary, ) from sqlbuild.observability import LifecycleEvent from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError -from sqlbuild.runtime.output_capture.constants import COMMAND_OUTPUT_LOSS_RECORD_TYPE from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream -from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterDispatcherTestCase, HealthIntervalTestCase, diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py new file mode 100644 index 0000000000..6faca8e083 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/helpers.py @@ -0,0 +1,29 @@ +"""Shared event export test data.""" + +from __future__ import annotations + +CATALOGUED_EVENT_TYPES: tuple[str, ...] = ( + "invocation_started", + "invocation_completed", + "invocation_failed", + "run_started", + "run_completed", + "run_failed", + "resource_attempt_started", + "resource_attempt_completed", + "resource_attempt_failed", + "resource_attempt_skipped", + "operation_started", + "operation_completed", + "operation_failed", + "retry_scheduled", + "audit_completed", + "statement_started", + "statement_submitted", + "statement_heartbeat", + "statement_completed", + "statement_failed", + "microbatch_started", + "microbatch_completed", + "microbatch_failed", +) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py index 4f82779a20..dab0c4b482 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/test_policy.py @@ -1,23 +1,23 @@ import pytest from sqlbuild._native import ( + EventExportSeverity, event_export_severity_at_least, lifecycle_export_policy_catalog, stricter_event_export_severity, ) from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS from sqlbuild.runtime.event_exporting.types import LifecycleEventKind -from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOG -from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting._test_types import ( EventExportPolicyTestCase, SeverityNarrowingTestCase, ) +from tests.unit.src.sqlbuild.runtime.event_exporting.helpers import CATALOGUED_EVENT_TYPES @pytest.mark.parametrize( "test_case", - (EventExportPolicyTestCase("complete canonical mapping", len(LIFECYCLE_EVENT_CATALOG)),), + (EventExportPolicyTestCase("complete canonical mapping", len(CATALOGUED_EVENT_TYPES)),), ids=lambda case: case.description, ) def test_given_canonical_catalog_when_building_export_policy_then_every_type_is_mapped_once( @@ -25,7 +25,7 @@ def test_given_canonical_catalog_when_building_export_policy_then_every_type_is_ ) -> None: policy: dict[str, tuple[str, EventExportSeverity, int]] = lifecycle_export_policy_catalog() assert len(policy) == test_case.expected_event_count - assert policy.keys() == LIFECYCLE_EVENT_CATALOG.keys() + assert tuple(policy) == CATALOGUED_EVENT_TYPES assert {kind for kind, _, _ in policy.values()} == EVENT_EXPORT_KINDS assert policy["invocation_failed"] == ("invocation", EventExportSeverity.ERROR, 3) assert policy["resource_attempt_skipped"] == ("resource", EventExportSeverity.INFO, 2) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py b/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py index b7a1e8da10..9153782600 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/test_sinks.py @@ -5,6 +5,7 @@ import pytest +from sqlbuild._native import EventExportSeverity from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError from sqlbuild.sinks import ( CommandOutputSinkDefinition, @@ -16,7 +17,6 @@ get_lifecycle_event_sink_definition, lifecycle_event_sink, ) -from sqlbuild.spec.contracts.types import EventExportSeverity from tests.unit.src.sqlbuild.runtime.event_exporting._test_types import SinkApiTestCase diff --git a/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py b/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py index a17610d6bb..42a8d417e3 100644 --- a/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py +++ b/tests/unit/src/sqlbuild/runtime/execution_history/conformance/conftest.py @@ -16,6 +16,11 @@ import pytest +from sqlbuild._native import ( + CURRENT_EVENT_LOG_SCHEMA_VERSION, + CURRENT_RUN_STORAGE_SCHEMA_VERSION, + DEFAULT_PAGE_LIMIT, +) from sqlbuild.execution_history import ( CanonicalLifecycleEvent, EventFilter, @@ -35,11 +40,6 @@ project_runs, validate_page_limit, ) -from sqlbuild.runtime.execution_history.constants import ( - CURRENT_EVENT_LOG_SCHEMA_VERSION, - CURRENT_RUN_STORAGE_SCHEMA_VERSION, - DEFAULT_PAGE_LIMIT, -) from sqlbuild.runtime.observability.models import LifecycleEvent, OpaqueLifecycleEvent from sqlbuild.runtime.observability.types import JSONValue from sqlbuild.sqlite_history import SQLiteExecutionHistory diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py b/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py index 52e65f7816..e14d16718b 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_catalog.py @@ -5,9 +5,13 @@ import pytest +from sqlbuild._native import ( + LIFECYCLE_EVENT_SCHEMA_VERSIONS, + lifecycle_event_catalogued, + lifecycle_event_terminal, +) from sqlbuild.observability import create_lifecycle_event, invocation_scope, run_scope from sqlbuild.runtime.observability.classes.run_lifecycle import RunLifecycle -from sqlbuild.runtime.observability.constants import LIFECYCLE_EVENT_CATALOGS from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( canonicalize_operation_adapter, @@ -398,10 +402,12 @@ def test_given_catalog_when_looking_up_events_then_authority_is_version_dimensio test_case: CatalogVersionCase, ) -> None: assert ( - LIFECYCLE_EVENT_CATALOGS[test_case.schema_version][test_case.event_type].terminal + lifecycle_event_terminal( + schema_version=test_case.schema_version, event_type=test_case.event_type + ) is test_case.expected_terminal ) - assert test_case.expected_unsupported_version not in LIFECYCLE_EVENT_CATALOGS + assert test_case.expected_unsupported_version not in LIFECYCLE_EVENT_SCHEMA_VERSIONS @pytest.mark.parametrize( @@ -426,8 +432,11 @@ def test_given_microbatch_events_when_inspecting_versioned_catalog_then_v1_remai test_case: CatalogEventPresenceCase, ) -> None: assert ( - test_case.event_type in LIFECYCLE_EVENT_CATALOGS[test_case.schema_version] - ) is test_case.expected_present + lifecycle_event_catalogued( + schema_version=test_case.schema_version, event_type=test_case.event_type + ) + is test_case.expected_present + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_models.py b/tests/unit/src/sqlbuild/runtime/observability/test_models.py index 25e67c29bf..a98a51c61c 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_models.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_models.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.runtime.observability._helpers.validation import freeze_json +from sqlbuild._native import freeze_lifecycle_json from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.models import DiagnosticLog, LifecycleEvent from tests.unit.src.sqlbuild.runtime.observability._test_types import ( @@ -173,7 +173,7 @@ def test_given_invalid_nested_payload_when_freezing_then_error_names_full_path( test_case: FreezeJsonErrorCase, ) -> None: with pytest.raises(ObservabilityValidationError) as error_info: - freeze_json(value=test_case.payload, path="payload") + freeze_lifecycle_json(value=test_case.payload, path="payload") assert str(error_info.value) == test_case.expected_error diff --git a/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py b/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py index 19a5031ccd..fd5de7c766 100644 --- a/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py +++ b/tests/unit/src/sqlbuild/runtime/output_capture/main/test_invocation_context.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import invocation_context_from_environment -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV +from sqlbuild._native import INVOCATION_CONTEXT_ENV, invocation_context_from_environment from sqlbuild.runtime.output_capture.exceptions import OutputCaptureInputError from tests.unit.src.sqlbuild.runtime.output_capture.main._test_types import ( InvocationContextTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index a9cb661203..28a6afdfd6 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -2,17 +2,17 @@ from dataclasses import dataclass, field -from sqlbuild._native import CollectionRendering -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( AuthoredTimeTravelRetention, + CollectionRendering, ExecutionLimitsConfig, LoaderDestinationParts, LocalConfig, + MissingMigrationOriginPolicy, ProjectConfig, TargetConfig, TargetWarehousesConfig, ) -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py index 641d236bfd..485c0c1cb3 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_constants.py @@ -2,8 +2,12 @@ import pytest -from sqlbuild._native import CollectionRendering, resolve_effective_collection_rendering -from sqlbuild.spec.contracts.models import ConstantsConfig, ProjectConfig +from sqlbuild._native import ( + CollectionRendering, + ConstantsConfig, + ProjectConfig, + resolve_effective_collection_rendering, +) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( EffectiveCollectionRenderingResolutionTestCase, ) diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py index 7001a5e9e4..f9bc291b6e 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_loader_destination_parts.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import SpecConfigError, loader_destination_parts -from sqlbuild.spec.contracts.models import LoaderDestinationParts +from sqlbuild._native import LoaderDestinationParts, SpecConfigError, loader_destination_parts from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidLoaderDestinationTestCase, LoaderDestinationPartsTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py index 9c55b71425..1fc5a111ee 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import ( + DiscoveredProjectInputs, ProjectConfigError, discover_project_inputs, resolve_effective_scenario_config, ) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( InvalidScenarioNamespaceConfigTestCase, ScenarioNamespaceConfigTestCase, diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index 88594ac3da..b1fc909c0f 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -2,23 +2,22 @@ import pytest -from sqlbuild._native import SpecConfigError, resolve_target_config -from sqlbuild.spec.contracts.models import ( +from sqlbuild._native import ( AuthoredTimeTravelRetention, ClonePolicy, ExecutionLimitsConfig, LocalConfig, LocalTargetConfig, - ProjectConfig, - TargetConfig, - TargetDiffConfig, - TargetWarehousesConfig, -) -from sqlbuild.spec.contracts.types import ( MissingMigrationOriginPolicy, + ProjectConfig, RetentionDecreasePolicy, + SpecConfigError, TableType, TableTypeDowngradePolicy, + TargetConfig, + TargetDiffConfig, + TargetWarehousesConfig, + resolve_target_config, ) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( AmbiguousTargetConnectionTestCase, diff --git a/tests/unit/src/sqlbuild/sql_values/_test_types.py b/tests/unit/src/sqlbuild/sql_values/_test_types.py index b79e8082f9..5ca8456e01 100644 --- a/tests/unit/src/sqlbuild/sql_values/_test_types.py +++ b/tests/unit/src/sqlbuild/sql_values/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild._native import SqlValueLimits +from sqlbuild._native import SqlValueLimits, StateSqlValueType @dataclass(frozen=True) @@ -38,3 +38,16 @@ class SqlValueBehaviorTestCase: class StateLiteralGoldenTestCase: description: str expected_literals: dict[str, str] + + +@dataclass(frozen=True) +class StateSqlGoldenTestCase: + description: str + expected_sql: str + + +@dataclass(frozen=True) +class StateSqlTypeErrorTestCase: + description: str + column_type: StateSqlValueType + expected_error: type[Exception] diff --git a/tests/unit/src/sqlbuild/sql_values/test_normalization.py b/tests/unit/src/sqlbuild/sql_values/test_normalization.py index 3796531f8b..2be94bc7e1 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_normalization.py +++ b/tests/unit/src/sqlbuild/sql_values/test_normalization.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( AuthoredSqlSet, + SqlValue, SqlValueLimits, SqlValueValidationError, normalize_sql_value, ) -from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.sql_values._test_types import ( NormalizeSqlValueErrorTestCase, NormalizeSqlValueTestCase, diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py index 7f6f3a5932..75d1b3124e 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_state_literal.py +++ b/tests/unit/src/sqlbuild/sql_values/test_state_literal.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import render_state_sql_literal -from sqlbuild.sql_values.types import StateSqlValueType +from sqlbuild._native import StateSqlValueType, render_state_sql_literal from tests.unit.src.sqlbuild.sql_values._test_types import StateLiteralGoldenTestCase diff --git a/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py new file mode 100644 index 0000000000..74e61e3e14 --- /dev/null +++ b/tests/unit/src/sqlbuild/sql_values/test_state_table_sql.py @@ -0,0 +1,120 @@ +"""Golden SQL tests for native append-only state table rendering.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from sqlbuild._native import ( + SourceFreshnessRecord, + StateSqlValueType, + render_insert_source_freshness_records_sql, + render_state_table_create_sql, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.unit.src.sqlbuild.sql_values._test_types import ( + StateSqlGoldenTestCase, + StateSqlTypeErrorTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StateSqlGoldenTestCase( + description="representative source freshness insert", + expected_sql=( + "INSERT INTO analytics._sqlbuild_source_freshness (source_name, " + "target_database, target_schema, target_name, run_id, strategy, value_kind, " + "data_version, data_version_hash, observed_at) VALUES ('raw.o''rders', NULL, " + "'raw', 'orders', 'run-1', 'adapter', 'timestamp', " + "'2026-01-01T01:00:00+01:00', 'hash', '2026-01-01T00:00:00+00:00')" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_representative_freshness_record_when_rendering_then_sql_matches_golden( + test_case: StateSqlGoldenTestCase, +) -> None: + sql: str = render_insert_source_freshness_records_sql( + database=None, + schema="analytics", + records=( + SourceFreshnessRecord( + source_name="raw.o'rders", + target_database=None, + target_schema="raw", + target_name="orders", + run_id="run-1", + strategy="adapter", + value_kind="timestamp", + data_version="2026-01-01T01:00:00+01:00", + data_version_hash="hash", + observed_at=datetime(2026, 1, 1, 1, tzinfo=timezone(timedelta(hours=1))), + ), + ), + render_qualified_name=DuckDbAdapter().render_qualified_name, + ) + + assert sql == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + StateSqlGoldenTestCase( + description="string integer and text timestamp columns with required columns", + expected_sql=( + "CREATE TABLE IF NOT EXISTS main.state " + "(name VARCHAR NOT NULL, row_count BIGINT, observed_at TIMESTAMP NOT NULL)" + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_state_columns_when_rendering_table_ddl_then_sql_matches_golden( + test_case: StateSqlGoldenTestCase, +) -> None: + sql: str = render_state_table_create_sql( + qualified_name="main.state", + columns=("name", "row_count", "observed_at"), + column_types={ + "name": StateSqlValueType.STRING, + "row_count": StateSqlValueType.INTEGER, + "observed_at": StateSqlValueType.TEXT_TIMESTAMP, + }, + required_columns=frozenset({"name", "observed_at"}), + render_framework_type=DuckDbAdapter().render_framework_type, + ) + + assert sql == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + StateSqlTypeErrorTestCase( + description="boolean columns are not state table columns", + column_type=StateSqlValueType.BOOLEAN, + expected_error=KeyError, + ) + ], + ids=lambda case: case.description, +) +def test_given_unsupported_column_type_when_rendering_table_ddl_then_raises_error( + test_case: StateSqlTypeErrorTestCase, +) -> None: + with pytest.raises(test_case.expected_error): + render_state_table_create_sql( + qualified_name="main.state", + columns=("flag",), + column_types={"flag": test_case.column_type}, + required_columns=frozenset(), + render_framework_type=DuckDbAdapter().render_framework_type, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])