diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 383892f92e..7046ea142a 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,9 @@ pub mod lineage_command; pub mod lineage_graph; pub mod manifest; pub mod planner_changes; +pub mod planner_identity; +pub mod planner_scope; +pub mod planner_selection; pub mod planner_strategy; pub mod python_plan; pub mod python_selection; @@ -33,4 +36,5 @@ pub mod semantic_usage; pub mod semantic_validation; pub mod sql_test_attachment; pub mod sql_tokens; +pub mod target_paths; pub mod type_system; diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs new file mode 100644 index 0000000000..a8e54ec089 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs @@ -0,0 +1,13 @@ +//! A node identity composed from its local hash and the hashes of its available upstreams. + +use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; + +/// The hash of the local hash and each upstream hash, one per line. +pub(crate) fn compose_graph_identity(local_hash: &str, upstream_hashes: &[&str]) -> String { + let mut text: String = local_hash.to_owned(); + for upstream_hash in upstream_hashes { + text.push('\n'); + text.push_str(upstream_hash); + } + stable_identity_hash(&text) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs new file mode 100644 index 0000000000..bf476b3d2a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs @@ -0,0 +1,65 @@ +//! The dependency-cycle message naming the keys left unordered and the injected edges between them. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_identity::models::GraphKey; + +/// Upstream edges and the origins of injected edges. +type CycleEdges<'a> = ( + &'a [(GraphKey, Vec)], + &'a HashMap<(GraphKey, GraphKey), String>, +); + +/// The message for the keys Kahn's algorithm could not order. +pub(crate) fn cycle_message( + ordered: &[GraphKey], + node_keys: HashSet<&GraphKey>, + (upstream, injected_edge_origins): CycleEdges<'_>, +) -> String { + let ordered_set: HashSet<&GraphKey> = ordered.iter().collect(); + let mut cycle_keys: Vec<&GraphKey> = node_keys + .into_iter() + .filter(|key| !ordered_set.contains(key)) + .collect(); + cycle_keys.sort(); + let names: Vec = cycle_keys + .iter() + .map(|key| format!("{}:{}", key.kind, key.name)) + .collect(); + format!( + "Dependency cycle detected involving: {}{}", + names.join(", "), + cycle_origin_notes(&cycle_keys, upstream, injected_edge_origins) + ) +} + +fn cycle_origin_notes( + cycle_keys: &[&GraphKey], + upstream: &[(GraphKey, Vec)], + injected_edge_origins: &HashMap<(GraphKey, GraphKey), String>, +) -> String { + if injected_edge_origins.is_empty() { + return String::new(); + } + let cycle_set: HashSet<&GraphKey> = cycle_keys.iter().copied().collect(); + let deps: HashMap<&GraphKey, &Vec> = + upstream.iter().map(|(key, deps)| (key, deps)).collect(); + let mut notes: BTreeSet<&str> = BTreeSet::new(); + for key in cycle_keys { + for dep_key in deps.get(key).into_iter().flat_map(|deps| deps.iter()) { + if !cycle_set.contains(dep_key) { + continue; + } + if let Some(origin) = injected_edge_origins.get(&((*key).clone(), dep_key.clone())) { + notes.insert(origin); + } + } + } + if notes.is_empty() { + return String::new(); + } + format!( + " (via {})", + notes.into_iter().collect::>().join("; ") + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs new file mode 100644 index 0000000000..ff60300336 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs @@ -0,0 +1,35 @@ +//! Kahn's algorithm inputs: every key, each key's upstream count and its downstream keys. + +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::models::GraphKey; + +/// Every key named in the edges, its number of upstream entries and its downstream keys, one per edge. +pub(crate) struct DependencyCounts<'a> { + pub(crate) node_keys: HashSet<&'a GraphKey>, + pub(crate) indegree: HashMap<&'a GraphKey, usize>, + pub(crate) downstream: HashMap<&'a GraphKey, Vec<&'a GraphKey>>, +} + +/// The counts of `upstream`, counting repeated upstream entries once per entry. +pub(crate) fn dependency_counts(upstream: &[(GraphKey, Vec)]) -> DependencyCounts<'_> { + let mut counts = DependencyCounts { + node_keys: HashSet::new(), + indegree: HashMap::new(), + downstream: HashMap::new(), + }; + for (key, dep_keys) in upstream { + counts.node_keys.insert(key); + for dep_key in dep_keys { + counts.node_keys.insert(dep_key); + counts.downstream.entry(dep_key).or_default().push(key); + } + } + for key in &counts.node_keys { + counts.indegree.insert(key, 0); + } + for (key, dep_keys) in upstream { + counts.indegree.insert(key, dep_keys.len()); + } + counts +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs new file mode 100644 index 0000000000..ca922cbe9e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs @@ -0,0 +1,169 @@ +//! Memoized identity resolution over one dependency graph, each node once, with an explicit stack. + +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::_helpers::compose_graph_identity::compose_graph_identity; +use crate::planner_identity::models::{GraphKey, IdentityNode}; +use crate::planner_identity::types::IdentityHashes; + +/// Which hashes a resolution composes from. +#[derive(Clone, Copy)] +enum Mode { + Expected, + Write, +} + +/// A node whose upstream hashes are being resolved. +struct Frame<'a> { + key: &'a GraphKey, + node: &'a IdentityNode, + local_hash: &'a str, + next_upstream: usize, + upstream_hashes: Vec, +} + +/// What entering a node yields: its hash already, or a frame to resolve. +enum Entered<'a> { + Resolved(Option), + Pending(Frame<'a>), +} + +/// Expected or selected-write identity resolution state. +pub(crate) struct IdentityResolver<'a> { + nodes: &'a HashMap, + selected: Option<&'a HashSet>, + hashes: IdentityHashes, + positions: HashMap, + resolved_selected: HashMap, + visiting: HashSet, +} + +impl<'a> IdentityResolver<'a> { + pub(crate) fn new( + nodes: &'a HashMap, + selected: Option<&'a HashSet>, + base_hashes: Vec<(GraphKey, String)>, + ) -> Self { + let mut resolver = Self { + nodes, + selected, + hashes: Vec::new(), + positions: HashMap::new(), + resolved_selected: HashMap::new(), + visiting: HashSet::new(), + }; + for (key, hash) in base_hashes { + resolver.record(&key, Some(hash)); + } + resolver + } + + /// Every hash recorded so far, in first-recorded order. + pub(crate) fn into_hashes(self) -> IdentityHashes { + self.hashes + } + + /// A node's expected hash, composed from its upstream expected hashes. + pub(crate) fn resolve_expected(&mut self, key: &'a GraphKey) -> Option { + self.resolve(key, Mode::Expected) + } + + /// A selected node's hash recomputed from upstream write hashes; other nodes keep theirs. + pub(crate) fn resolve_write(&mut self, key: &'a GraphKey) -> Option { + self.resolve(key, Mode::Write) + } + + fn resolve(&mut self, root: &'a GraphKey, mode: Mode) -> Option { + let mut stack: Vec> = match self.enter(root, mode) { + Entered::Resolved(hash) => return hash, + Entered::Pending(frame) => vec![frame], + }; + while let Some(frame) = stack.last_mut() { + if let Some(upstream_key) = frame.node.upstream_keys.get(frame.next_upstream) { + frame.next_upstream += 1; + match self.enter(upstream_key, mode) { + Entered::Resolved(hash) => frame.upstream_hashes.extend(hash), + Entered::Pending(upstream_frame) => stack.push(upstream_frame), + } + continue; + } + let Some(finished) = stack.pop() else { + break; + }; + let composed: String = self.finish(&finished, mode); + match stack.last_mut() { + Some(parent) => parent.upstream_hashes.push(composed), + None => return Some(composed), + } + } + None + } + + fn enter(&mut self, key: &'a GraphKey, mode: Mode) -> Entered<'a> { + if let Mode::Write = mode { + if !self.selected.is_some_and(|selected| selected.contains(key)) { + return Entered::Resolved(self.recorded(key)); + } + if let Some(hash) = self.resolved_selected.get(key) { + return Entered::Resolved(Some(hash.clone())); + } + } else if let Some(position) = self.positions.get(key) { + return Entered::Resolved(self.hashes[*position].1.clone()); + } + let nodes: &'a HashMap = self.nodes; + let Some((node, local_hash)) = nodes + .get(key) + .and_then(|node| node.local_hash.as_deref().map(|local| (node, local))) + else { + return Entered::Resolved(match mode { + Mode::Expected => { + self.record(key, None); + None + } + Mode::Write => self.recorded(key), + }); + }; + if self.visiting.contains(key) { + if let Mode::Expected = mode { + self.record(key, Some(local_hash.to_owned())); + } + return Entered::Resolved(Some(local_hash.to_owned())); + } + self.visiting.insert(key.clone()); + Entered::Pending(Frame { + key, + node, + local_hash, + next_upstream: 0, + upstream_hashes: Vec::new(), + }) + } + + fn finish(&mut self, frame: &Frame<'a>, mode: Mode) -> String { + self.visiting.remove(frame.key); + let upstream: Vec<&str> = frame.upstream_hashes.iter().map(String::as_str).collect(); + let composed: String = compose_graph_identity(frame.local_hash, &upstream); + self.record(frame.key, Some(composed.clone())); + if let Mode::Write = mode { + self.resolved_selected + .insert(frame.key.clone(), composed.clone()); + } + composed + } + + fn recorded(&self, key: &GraphKey) -> Option { + self.positions + .get(key) + .and_then(|position| self.hashes[*position].1.clone()) + } + + fn record(&mut self, key: &GraphKey, hash: Option) { + match self.positions.get(key) { + Some(position) => self.hashes[*position].1 = hash, + None => { + self.positions.insert(key.clone(), self.hashes.len()); + self.hashes.push((key.clone(), hash)); + } + } + } +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..7cf4c87d38 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod compose_graph_identity; +pub(crate) mod cycle_message; +pub(crate) mod dependency_counts; +pub(crate) mod identity_resolver; +pub(crate) mod staleness_walk; diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs new file mode 100644 index 0000000000..5670b88247 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs @@ -0,0 +1,137 @@ +//! The upstream staleness walk of one selected model, with an explicit stack. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_identity::constants::{MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE}; +use crate::planner_identity::models::{GraphKey, StalenessGraph}; + +/// A model whose parents are being visited. +struct Frame<'a> { + key: &'a GraphKey, + parents: &'a [GraphKey], + next_parent: usize, + in_run_set: bool, + ancestor_stale: bool, +} + +/// What entering a node yields: its staleness already, or a frame to walk. +enum Entered<'a> { + Resolved(bool), + Pending(Frame<'a>), +} + +/// The upstream walk of one selected model. +pub(crate) struct StalenessWalk<'a> { + pub(crate) graph: &'a StalenessGraph, + pub(crate) model_key: &'a GraphKey, + pub(crate) names: BTreeSet, + pub(crate) visiting: HashSet, + pub(crate) stale: HashMap, +} + +impl<'a> StalenessWalk<'a> { + pub(crate) fn visit(&mut self, root: &'a GraphKey) -> bool { + let mut stack: Vec> = match self.enter(root) { + Entered::Resolved(is_stale) => return is_stale, + Entered::Pending(frame) => vec![frame], + }; + let graph: &'a StalenessGraph = self.graph; + while let Some(frame) = stack.last_mut() { + if let Some(parent_key) = frame.parents.get(frame.next_parent) { + frame.next_parent += 1; + match self.enter(parent_key) { + Entered::Resolved(parent_stale) => { + frame.ancestor_stale = + absorbed(frame.ancestor_stale, (graph, parent_key), parent_stale); + } + Entered::Pending(parent_frame) => stack.push(parent_frame), + } + continue; + } + let Some(finished) = stack.pop() else { + break; + }; + let is_stale: bool = self.finish(&finished); + match stack.last_mut() { + Some(child) => { + child.ancestor_stale = + absorbed(child.ancestor_stale, (graph, finished.key), is_stale); + } + None => return is_stale, + } + } + false + } + + fn enter(&mut self, key: &'a GraphKey) -> Entered<'a> { + if let Some(cached) = self.stale.get(key) { + return Entered::Resolved(*cached); + } + if self.visiting.contains(key) { + return Entered::Resolved(false); + } + let graph: &'a StalenessGraph = self.graph; + let is_stale: bool = match key.kind.as_str() { + MODEL_RESOURCE => { + let in_run_set: bool = graph.run.models.contains(&key.name); + if graph.changed.models.contains(&key.name) && !in_run_set { + self.names.insert(key.name.clone()); + true + } else { + self.visiting.insert(key.clone()); + return Entered::Pending(Frame { + key, + parents: graph.upstream_deps.get(key).map_or(&[], Vec::as_slice), + next_parent: 0, + in_run_set, + ancestor_stale: false, + }); + } + } + SEED_RESOURCE => self.leaf(key, (&graph.changed.seeds, &graph.run.seeds)), + SOURCE_RESOURCE => self.leaf(key, (&graph.changed.sources, &graph.run.sources)), + _ => false, + }; + self.stale.insert(key.clone(), is_stale); + Entered::Resolved(is_stale) + } + + fn finish(&mut self, frame: &Frame<'a>) -> bool { + if frame.ancestor_stale && !frame.in_run_set && frame.key != self.model_key { + self.names.insert(frame.key.name.clone()); + } + self.visiting.remove(frame.key); + self.stale.insert(frame.key.clone(), frame.ancestor_stale); + frame.ancestor_stale + } + + fn leaf( + &mut self, + key: &GraphKey, + (changed, run): (&HashSet, &HashSet), + ) -> bool { + let is_changed: bool = changed.contains(&key.name); + if is_changed && !run.contains(&key.name) { + self.names.insert(key.name.clone()); + } + is_changed + } +} + +fn absorbed( + ancestor_stale: bool, + (graph, parent_key): (&StalenessGraph, &GraphKey), + parent_stale: bool, +) -> bool { + parent_stale || ancestor_stale || run_parent_changed(graph, parent_key) +} + +fn run_parent_changed(graph: &StalenessGraph, parent_key: &GraphKey) -> bool { + let (run, changed): (&HashSet, &HashSet) = match parent_key.kind.as_str() { + MODEL_RESOURCE => (&graph.run.models, &graph.changed.models), + SEED_RESOURCE => (&graph.run.seeds, &graph.changed.seeds), + SOURCE_RESOURCE => (&graph.run.sources, &graph.changed.sources), + _ => return false, + }; + run.contains(&parent_key.name) && changed.contains(&parent_key.name) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/constants.rs b/crates/sqlbuild-analysis/src/planner_identity/constants.rs new file mode 100644 index 0000000000..3faff6781e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/constants.rs @@ -0,0 +1,77 @@ +//! Planner identity constants shared with the Python planner. + +/// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`. +pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 34] = [ + "append_cursor_inclusive", + "batch_size", + "check_columns", + "cursor", + "cursor_grain", + "cursor_start", + "cursor_end", + "cursor_start_max_ahead", + "cursor_start_max_action", + "cursor_future_max_distance", + "cursor_future_action", + "cursor_type", + "historical_input", + "incremental_mode", + "microbatch_strategy", + "cursor_watermark_mode", + "max_microbatches", + "microbatch_limit", + "incremental_strategy", + "merge_exclude_columns", + "full_refresh", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "observed_at", + "on_schema_change", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", +]; +/// Model config key of cursor inputs, recorded before every other identity key. +pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; +/// Model config key of user custom config. +pub const MODEL_CUSTOM_CONFIG_KEY: &str = "config"; +/// Model config key of custom placeholders. +pub const MODEL_PLACEHOLDERS_CONFIG_KEY: &str = "placeholders"; +/// Model config key of pre-hooks. +pub const MODEL_PRE_HOOKS_CONFIG_KEY: &str = "pre_hooks"; +/// Model config key of post-hooks. +pub const MODEL_POST_HOOKS_CONFIG_KEY: &str = "post_hooks"; +/// Model config key of the contract policy. +pub const CONTRACT_CONFIG_KEY: &str = "contract"; +/// The contract policy that enforces declared columns. +pub const ENFORCED_CONTRACT_POLICY: &str = "enforced"; +/// Hook fields that participate in SQL hook identity. +pub const SQL_HOOK_IDENTITY_FIELDS: [&str; 4] = ["statement", "name", "definition_sql", "kwargs"]; +/// Error code of a model or function that cannot be fingerprinted. +pub const QUERY_FINGERPRINT_FAILED: &str = "S025"; +/// Help text of a model or function that cannot be fingerprinted. +pub const FINGERPRINT_HELP: &str = "SQLBuild fingerprints the dialect token stream of compiled SQL to detect changes. Fix the SQL so the warehouse dialect tokenizer accepts it."; +/// Resource type of models. +pub const MODEL_RESOURCE: &str = "model"; +/// Resource type of seeds. +pub const SEED_RESOURCE: &str = "seed"; +/// Resource type of sources. +pub const SOURCE_RESOURCE: &str = "source"; +/// Resource type of scalar functions. +pub const UDF_RESOURCE: &str = "udf"; +/// Resource type of table functions. +pub const TABLE_FN_RESOURCE: &str = "table_fn"; +/// Resource type of SQL tests. +pub const SQL_TEST_RESOURCE: &str = "sql_test"; +/// Resource type of dbt references. +pub const DBT_REF_RESOURCE: &str = "dbt_ref"; +/// At most this many stale triggers are listed in one warning. +pub const STALE_TRIGGER_DISPLAY_LIMIT: usize = 5; +/// At most this many items are named in a missing-upstream message. +pub const MISSING_MESSAGE_ITEM_LIMIT: usize = 5; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs b/crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs new file mode 100644 index 0000000000..9cde7e0cdf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs @@ -0,0 +1,37 @@ +//! Selected models stale through changed upstreams outside the run set. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_identity::_helpers::staleness_walk::StalenessWalk; + +use crate::planner_identity::constants::MODEL_RESOURCE; +use crate::planner_identity::models::{GraphKey, StaleWarning, StalenessGraph}; + +/// One warning per selected model, by name, whose upstream closure reaches a changed node that does not run. +pub fn classify_selection_staleness(graph: &StalenessGraph) -> Vec { + let selected: BTreeSet<&String> = graph.selected_model_names.iter().collect(); + let mut warnings: Vec = Vec::new(); + for model_name in selected { + let model_key = GraphKey { + kind: MODEL_RESOURCE.to_owned(), + name: model_name.clone(), + }; + let mut walk = StalenessWalk { + graph, + model_key: &model_key, + names: BTreeSet::new(), + visiting: HashSet::new(), + stale: HashMap::new(), + }; + for upstream_key in graph.upstream_deps.get(&model_key).into_iter().flatten() { + let _ = walk.visit(upstream_key); + } + if !walk.names.is_empty() { + warnings.push(StaleWarning { + model_name: model_name.clone(), + trigger_names: walk.names.into_iter().collect(), + }); + } + } + warnings +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs b/crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs new file mode 100644 index 0000000000..7535405f79 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs @@ -0,0 +1,14 @@ +//! Expected identity hashes of every node of a dependency graph. + +use crate::planner_identity::_helpers::identity_resolver::IdentityResolver; +use crate::planner_identity::models::IdentityGraph; +use crate::planner_identity::types::IdentityHashes; + +/// Each node's hash composed from its upstream expected hashes, in first-recorded order; `None` for nodes without a local hash. +pub fn expected_graph_identity_hashes(graph: &IdentityGraph) -> IdentityHashes { + let mut resolver = IdentityResolver::new(&graph.nodes, None, Vec::new()); + for key in &graph.execution_order { + let _ = resolver.resolve_expected(key); + } + resolver.into_hashes() +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs b/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs new file mode 100644 index 0000000000..8317a9fbf2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs @@ -0,0 +1,26 @@ +//! Stable function definition text used for change fingerprints. + +use crate::planner_identity::models::FunctionFingerprintFields; + +/// The language, signature, runtime fields and body, one field per line. +pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String { + let render = |pairs: &[(String, String)]| -> String { + pairs + .iter() + .map(|(name, kind)| format!("{name}:{kind}")) + .collect::>() + .join(",") + }; + [ + format!("language={}", fields.language), + format!("arguments={}", render(&fields.arguments)), + format!("returns={}", fields.returns), + format!("return_columns={}", render(&fields.return_columns)), + format!("runtime_version={}", fields.runtime_version), + format!("entry_point={}", fields.entry_point), + format!("packages={}", fields.packages.join(",")), + "body=".to_owned(), + fields.body_sql.clone(), + ] + .join("\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs b/crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs new file mode 100644 index 0000000000..88b507fa00 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs @@ -0,0 +1,25 @@ +//! Write identity hashes from the upstream hashes actually available this run. + +use std::collections::HashSet; + +use crate::planner_identity::_helpers::identity_resolver::IdentityResolver; +use crate::planner_identity::models::{GraphKey, IdentityGraph}; + +/// The base hashes with each selected node recomposed from upstream write hashes, in first-recorded order. +pub fn graph_write_identity_hashes( + graph: &IdentityGraph, + selected_keys: &HashSet, + base_hashes: Vec<(GraphKey, String)>, +) -> Vec<(GraphKey, String)> { + let mut resolver = IdentityResolver::new(&graph.nodes, Some(selected_keys), base_hashes); + for key in &graph.execution_order { + let _ = resolver.resolve_write(key); + } + let mut hashes: Vec<(GraphKey, String)> = Vec::new(); + for (key, hash) in resolver.into_hashes() { + if let Some(hash) = hash { + hashes.push((key, hash)); + } + } + hashes +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs new file mode 100644 index 0000000000..5afe8280f0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -0,0 +1,70 @@ +//! The error message of a selected scope with missing upstreams. + +use std::collections::HashMap; + +use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; +use crate::planner_identity::models::{GraphKey, MissingUpstream}; + +/// Missing lineage upstreams by name and, when only an audit or hook read requires one, that read's origin. +pub fn missing_upstream_message( + missing: &[MissingUpstream], + edge_origins: &HashMap<(GraphKey, GraphKey), String>, +) -> String { + let mut lineage_missing: Vec<&MissingUpstream> = Vec::new(); + let mut read_names: Vec<&str> = Vec::new(); + for entry in missing { + let origins: Vec<&str> = entry + .required_by + .iter() + .filter_map(|dependent| { + edge_origins + .get(&(dependent.clone(), entry.key.clone())) + .map(String::as_str) + }) + .collect(); + if origins.len() < entry.required_by.len() { + lineage_missing.push(entry); + continue; + } + for origin in origins { + if !read_names.contains(&origin) { + read_names.push(origin); + } + } + } + let mut parts: Vec = Vec::new(); + if !lineage_missing.is_empty() { + let names: Vec<&str> = lineage_missing + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .map(|entry| entry.key.name.as_str()) + .collect(); + parts.push(format!( + "{} missing upstream dependencies ({})", + lineage_missing.len(), + names.join(", ") + )); + } + if let [name] = read_names.as_slice() { + parts.push(format!( + "{name}, which is not selected and does not exist in the warehouse; select it or build it first" + )); + } else if !read_names.is_empty() { + let shown: String = read_names + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .copied() + .collect::>() + .join("; "); + let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT { + "; ..." + } else { + "" + }; + parts.push(format!( + "{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first", + read_names.len() + )); + } + format!("cannot build selected scope: {}", parts.join("; ")) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs new file mode 100644 index 0000000000..59cb1c90a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs @@ -0,0 +1,51 @@ +//! Upstream dependencies of selected keys that neither scope nor warehouse provides. + +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::constants::{DBT_REF_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE}; +use crate::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; + +/// Missing upstreams, most dependents first, then by `(kind, name)`; dependents sorted by `(kind, name)`. +pub fn missing_upstreams( + selected_keys: &[GraphKey], + upstream_deps: &HashMap>, + available: &AvailableUpstreams, +) -> Vec { + let selected: HashSet<&GraphKey> = selected_keys.iter().collect(); + let mut missing: HashMap> = HashMap::new(); + for selected_key in selected_keys { + for dep_key in upstream_deps.get(selected_key).into_iter().flatten() { + if [SQL_TEST_RESOURCE, DBT_REF_RESOURCE, SOURCE_RESOURCE] + .contains(&dep_key.kind.as_str()) + || selected.contains(dep_key) + || available.satisfied_keys.contains(dep_key) + || available.existing_relations.contains(&dep_key.name) + || available + .deferred_relations + .as_ref() + .is_some_and(|deferred| deferred.contains(&dep_key.name)) + { + continue; + } + missing + .entry(dep_key.clone()) + .or_default() + .push(selected_key.clone()); + } + } + let mut entries: Vec = missing + .into_iter() + .map(|(key, mut required_by)| { + required_by.sort(); + MissingUpstream { key, required_by } + }) + .collect(); + entries.sort_by(|left, right| { + right + .required_by + .len() + .cmp(&left.required_by.len()) + .then_with(|| left.key.cmp(&right.key)) + }); + entries +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..219d458e53 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs @@ -0,0 +1,10 @@ +pub mod classify_selection_staleness; +pub mod expected_graph_identity_hashes; +pub mod function_fingerprint_sql; +pub mod graph_write_identity_hashes; +pub mod missing_upstream_message; +pub mod missing_upstreams; +pub mod model_local_identity_hash; +pub mod stable_identity_hash; +pub mod stale_upstream_warning_message; +pub mod topological_order; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs b/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs new file mode 100644 index 0000000000..11148c52dc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs @@ -0,0 +1,8 @@ +//! A model's local identity hash from its query fingerprint and non-query metadata. + +use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; + +/// The hash of the query fingerprint and metadata JSON, one per line. +pub fn model_local_identity_hash(query_fingerprint: &str, metadata_json: &str) -> String { + stable_identity_hash(&format!("{query_fingerprint}\n{metadata_json}")) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs b/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs new file mode 100644 index 0000000000..f119ec5c7f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs @@ -0,0 +1,15 @@ +//! The SHA-256 hash used for local and composed version identities. + +use std::fmt::Write; + +use sha2::{Digest, Sha256}; + +/// The lowercase hex SHA-256 digest of the UTF-8 text. +pub fn stable_identity_hash(value: &str) -> String { + Sha256::digest(value.as_bytes()) + .iter() + .fold(String::with_capacity(64), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs new file mode 100644 index 0000000000..612caef505 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs @@ -0,0 +1,25 @@ +//! A multi-line stale-selection warning: summary, capped bullet list and one hint. + +use crate::planner_identity::constants::STALE_TRIGGER_DISPLAY_LIMIT; + +/// The warning text for `model_name` building on stale `trigger_names`. +pub fn stale_upstream_warning_message( + (model_label, model_name): (&str, &str), + trigger_label: &str, + trigger_names: &[String], +) -> String { + let count: usize = trigger_names.len(); + let mut lines: Vec = vec![format!( + "{model_label} '{model_name}' will build on {count} stale {trigger_label} not selected for rebuild:" + )]; + for name in trigger_names.iter().take(STALE_TRIGGER_DISPLAY_LIMIT) { + lines.push(format!(" - {name}")); + } + if count > STALE_TRIGGER_DISPLAY_LIMIT { + lines.push(format!(" +{} more", count - STALE_TRIGGER_DISPLAY_LIMIT)); + } + lines.push(format!( + " rebuild the closure to refresh them: --select +{model_name}" + )); + lines.join("\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs b/crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs new file mode 100644 index 0000000000..6a183eb1d7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs @@ -0,0 +1,45 @@ +//! Stable dependency order with Kahn's algorithm, smallest `(kind, name)` first. + +use std::cmp::Reverse; +use std::collections::{BinaryHeap, HashMap}; + +use crate::planner_identity::_helpers::cycle_message::cycle_message; +use crate::planner_identity::_helpers::dependency_counts::{DependencyCounts, dependency_counts}; +use crate::planner_identity::models::GraphKey; + +/// Every key in dependency order, or the dependency-cycle message naming the keys left over and the injected edges between them. +pub fn topological_order( + upstream: &[(GraphKey, Vec)], + injected_edge_origins: &HashMap<(GraphKey, GraphKey), String>, +) -> Result, String> { + let DependencyCounts { + node_keys, + mut indegree, + downstream, + } = dependency_counts(upstream); + let mut ready: BinaryHeap> = indegree + .iter() + .filter(|(_, count)| **count == 0) + .map(|(key, _)| Reverse(*key)) + .collect(); + let mut ordered: Vec = Vec::with_capacity(node_keys.len()); + while let Some(Reverse(current)) = ready.pop() { + ordered.push(current.clone()); + for downstream_key in downstream.get(current).into_iter().flatten() { + if let Some(count) = indegree.get_mut(downstream_key) { + *count = count.saturating_sub(1); + if *count == 0 { + ready.push(Reverse(downstream_key)); + } + } + } + } + if ordered.len() == node_keys.len() { + return Ok(ordered); + } + Err(cycle_message( + &ordered, + node_keys, + (upstream, injected_edge_origins), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/mod.rs new file mode 100644 index 0000000000..efa80954a5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/mod.rs @@ -0,0 +1,10 @@ +//! Planner identity, pruning and graph order: version identity hashes, dependency-aware identity composition, stale-selection warnings, buildability and execution order. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs new file mode 100644 index 0000000000..54a84b5fda --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -0,0 +1,76 @@ +//! Planner identity, pruning and graph-order data. + +use std::collections::{HashMap, HashSet}; + +/// A graph node identity: resource type (or fingerprint node type) and name, ordered as Python's `(resource_type, name)`. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct GraphKey { + pub kind: String, + pub name: String, +} + +/// One node of an identity graph: its upstream keys in order and its local hash. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IdentityNode { + pub upstream_keys: Vec, + pub local_hash: Option, +} + +/// An identity graph: nodes by key and the order nodes are resolved in. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct IdentityGraph { + pub nodes: HashMap, + pub execution_order: Vec, +} + +/// A function definition's fingerprint fields, already rendered as text. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FunctionFingerprintFields { + pub language: String, + pub arguments: Vec<(String, String)>, + pub returns: String, + pub return_columns: Vec<(String, String)>, + pub runtime_version: String, + pub entry_point: String, + pub packages: Vec, + pub body_sql: String, +} + +/// Names that changed or run this time, per resource kind, for stale-selection classification. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StalenessNames { + pub models: HashSet, + pub seeds: HashSet, + pub sources: HashSet, +} + +/// Graph inputs of selected-model stale warning classification. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StalenessGraph { + pub upstream_deps: HashMap>, + pub selected_model_names: Vec, + pub run: StalenessNames, + pub changed: StalenessNames, +} + +/// A selected model that builds on changed upstreams outside the run set. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StaleWarning { + pub model_name: String, + pub trigger_names: Vec, +} + +/// What satisfies an upstream dependency of a selected key. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AvailableUpstreams { + pub satisfied_keys: HashSet, + pub existing_relations: HashSet, + pub deferred_relations: Option>, +} + +/// One upstream dependency missing from both scope and warehouse, with its dependents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MissingUpstream { + pub key: GraphKey, + pub required_by: Vec, +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests.rs b/crates/sqlbuild-analysis/src/planner_identity/tests.rs new file mode 100644 index 0000000000..a26a2fda78 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_identity.rs"] +mod planner_identity; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs new file mode 100644 index 0000000000..57a7e0faf0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs @@ -0,0 +1,147 @@ +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::_helpers::compose_graph_identity::compose_graph_identity; +use crate::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode, StalenessGraph}; + +pub(super) fn key(kind: &str, name: &str) -> GraphKey { + GraphKey { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +pub(super) fn model(name: &str) -> GraphKey { + key("model", name) +} + +pub(super) fn names(values: &[&str]) -> HashSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Nodes `a`, `b` over `a`, `c` over `a`, `b` and unknown `x`, and `d` over `c` without a local hash. +pub(super) fn identity_graph() -> IdentityGraph { + let node = |upstream: &[&str], local: Option<&str>| IdentityNode { + upstream_keys: upstream.iter().map(|name| model(name)).collect(), + local_hash: local.map(str::to_owned), + }; + IdentityGraph { + nodes: HashMap::from([ + (model("a"), node(&[], Some("la"))), + (model("b"), node(&["a"], Some("lb"))), + (model("c"), node(&["a", "b", "x"], Some("lc"))), + (model("d"), node(&["c"], None)), + ]), + execution_order: vec![model("c"), model("a"), model("b"), model("d")], + } +} + +pub(super) fn staleness_graph( + edges: &[(GraphKey, &[GraphKey])], + selected: &[&str], +) -> StalenessGraph { + StalenessGraph { + upstream_deps: edges + .iter() + .map(|(node, upstream)| (node.clone(), upstream.to_vec())) + .collect(), + selected_model_names: texts(selected), + ..StalenessGraph::default() + } +} + +/// Two models reading each other. +pub(super) fn cyclic_identity_graph() -> IdentityGraph { + let node = |upstream: &str, local: &str| IdentityNode { + upstream_keys: vec![model(upstream)], + local_hash: Some(local.to_owned()), + }; + IdentityGraph { + nodes: HashMap::from([(model("a"), node("b", "la")), (model("b"), node("a", "lb"))]), + execution_order: vec![model("a"), model("b")], + } +} + +/// A dense ladder: each layer's two nodes read both nodes of the previous layer. +pub(super) fn ladder_identity_graph(layer_count: usize) -> IdentityGraph { + let mut graph = IdentityGraph::default(); + let mut previous: Vec = Vec::new(); + for layer in 0..layer_count { + let current: Vec = ["a", "b"] + .iter() + .map(|suffix| model(&format!("L{layer}_{suffix}"))) + .collect(); + for key in ¤t { + graph.nodes.insert( + key.clone(), + IdentityNode { + upstream_keys: previous.clone(), + local_hash: Some(format!("local_{}", key.name)), + }, + ); + graph.execution_order.push(key.clone()); + } + previous = current; + } + graph +} + +/// A chain where each model reads the previous one, resolved from its most downstream model. +pub(super) fn chain_identity_graph(depth: usize) -> IdentityGraph { + let mut graph = IdentityGraph::default(); + for index in 0..depth { + let upstream_keys: Vec = index + .checked_sub(1) + .map(|upstream| model(&format!("m{upstream}"))) + .into_iter() + .collect(); + graph.nodes.insert( + model(&format!("m{index}")), + IdentityNode { + upstream_keys, + local_hash: Some(format!("local_m{index}")), + }, + ); + } + graph.execution_order = (0..depth) + .rev() + .map(|index| model(&format!("m{index}"))) + .collect(); + graph +} + +/// The most downstream hash of `chain_identity_graph`, composed one link at a time. +pub(super) fn chain_head_hash(depth: usize) -> Option { + let mut hash: Option = None; + for index in 0..depth { + let upstream: Vec<&str> = hash.iter().map(String::as_str).collect(); + hash = Some(compose_graph_identity( + &format!("local_m{index}"), + &upstream, + )); + } + hash +} + +/// The staleness graph of a model chain whose most upstream model changed outside the run set. +pub(super) fn chain_staleness_graph(depth: usize) -> StalenessGraph { + let head: String = format!("m{}", depth - 1); + let mut graph = StalenessGraph { + upstream_deps: (1..depth) + .map(|index| { + ( + model(&format!("m{index}")), + vec![model(&format!("m{}", index - 1))], + ) + }) + .collect(), + selected_model_names: vec![head.clone()], + ..StalenessGraph::default() + }; + graph.changed.models = names(&["m0"]); + graph.run.models = HashSet::from([head]); + graph +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs new file mode 100644 index 0000000000..4ce46945f6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -0,0 +1,450 @@ +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::_helpers::compose_graph_identity::compose_graph_identity; +use crate::planner_identity::main::classify_selection_staleness::classify_selection_staleness; +use crate::planner_identity::main::expected_graph_identity_hashes::expected_graph_identity_hashes; +use crate::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; +use crate::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; +use crate::planner_identity::main::missing_upstream_message::missing_upstream_message; +use crate::planner_identity::main::missing_upstreams::missing_upstreams; +use crate::planner_identity::main::model_local_identity_hash::model_local_identity_hash; +use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; +use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; +use crate::planner_identity::main::topological_order::topological_order; +use crate::planner_identity::models::{ + AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, StaleWarning, + StalenessNames, +}; +use crate::planner_identity::tests::helpers::{ + chain_head_hash, chain_identity_graph, chain_staleness_graph, cyclic_identity_graph, + identity_graph, key, ladder_identity_graph, model, names, staleness_graph, texts, +}; +use crate::planner_identity::tests::test_types::{ + BuildabilityTestCase, DeepChainTestCase, FunctionSqlTestCase, HashTestCase, + IdentityGraphSizeTestCase, IdentityGraphTestCase, OrderTestCase, StalenessTestCase, + WarningTextTestCase, +}; +use crate::planner_identity::types::IdentityHashes; + +#[test] +fn given_persisted_identity_inputs_when_hashing_then_matches_pinned_python_hashes() { + let test_cases = [ + HashTestCase { + description: "empty text", + actual: stable_identity_hash(""), + expected_hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + HashTestCase { + description: "unicode and carriage returns", + actual: stable_identity_hash("orders\r\ncafé ☃"), + expected_hash: "ba7e194792c57a2f2e3aa77441f680ac5823e5a19cbbef0113bd20d701aa4ab2", + }, + HashTestCase { + description: "composed with upstreams", + actual: compose_graph_identity("abc", &["d1", "d2"]), + expected_hash: "5be4a7558096ea1ae6d8c0f97fac33972a28951ded9ce7e9a2c1898c2b8389cc", + }, + HashTestCase { + description: "composed without upstreams", + actual: compose_graph_identity("abc", &[]), + expected_hash: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }, + HashTestCase { + description: "model local identity", + actual: model_local_identity_hash("q1", r#"{"config":{}}"#), + expected_hash: "3f16f63bb7e4c297529a7814cdce6737d9fcc1f3ad695905c58daa2438613c16", + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.actual, test_case.expected_hash, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_identity_graph_when_resolving_then_matches_pinned_python_hashes() { + let pinned = |hash: &str| hash.to_owned(); + let test_cases = [ + IdentityGraphTestCase { + description: "acyclic graph with an unknown upstream and a node without a local hash", + graph: identity_graph(), + selected: HashSet::from([model("b"), model("c")]), + base_hashes: vec![(model("a"), "base_a".to_owned())], + expected_hashes: vec![ + ( + model("a"), + Some(pinned( + "b737ca7ee563ae80e457bb3d1dfe64edd2b4c015a8f88b6f87d5c113b68897fd", + )), + ), + ( + model("b"), + Some(pinned( + "b1d269f24b600f5b30f9b68dc34f0305ebf3c23b3657ebb147b6cae7ffaf8282", + )), + ), + (model("x"), None), + ( + model("c"), + Some(pinned( + "3f7d5648557bef56e83e9b9b9cc1789cf758e32310f142d733b3d57a5caabc15", + )), + ), + (model("d"), None), + ], + expected_write_hashes: vec![ + (model("a"), pinned("base_a")), + ( + model("b"), + pinned("1780fb66cc0e91d38e4be3c63ffa49e52abaa5365a68aad535ff04218f58ecc9"), + ), + ( + model("c"), + pinned("d41afd29f6fdfaa9bc2a55fd9e5f1a75b2e8dbe100f431706a67938e21fd1b86"), + ), + ], + }, + IdentityGraphTestCase { + description: "cycle broken with the local hash of the node being resolved", + graph: cyclic_identity_graph(), + selected: HashSet::from([model("a"), model("b")]), + base_hashes: Vec::new(), + expected_hashes: vec![ + ( + model("a"), + Some(pinned( + "9c389106b00932fd7ff1b7dd3778747eac217990a9a73b16c80a9f0bbbf9f527", + )), + ), + ( + model("b"), + Some(pinned( + "b8c36ab63f3421b0ce0f3b1b55428334e42f1c1a8db65d10707bdc236a81da76", + )), + ), + ], + expected_write_hashes: vec![ + ( + model("b"), + pinned("b8c36ab63f3421b0ce0f3b1b55428334e42f1c1a8db65d10707bdc236a81da76"), + ), + ( + model("a"), + pinned("9c389106b00932fd7ff1b7dd3778747eac217990a9a73b16c80a9f0bbbf9f527"), + ), + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + expected_graph_identity_hashes(&test_case.graph), + graph_write_identity_hashes( + &test_case.graph, + &test_case.selected, + test_case.base_hashes + ), + ), + (test_case.expected_hashes, test_case.expected_write_hashes), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dense_ladder_when_resolving_write_hashes_then_resolves_each_node_once() { + let test_cases = [IdentityGraphSizeTestCase { + description: "four hundred fully selected layers", + layer_count: 400, + expected_hash_count: 800, + }]; + for test_case in test_cases { + let graph = ladder_identity_graph(test_case.layer_count); + let selected: HashSet = graph.nodes.keys().cloned().collect(); + assert_eq!( + graph_write_identity_hashes(&graph, &selected, Vec::new()).len(), + test_case.expected_hash_count, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_deep_chain_when_walking_graph_then_does_not_overflow_the_stack() { + let test_cases = [DeepChainTestCase { + description: "one hundred thousand chained models", + depth: 100_000, + expected_hash_count: 100_000, + expected_trigger_count: 99_999, + }]; + for test_case in test_cases { + let graph = chain_identity_graph(test_case.depth); + let head: GraphKey = model(&format!("m{}", test_case.depth - 1)); + let expected: IdentityHashes = expected_graph_identity_hashes(&graph); + assert_eq!( + expected.len(), + test_case.expected_hash_count, + "{}", + test_case.description + ); + assert_eq!( + expected.last().map(|(_, hash)| hash.clone()), + Some(chain_head_hash(test_case.depth)), + "{}", + test_case.description + ); + let selected: HashSet = graph.nodes.keys().cloned().collect(); + let written: Vec<(GraphKey, String)> = + graph_write_identity_hashes(&graph, &selected, Vec::new()); + assert_eq!( + written.len(), + test_case.expected_hash_count, + "{}", + test_case.description + ); + assert_eq!( + written + .iter() + .find(|(key, _)| *key == head) + .map(|(_, hash)| hash.clone()), + chain_head_hash(test_case.depth), + "{}", + test_case.description + ); + let warnings: Vec = + classify_selection_staleness(&chain_staleness_graph(test_case.depth)); + assert_eq!( + warnings + .iter() + .map(|warning| warning.trigger_names.len()) + .sum::(), + test_case.expected_trigger_count, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_function_fields_when_rendering_then_writes_one_field_per_line() { + let test_cases = [FunctionSqlTestCase { + description: "python function with packages and an unnamed argument", + fields: FunctionFingerprintFields { + language: "python".to_owned(), + arguments: vec![ + ("x".to_owned(), "int".to_owned()), + (String::new(), "text".to_owned()), + ], + returns: "table".to_owned(), + return_columns: vec![("total".to_owned(), "double".to_owned())], + runtime_version: "3.12".to_owned(), + entry_point: String::new(), + packages: texts(&["numpy", "pandas"]), + body_sql: "return x\r\n".to_owned(), + }, + expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\ + runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n", + }]; + for test_case in test_cases { + assert_eq!( + function_fingerprint_sql(&test_case.fields), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selection_when_classifying_staleness_then_names_changed_upstreams_outside_run() { + let mut changed_outside = staleness_graph( + &[ + (model("report"), &[model("orders"), key("seed", "regions")]), + (model("orders"), &[model("raw_orders")]), + ], + &["report"], + ); + changed_outside.changed = StalenessNames { + models: names(&["raw_orders"]), + seeds: names(&["regions"]), + sources: HashSet::new(), + }; + let mut run_parent = staleness_graph( + &[ + (model("report"), &[model("orders")]), + (model("orders"), &[model("raw_orders")]), + ], + &["report"], + ); + run_parent.run.models = names(&["raw_orders", "report"]); + run_parent.changed.models = names(&["raw_orders"]); + let mut source_in_run = staleness_graph( + &[(model("report"), &[key("source", "events")])], + &["report"], + ); + source_in_run.changed.sources = names(&["events"]); + source_in_run.run.sources = names(&["events"]); + let test_cases = [ + StalenessTestCase { + description: "changed model and seed outside the run set", + graph: changed_outside, + expected_warnings: vec![StaleWarning { + model_name: "report".to_owned(), + trigger_names: texts(&["orders", "raw_orders", "regions"]), + }], + }, + StalenessTestCase { + description: "changed parent runs but the middle model does not", + graph: run_parent, + expected_warnings: vec![StaleWarning { + model_name: "report".to_owned(), + trigger_names: texts(&["orders"]), + }], + }, + StalenessTestCase { + description: "changed source in the run set", + graph: source_in_run, + expected_warnings: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + classify_selection_staleness(&test_case.graph), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_trigger_names_when_formatting_then_caps_the_list() { + let test_cases = [ + WarningTextTestCase { + description: "one trigger", + trigger_names: texts(&["orders"]), + expected_message: "selected model 'report' will build on 1 stale upstream(s) not selected for rebuild:\n - orders\n rebuild the closure to refresh them: --select +report", + }, + WarningTextTestCase { + description: "more than the display limit", + trigger_names: texts(&["a", "b", "c", "d", "e", "f", "g"]), + expected_message: "selected model 'report' will build on 7 stale upstream(s) not selected for rebuild:\n - a\n - b\n - c\n - d\n - e\n +2 more\n rebuild the closure to refresh them: --select +report", + }, + ]; + for test_case in test_cases { + assert_eq!( + stale_upstream_warning_message( + ("selected model", "report"), + "upstream(s)", + &test_case.trigger_names + ), + test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_upstream_edges_when_ordering_then_orders_stably_or_reports_cycles() { + let test_cases = [ + OrderTestCase { + description: "smallest ready key first", + upstream: vec![ + (model("report"), vec![model("orders"), key("seed", "regions")]), + (model("orders"), vec![key("source", "raw")]), + ], + origins: HashMap::new(), + expected_order: Ok(vec![ + key("seed", "regions"), + key("source", "raw"), + model("orders"), + model("report"), + ]), + }, + OrderTestCase { + description: "cycle through an injected edge", + upstream: vec![ + (model("a"), vec![key("sql_test", "t")]), + (key("sql_test", "t"), vec![model("a")]), + (model("z"), vec![]), + ], + origins: HashMap::from([( + (model("a"), key("sql_test", "t")), + "SQL test 't' runs before 'a'".to_owned(), + )]), + expected_order: Err("Dependency cycle detected involving: model:a, sql_test:t (via SQL test 't' runs before 'a')".to_owned()), + }, + ]; + for test_case in test_cases { + assert_eq!( + topological_order(&test_case.upstream, &test_case.origins), + test_case.expected_order, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams() { + let test_cases = [ + BuildabilityTestCase { + description: "lineage upstream missing for two dependents", + selected: vec![model("b"), model("a")], + upstream: HashMap::from([ + (model("a"), vec![model("raw"), key("source", "events")]), + (model("b"), vec![model("raw"), model("built"), model("a")]), + ]), + available: AvailableUpstreams { + existing_relations: names(&["built"]), + ..AvailableUpstreams::default() + }, + origins: HashMap::new(), + expected_missing: vec![MissingUpstream { + key: model("raw"), + required_by: vec![model("a"), model("b")], + }], + expected_message: "cannot build selected scope: 1 missing upstream dependencies (raw)", + }, + BuildabilityTestCase { + description: "only an audit read requires the upstream", + selected: vec![model("a")], + upstream: HashMap::from([(model("a"), vec![model("lookup")])]), + available: AvailableUpstreams::default(), + origins: HashMap::from([( + (model("a"), model("lookup")), + "audit 'check' on 'a' reads 'lookup'".to_owned(), + )]), + expected_missing: vec![MissingUpstream { + key: model("lookup"), + required_by: vec![model("a")], + }], + expected_message: "cannot build selected scope: audit 'check' on 'a' reads 'lookup', which is not selected and does not exist in the warehouse; select it or build it first", + }, + ]; + for test_case in test_cases { + let missing = missing_upstreams( + &test_case.selected, + &test_case.upstream, + &test_case.available, + ); + assert_eq!( + ( + missing_upstream_message(&missing, &test_case.origins), + missing + ), + ( + test_case.expected_message.to_owned(), + test_case.expected_missing + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs new file mode 100644 index 0000000000..9c701080c4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs @@ -0,0 +1,70 @@ +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::models::{ + AvailableUpstreams, FunctionFingerprintFields, GraphKey, IdentityGraph, MissingUpstream, + StaleWarning, StalenessGraph, +}; +use crate::planner_identity::types::IdentityHashes; + +pub(super) struct HashTestCase { + pub(super) description: &'static str, + pub(super) actual: String, + pub(super) expected_hash: &'static str, +} + +pub(super) struct StalenessTestCase { + pub(super) description: &'static str, + pub(super) graph: StalenessGraph, + pub(super) expected_warnings: Vec, +} + +pub(super) struct OrderTestCase { + pub(super) description: &'static str, + pub(super) upstream: Vec<(GraphKey, Vec)>, + pub(super) origins: HashMap<(GraphKey, GraphKey), String>, + pub(super) expected_order: Result, String>, +} + +pub(super) struct BuildabilityTestCase { + pub(super) description: &'static str, + pub(super) selected: Vec, + pub(super) upstream: HashMap>, + pub(super) available: AvailableUpstreams, + pub(super) origins: HashMap<(GraphKey, GraphKey), String>, + pub(super) expected_missing: Vec, + pub(super) expected_message: &'static str, +} + +pub(super) struct WarningTextTestCase { + pub(super) description: &'static str, + pub(super) trigger_names: Vec, + pub(super) expected_message: &'static str, +} + +pub(super) struct IdentityGraphTestCase { + pub(super) description: &'static str, + pub(super) graph: IdentityGraph, + pub(super) selected: HashSet, + pub(super) base_hashes: Vec<(GraphKey, String)>, + pub(super) expected_hashes: IdentityHashes, + pub(super) expected_write_hashes: Vec<(GraphKey, String)>, +} + +pub(super) struct IdentityGraphSizeTestCase { + pub(super) description: &'static str, + pub(super) layer_count: usize, + pub(super) expected_hash_count: usize, +} + +pub(super) struct FunctionSqlTestCase { + pub(super) description: &'static str, + pub(super) fields: FunctionFingerprintFields, + pub(super) expected_sql: &'static str, +} + +pub(super) struct DeepChainTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + pub(super) expected_hash_count: usize, + pub(super) expected_trigger_count: usize, +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/types.rs b/crates/sqlbuild-analysis/src/planner_identity/types.rs new file mode 100644 index 0000000000..a60fe417d6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/types.rs @@ -0,0 +1,6 @@ +//! Planner identity type aliases. + +use crate::planner_identity::models::GraphKey; + +/// Identity hashes in first-recorded order. +pub type IdentityHashes = Vec<(GraphKey, Option)>; diff --git a/crates/sqlbuild-analysis/src/planner_scope/constants.rs b/crates/sqlbuild-analysis/src/planner_scope/constants.rs new file mode 100644 index 0000000000..cd3b1d2877 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/constants.rs @@ -0,0 +1,5 @@ +//! Planner resource and selector kinds. + +pub const SOURCE: &str = "source"; +pub const MODEL: &str = "model"; +pub const NAME: &str = "name"; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs new file mode 100644 index 0000000000..9c33d9f004 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs @@ -0,0 +1,67 @@ +//! Expand validated loader DAGs without recursive traversal. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::SOURCE; +use crate::planner_scope::models::LoaderGraph; +use std::collections::{HashMap, HashSet}; + +/// Add loader dependency edges and executable upstream source keys in input iteration order. +pub fn expand_loaders( + graph: &LoaderGraph, + mut selected: Vec, + mut upstream: Vec<(ObjectKey, Vec)>, + executable: &HashSet, +) -> (Vec, Vec<(ObjectKey, Vec)>) { + let mut positions: HashMap = upstream + .iter() + .enumerate() + .map(|(i, (k, _))| (k.clone(), i)) + .collect(); + let mut selected_seen: HashSet = selected.iter().cloned().collect(); + for key in selected.clone() { + if key.0 != SOURCE { + continue; + } + let Some(loader) = graph.source_loader(&key) else { + if !graph.sources.iter().any(|(name, _)| name == &key.1) + && graph.loaders.contains_key(&key.1) + && !positions.contains_key(&key) + { + positions.insert(key.clone(), upstream.len()); + upstream.push((key, Vec::new())); + } + continue; + }; + let Some(deps) = graph.loaders.get(loader) else { + continue; + }; + let mut pending = vec![(key.clone(), deps)]; + let mut visited: HashSet = HashSet::new(); + while let Some((node, dependencies)) = pending.pop() { + if !visited.insert(node.clone()) { + continue; + } + let edges: Vec = dependencies.iter().map(|name| graph.key(name)).collect(); + if let Some(index) = positions.get(&node) { + upstream[*index].1 = edges; + } else { + positions.insert(node.clone(), upstream.len()); + upstream.push((node, edges)); + } + for name in dependencies.iter().rev() { + if let Some(next) = graph.loaders.get(name) { + pending.push((graph.key(name), next)); + } + } + } + if executable.contains(&key) { + for name in graph.upstream(deps) { + let node = graph.key(&name); + if selected_seen.insert(node.clone()) { + selected.push(node); + } + } + } + } + (selected, upstream) +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs b/crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs new file mode 100644 index 0000000000..bfe0a7146e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs @@ -0,0 +1,38 @@ +//! Intermediate source names required by selected loader nodes. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::SOURCE; +use crate::planner_scope::models::LoaderGraph; +use std::collections::HashSet; + +/// Intermediate loader names in selected or upstream first-seen order. +pub fn intermediate_sources( + graph: &LoaderGraph, + selected: &[ObjectKey], + upstream: bool, +) -> Vec { + let mut names: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for key in selected { + if key.0 != SOURCE { + continue; + } + let candidates = if upstream { + graph + .source_loader(key) + .and_then(|loader| graph.loaders.get(loader)) + .map(|deps| graph.upstream(deps)) + .unwrap_or_default() + } else if !graph.sources.iter().any(|(name, _)| name == &key.1) { + vec![key.1.clone()] + } else { + Vec::new() + }; + for name in candidates { + if graph.intermediate(&name) && seen.insert(name.clone()) { + names.push(name); + } + } + } + names +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs b/crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs new file mode 100644 index 0000000000..93222c064c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs @@ -0,0 +1,21 @@ +//! Direct managed sources required by selected models. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::{MODEL, SOURCE}; +use std::collections::HashSet; + +/// Select only direct managed source dependencies of selected models. +pub fn managed_sources( + selected: &[ObjectKey], + upstream: &[(ObjectKey, Vec)], + managed: &HashSet, +) -> HashSet { + let selected: HashSet<&ObjectKey> = selected.iter().filter(|key| key.0 == MODEL).collect(); + upstream + .iter() + .filter(|(key, _)| selected.contains(key)) + .flat_map(|(_, deps)| deps) + .filter(|key| key.0 == SOURCE && managed.contains(&key.1)) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs new file mode 100644 index 0000000000..3fbafe8e4c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs @@ -0,0 +1,6 @@ +//! Planner scope entry points. + +pub mod expand_loaders; +pub mod intermediate_sources; +pub mod managed_sources; +pub mod source_selectors; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs b/crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs new file mode 100644 index 0000000000..3b66fcebc4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs @@ -0,0 +1,42 @@ +//! Explicit upstream source selector eligibility. + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::ParsedSelector; +use crate::planner_scope::constants::{NAME, SOURCE}; +use std::collections::HashSet; + +/// Find selected sources explicitly named by upstream source or bare-name selectors. +pub fn source_selectors( + select: &[String], + names: &[(String, ObjectKey)], + selected: &[ObjectKey], +) -> Result, SelectorError> { + let mut keys: HashSet = HashSet::new(); + for part in select + .iter() + .flat_map(|raw| raw.split_whitespace()) + .flat_map(|token| token.split(',')) + { + let ParsedSelector::Kind { + kind, + value, + upstream: true, + .. + } = parse_selector(part)? + else { + continue; + }; + if kind != NAME && kind != SOURCE { + continue; + } + if let Some((_, key)) = names.iter().rev().find(|(name, _)| name == &value) + && key.0 == SOURCE + && selected.contains(key) + { + keys.insert(key.clone()); + } + } + Ok(keys) +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/mod.rs b/crates/sqlbuild-analysis/src/planner_scope/mod.rs new file mode 100644 index 0000000000..361e6c6bec --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/mod.rs @@ -0,0 +1,8 @@ +//! Planner scope and loader dependency selection. + +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_scope/models.rs b/crates/sqlbuild-analysis/src/planner_scope/models.rs new file mode 100644 index 0000000000..38a1602733 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/models.rs @@ -0,0 +1,61 @@ +//! Ordered loader facts used by planner scope resolution. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::SOURCE; + +/// Loader dependencies resolved from Python function objects at the boundary. +#[derive(Debug, Default)] +pub struct LoaderGraph { + pub loaders: HashMap>, + pub sources: Vec<(String, Option)>, +} + +impl LoaderGraph { + /// Upstream loader names in breadth-first, first-seen order. + pub fn upstream(&self, dependencies: &[String]) -> Vec { + let mut pending: VecDeque = dependencies.iter().cloned().collect(); + let mut seen: HashSet = HashSet::new(); + let mut names: Vec = Vec::new(); + while let Some(name) = pending.pop_front() { + if !seen.insert(name.clone()) { + continue; + } + if let Some(dependencies) = self.loaders.get(&name) { + pending.extend(dependencies.iter().cloned()); + } + names.push(name); + } + names + } + + /// Source key for a loader, using the last source declaration for that loader. + pub fn key(&self, loader: &str) -> ObjectKey { + let name = self.sources.iter().rev().find_map(|(name, source_loader)| { + (source_loader.as_deref() == Some(loader)).then_some(name.as_str()) + }); + ("source".to_owned(), name.unwrap_or(loader).to_owned()) + } + + /// Managed loader name for a source key. + pub fn source_loader(&self, key: &ObjectKey) -> Option<&str> { + if key.0 != SOURCE { + return None; + } + self.sources + .iter() + .rev() + .find_map(|(name, loader)| (name == &key.1).then_some(loader.as_deref())) + .flatten() + } + + /// Whether a loader is intermediate rather than attached to a declared source. + pub fn intermediate(&self, name: &str) -> bool { + self.loaders.contains_key(name) + && !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests.rs b/crates/sqlbuild-analysis/src/planner_scope/tests.rs new file mode 100644 index 0000000000..1f484c175a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/tests.rs @@ -0,0 +1,6 @@ +//! Planner scope regression tests. + +#[path = "tests/test_loaders.rs"] +mod test_loaders; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs new file mode 100644 index 0000000000..838ceeee51 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -0,0 +1,122 @@ +//! Iterative loader DAG ordering and execution eligibility regressions. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::main::expand_loaders::expand_loaders; +use crate::planner_scope::models::LoaderGraph; +use crate::planner_scope::tests::test_types::{DeepLoaderTestCase, LoaderTestCase}; +use std::collections::HashSet; + +#[test] +fn given_shared_loader_dependencies_when_expanding_then_preserves_depth_first_edges_and_breadth_first_names() + { + let test_cases = [ + LoaderTestCase { + description: "read only upstream", + executable: vec![], + expected_selected: vec!["raw_orders"], + }, + LoaderTestCase { + description: "execute upstream", + executable: vec!["raw_orders"], + expected_selected: vec!["raw_orders", "first", "shared", "last", "leaf"], + }, + ]; + for test_case in test_cases { + let graph = LoaderGraph { + loaders: [ + ( + "terminal".to_owned(), + vec!["first".to_owned(), "shared".to_owned()], + ), + ( + "first".to_owned(), + vec!["shared".to_owned(), "last".to_owned()], + ), + ("shared".to_owned(), vec!["leaf".to_owned()]), + ("last".to_owned(), vec!["leaf".to_owned()]), + ("leaf".to_owned(), Vec::new()), + ] + .into_iter() + .collect(), + sources: vec![("raw_orders".to_owned(), Some("terminal".to_owned()))], + }; + let root: ObjectKey = ("source".to_owned(), "raw_orders".to_owned()); + let executable: HashSet = test_case + .executable + .iter() + .map(|name| ("source".to_owned(), (*name).to_owned())) + .collect(); + let (selected, edges) = expand_loaders(&graph, vec![root], Vec::new(), &executable); + assert_eq!( + selected + .iter() + .map(|key| key.1.as_str()) + .collect::>(), + test_case.expected_selected, + "{}", + test_case.description + ); + assert_eq!( + edges + .iter() + .map(|(key, _)| key.1.as_str()) + .collect::>(), + vec!["raw_orders", "first", "shared", "leaf", "last"], + "{}", + test_case.description + ); + assert_eq!( + graph.upstream(&graph.loaders["terminal"]), + vec!["first", "shared", "last", "leaf"], + "{}", + test_case.description + ); + } +} + +#[test] +fn given_deep_loader_chain_when_expanding_then_uses_iterative_traversal() { + let test_cases = [DeepLoaderTestCase { + description: "deep validated chain", + depth: 2048, + expected_count: 2048, + }]; + for test_case in test_cases { + let mut graph = LoaderGraph { + loaders: (0..test_case.depth - 1) + .map(|index| { + ( + format!("loader_{index}"), + vec![format!("loader_{}", index + 1)], + ) + }) + .collect(), + ..LoaderGraph::default() + }; + graph + .loaders + .insert(format!("loader_{}", test_case.depth - 1), Vec::new()); + graph + .sources + .push(("raw_orders".to_owned(), Some("loader_0".to_owned()))); + let root: ObjectKey = ("source".to_owned(), "raw_orders".to_owned()); + let (selected, edges) = expand_loaders( + &graph, + vec![root.clone()], + Vec::new(), + &HashSet::from([root]), + ); + assert_eq!( + selected.len(), + test_case.expected_count, + "{}", + test_case.description + ); + assert_eq!( + edges.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs new file mode 100644 index 0000000000..f3f3d6c266 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs @@ -0,0 +1,15 @@ +//! Loader graph test cases. + +#[derive(Debug)] +pub(crate) struct LoaderTestCase { + pub description: &'static str, + pub executable: Vec<&'static str>, + pub expected_selected: Vec<&'static str>, +} + +#[derive(Debug)] +pub(crate) struct DeepLoaderTestCase { + pub description: &'static str, + pub depth: usize, + pub expected_count: usize, +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs new file mode 100644 index 0000000000..aab0ae3d7c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod unit_test_tokens; diff --git a/crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs b/crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs new file mode 100644 index 0000000000..b1c6182885 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs @@ -0,0 +1,164 @@ +//! Classify selector groups, tokens and intersection parts as unit-test or resource selectors. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::graph::errors::SelectorError; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::main::selector_name_help::selector_name_help; +use crate::graph::models::ParsedSelector; +use crate::planner_selection::constants::{ + INTERSECTION_SEPARATOR, NAME_KIND, PATTERN_CHARACTERS, TEST_KIND, TOKEN_SEPARATOR, + UNIT_TEST_SELECTOR_CODE, UNKNOWN_SELECTOR_CODE, +}; +use crate::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; +use crate::planner_selection::models::UnitTestSelectorNames; + +/// The unit tests a selector names and whether it also selects resources. +type Classified = (BTreeSet, bool); + +/// The unit tests the groups select and the groups left with their unit-test tokens removed. +pub(crate) fn split_groups( + groups: &[String], + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result<(BTreeSet, Vec), SelectorError> { + let mut tests: BTreeSet = BTreeSet::new(); + let mut kept_groups: Vec = Vec::new(); + for group in groups { + let tokens: Vec<&str> = group + .split(char::is_whitespace) + .filter(|token| !token.is_empty()) + .collect(); + let mut kept_tokens: Vec<&str> = Vec::new(); + for token in &tokens { + let (matched, keep): Classified = split_token(token, names, accepts_unit_tests)?; + tests.extend(matched); + if keep { + kept_tokens.push(token); + } + } + if kept_tokens.len() == tokens.len() { + kept_groups.push(group.clone()); + } else if !kept_tokens.is_empty() { + kept_groups.push(kept_tokens.join(TOKEN_SEPARATOR)); + } + } + Ok((tests, kept_groups)) +} + +fn split_token( + token: &str, + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result { + let parts: Vec<&str> = token.split(INTERSECTION_SEPARATOR).collect(); + let mut classified: Vec = Vec::with_capacity(parts.len()); + for part in &parts { + classified.push(classify(part, names, accepts_unit_tests)?); + } + let selects_tests: bool = classified + .iter() + .any(|(matched, keep)| !matched.is_empty() && !keep); + if parts.len() > 1 && selects_tests { + for part in &parts { + parse_selector(part)?; + } + return Err(SelectorError { + code: UNIT_TEST_SELECTOR_CODE, + message: format!( + "selector '{token}' intersects a unit test with ','; unit tests can only be selected whole, by name" + ), + help: None, + }); + } + if parts.len() > 1 { + return Ok((BTreeSet::new(), true)); + } + Ok(classified + .into_iter() + .next() + .unwrap_or((BTreeSet::new(), true))) +} + +fn classify( + part: &str, + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result { + let Ok(ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + }) = parse_selector(part) + else { + return Ok((BTreeSet::new(), true)); + }; + if kind != NAME_KIND && kind != TEST_KIND { + return Ok((BTreeSet::new(), true)); + } + let matched_tests: BTreeSet = matching(&value, &names.tests); + if kind == NAME_KIND && !matching(&value, &names.resources).is_empty() { + return Ok((BTreeSet::new(), true)); + } + if matched_tests.is_empty() { + if kind == TEST_KIND || accepts_unit_tests { + return Err(unknown(&kind, &value, names)); + } + return Ok((BTreeSet::new(), true)); + } + if !accepts_unit_tests { + return Err(unit_test_selector_rejection(part)); + } + if upstream || downstream { + return Err(SelectorError { + code: UNIT_TEST_SELECTOR_CODE, + message: format!( + "selector '{part}' expands a unit test with '+'; unit tests have no lineage, so select them by name without '+'" + ), + help: None, + }); + } + Ok((matched_tests, false)) +} + +fn unknown(kind: &str, value: &str, names: &UnitTestSelectorNames) -> SelectorError { + let label: &str = if is_pattern(value) { "pattern" } else { "name" }; + let (subject, candidates): (&str, Vec<&str>) = if kind == NAME_KIND { + ( + "selector", + names + .tests + .union(&names.resources) + .map(String::as_str) + .collect(), + ) + } else { + ( + "unit test", + names.tests.iter().map(String::as_str).collect(), + ) + }; + SelectorError { + code: UNKNOWN_SELECTOR_CODE, + message: format!("unknown {subject} {label} '{value}'"), + help: selector_name_help(value, candidates), + } +} + +fn matching(value: &str, names: &BTreeSet) -> BTreeSet { + if !is_pattern(value) { + return names.get(value).into_iter().cloned().collect(); + } + names + .iter() + .filter(|name| glob_match(name, value)) + .cloned() + .collect() +} + +fn is_pattern(value: &str) -> bool { + value.contains(PATTERN_CHARACTERS) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/constants.rs b/crates/sqlbuild-analysis/src/planner_selection/constants.rs new file mode 100644 index 0000000000..ea8ed27ae0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/constants.rs @@ -0,0 +1,16 @@ +//! Planner selection codes, kinds and messages. + +pub const UNIT_TEST_SELECTOR_CODE: &str = "S013"; +pub const UNKNOWN_SELECTOR_CODE: &str = "S007"; +pub const UNIT_TEST_ONLY_TEST_AND_BUILD: &str = + "only `sqb test` and `sqb build` accept unit-test selectors"; +pub(crate) const NAME_KIND: &str = "name"; +pub(crate) const TEST_KIND: &str = "test"; +pub(crate) const INTERSECTION_SEPARATOR: char = ','; +pub(crate) const TOKEN_SEPARATOR: &str = " "; +pub(crate) const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; +pub(crate) const EXPANSION_MARKER: char = '+'; +pub(crate) const TAG_PREFIX: &str = "tag:"; +pub(crate) const PATH_PREFIX: &str = "path:"; +pub(crate) const FOLDER_SEPARATOR: char = '/'; +pub(crate) const WINDOWS_SEPARATOR: char = '\\'; diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/mod.rs b/crates/sqlbuild-analysis/src/planner_selection/main/mod.rs new file mode 100644 index 0000000000..23a007902f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/mod.rs @@ -0,0 +1,4 @@ +pub mod split_selector_expansion; +pub mod split_unit_test_selectors; +pub mod sqlbuild_model_selector_names; +pub mod unit_test_selector_rejection; diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs b/crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs new file mode 100644 index 0000000000..d9a01f8711 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs @@ -0,0 +1,30 @@ +//! Split `+selector+` marker syntax from the selector core without resolving the core. + +use crate::planner_selection::constants::EXPANSION_MARKER; +use crate::planner_selection::models::SelectorExpansion; + +/// The core and expansion flags, or the input error message. +pub fn split_selector_expansion(raw: &str) -> Result { + let stripped: &str = raw.trim_matches(char::is_whitespace); + if stripped.is_empty() { + return Err("empty selector".to_owned()); + } + let core: &str = stripped + .trim_start_matches(EXPANSION_MARKER) + .trim_end_matches(EXPANSION_MARKER); + if core.is_empty() { + return Err(format!( + "selector '{stripped}' has no name after removing '+' markers" + )); + } + if core.contains(EXPANSION_MARKER) { + return Err(format!( + "selector '{stripped}' contains '+' in an unsupported position" + )); + } + Ok(SelectorExpansion { + core: core.to_owned(), + upstream: stripped.starts_with(EXPANSION_MARKER), + downstream: stripped.ends_with(EXPANSION_MARKER), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs b/crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs new file mode 100644 index 0000000000..b2c9ca72a3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs @@ -0,0 +1,30 @@ +//! Move unit-test selectors out of resource selectors, or reject them for this command. + +use std::collections::BTreeSet; + +use crate::graph::errors::SelectorError; +use crate::planner_selection::_helpers::unit_test_tokens::split_groups; +use crate::planner_selection::models::{ + SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, +}; + +/// The resource selectors left and the unit tests selected or excluded by name. +pub fn split_unit_test_selectors( + groups: SelectorGroups<'_>, + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result { + let (test_names, select): (BTreeSet, Vec) = + split_groups(groups.select, names, accepts_unit_tests)?; + let (excluded_test_names, exclude): (BTreeSet, Vec) = + split_groups(groups.exclude, names, accepts_unit_tests)?; + let models_selected: bool = + groups.select.is_empty() || !select.is_empty() || test_names.is_empty(); + Ok(UnitTestSelectorSplit { + select, + exclude, + test_names, + excluded_test_names, + models_selected, + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs b/crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs new file mode 100644 index 0000000000..2ccd2525bf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs @@ -0,0 +1,43 @@ +//! Resolve one SQLBuild-owned model selector core to model names. + +use crate::planner_selection::constants::{ + FOLDER_SEPARATOR, PATH_PREFIX, TAG_PREFIX, WINDOWS_SEPARATOR, +}; +use crate::planner_selection::models::{ModelSelectorNames, SelectableModel}; + +/// A model name, `tag:` or `path:` term's model names in model order; other terms match nothing. +pub fn sqlbuild_model_selector_names(models: &[SelectableModel], term: &str) -> ModelSelectorNames { + if models.iter().any(|model| model.name == term) { + return ModelSelectorNames { + names: vec![term.to_owned()], + translated: None, + }; + } + if let Some(tag) = term.strip_prefix(TAG_PREFIX) { + return ModelSelectorNames { + names: models + .iter() + .filter(|model| has_tag(model, tag)) + .map(|model| model.name.clone()) + .collect(), + translated: None, + }; + } + let Some(raw_path) = term.strip_prefix(PATH_PREFIX) else { + return ModelSelectorNames::default(); + }; + let path: String = raw_path.replace(WINDOWS_SEPARATOR, &FOLDER_SEPARATOR.to_string()); + let nested: String = format!("{path}{FOLDER_SEPARATOR}"); + ModelSelectorNames { + names: models + .iter() + .filter(|model| model.folder == path || model.folder.starts_with(&nested)) + .map(|model| model.name.clone()) + .collect(), + translated: (path != raw_path).then(|| format!("{PATH_PREFIX}{path}")), + } +} + +fn has_tag(model: &SelectableModel, tag: &str) -> bool { + model.tags.iter().any(|candidate| candidate == tag) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs b/crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs new file mode 100644 index 0000000000..34383c5175 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs @@ -0,0 +1,15 @@ +//! The error for a unit-test selector given to a command that does not run unit tests. + +use crate::graph::errors::SelectorError; +use crate::planner_selection::constants::{UNIT_TEST_ONLY_TEST_AND_BUILD, UNIT_TEST_SELECTOR_CODE}; + +/// The coded rejection naming the selector. +pub fn unit_test_selector_rejection(selector: &str) -> SelectorError { + SelectorError { + code: UNIT_TEST_SELECTOR_CODE, + message: format!( + "selector '{selector}' selects a unit test; {UNIT_TEST_ONLY_TEST_AND_BUILD}" + ), + help: None, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/mod.rs b/crates/sqlbuild-analysis/src/planner_selection/mod.rs new file mode 100644 index 0000000000..53f788143d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/mod.rs @@ -0,0 +1,9 @@ +//! Planner selection boundary: unit-test selector splitting, `+` marker splitting and SQLBuild-owned model selectors. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_selection/models.rs b/crates/sqlbuild-analysis/src/planner_selection/models.rs new file mode 100644 index 0000000000..0d757a4f42 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/models.rs @@ -0,0 +1,50 @@ +//! Planner selection inputs and results. + +use std::collections::BTreeSet; + +/// Names selectors can match: unit tests, and every other selectable resource. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UnitTestSelectorNames { + pub tests: BTreeSet, + pub resources: BTreeSet, +} + +/// Raw `--select` and `--exclude` groups. +#[derive(Clone, Copy, Debug)] +pub struct SelectorGroups<'a> { + pub select: &'a [String], + pub exclude: &'a [String], +} + +/// Resource selectors left after unit-test selectors moved into a test selection. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UnitTestSelectorSplit { + pub select: Vec, + pub exclude: Vec, + pub test_names: BTreeSet, + pub excluded_test_names: BTreeSet, + pub models_selected: bool, +} + +/// One selector split into `+` expansion flags and core text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectorExpansion { + pub core: String, + pub upstream: bool, + pub downstream: bool, +} + +/// A model as SQLBuild-owned model selectors see it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectableModel { + pub name: String, + pub tags: Vec, + pub folder: String, +} + +/// Names one SQLBuild model selector resolves to, and its translated path selector when it changed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelSelectorNames { + pub names: Vec, + pub translated: Option, +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests.rs b/crates/sqlbuild-analysis/src/planner_selection/tests.rs new file mode 100644 index 0000000000..753a6cec33 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_selection.rs"] +mod planner_selection; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs new file mode 100644 index 0000000000..9a83291566 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs @@ -0,0 +1,57 @@ +use std::collections::BTreeSet; + +use crate::graph::errors::SelectorError; +use crate::planner_selection::models::{ + SelectableModel, UnitTestSelectorNames, UnitTestSelectorSplit, +}; + +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(super) fn name_set(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Unit tests `test_orders` and `test_customers`; resources `orders` and `customers`. +pub(super) fn selector_names() -> UnitTestSelectorNames { + UnitTestSelectorNames { + tests: name_set(&["test_customers", "test_orders"]), + resources: name_set(&["customers", "orders"]), + } +} + +pub(super) fn split( + select: &[&str], + tests: &[&str], + models_selected: bool, +) -> UnitTestSelectorSplit { + UnitTestSelectorSplit { + select: texts(select), + exclude: Vec::new(), + test_names: name_set(tests), + excluded_test_names: BTreeSet::new(), + models_selected, + } +} + +pub(super) fn failure(code: &'static str, message: &str, help: Option<&str>) -> SelectorError { + SelectorError { + code, + message: message.to_owned(), + help: help.map(str::to_owned), + } +} + +pub(super) fn models() -> Vec { + let model = |name: &str, tags: &[&str], folder: &str| SelectableModel { + name: name.to_owned(), + tags: texts(tags), + folder: folder.to_owned(), + }; + vec![ + model("orders", &["daily"], "models/staging"), + model("customers", &[], "models/staging/sub"), + model("report", &["daily"], "models/marts"), + ] +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs b/crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs new file mode 100644 index 0000000000..80b6a0c0b2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs @@ -0,0 +1,174 @@ +use crate::planner_selection::main::split_selector_expansion::split_selector_expansion; +use crate::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; +use crate::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; +use crate::planner_selection::models::{ModelSelectorNames, SelectorExpansion, SelectorGroups}; +use crate::planner_selection::tests::helpers::{failure, models, selector_names, split, texts}; +use crate::planner_selection::tests::test_types::{ + ExpansionTestCase, ModelSelectorTestCase, UnitTestSplitTestCase, +}; + +#[test] +fn given_selector_groups_when_splitting_unit_tests_then_moves_or_rejects_them() { + let test_cases = [ + UnitTestSplitTestCase { + description: "unit test token leaves the resource token in its group", + select: texts(&["test_orders customers"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Ok(split(&["customers"], &["test_orders"], true)), + }, + UnitTestSplitTestCase { + description: "pattern selects every matching unit test and no model", + select: texts(&["test_*"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Ok(split(&[], &["test_customers", "test_orders"], false)), + }, + UnitTestSplitTestCase { + description: "resource names win over unit tests", + select: texts(&["orders"]), + exclude: Vec::new(), + accepts_unit_tests: false, + expected_split: Ok(split(&["orders"], &[], true)), + }, + UnitTestSplitTestCase { + description: "commands without unit tests reject them", + select: texts(&["test_orders"]), + exclude: Vec::new(), + accepts_unit_tests: false, + expected_split: Err(failure( + "S013", + "selector 'test_orders' selects a unit test; only `sqb test` and `sqb build` accept unit-test selectors", + None, + )), + }, + UnitTestSplitTestCase { + description: "intersections cannot include unit tests", + select: texts(&["test_orders,orders"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Err(failure( + "S013", + "selector 'test_orders,orders' intersects a unit test with ','; unit tests can only be selected whole, by name", + None, + )), + }, + UnitTestSplitTestCase { + description: "unit tests have no lineage", + select: texts(&["+test_orders"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Err(failure( + "S013", + "selector '+test_orders' expands a unit test with '+'; unit tests have no lineage, so select them by name without '+'", + None, + )), + }, + UnitTestSplitTestCase { + description: "unknown unit test suggests the closest name", + select: Vec::new(), + exclude: texts(&["test:test_order"]), + accepts_unit_tests: false, + expected_split: Err(failure( + "S007", + "unknown unit test name 'test_order'", + Some("did you mean 'test_orders'?"), + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + split_unit_test_selectors( + SelectorGroups { + select: &test_case.select, + exclude: &test_case.exclude, + }, + &selector_names(), + test_case.accepts_unit_tests, + ), + test_case.expected_split, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_raw_selector_when_splitting_expansion_then_returns_core_and_flags() { + let test_cases = [ + ExpansionTestCase { + description: "both markers", + raw: " +orders+ ", + expected_expansion: Ok(SelectorExpansion { + core: "orders".to_owned(), + upstream: true, + downstream: true, + }), + }, + ExpansionTestCase { + description: "markers only", + raw: "++", + expected_expansion: Err( + "selector '++' has no name after removing '+' markers".to_owned() + ), + }, + ExpansionTestCase { + description: "inner marker", + raw: "a+b", + expected_expansion: Err( + "selector 'a+b' contains '+' in an unsupported position".to_owned() + ), + }, + ExpansionTestCase { + description: "blank", + raw: " ", + expected_expansion: Err("empty selector".to_owned()), + }, + ]; + for test_case in test_cases { + assert_eq!( + split_selector_expansion(test_case.raw), + test_case.expected_expansion, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_selector_term_when_resolving_then_returns_model_names() { + let names = |values: &[&str], translated: Option<&str>| ModelSelectorNames { + names: texts(values), + translated: translated.map(str::to_owned), + }; + let test_cases = [ + ModelSelectorTestCase { + description: "model name", + term: "report", + expected_names: names(&["report"], None), + }, + ModelSelectorTestCase { + description: "tag in model order", + term: "tag:daily", + expected_names: names(&["orders", "report"], None), + }, + ModelSelectorTestCase { + description: "windows path is translated and matches nested folders", + term: "path:models\\staging", + expected_names: names(&["orders", "customers"], Some("path:models/staging")), + }, + ModelSelectorTestCase { + description: "other terms", + term: "source:raw", + expected_names: names(&[], None), + }, + ]; + for test_case in test_cases { + assert_eq!( + sqlbuild_model_selector_names(&models(), test_case.term), + test_case.expected_names, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs new file mode 100644 index 0000000000..9c626c1f7b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs @@ -0,0 +1,24 @@ +use crate::graph::errors::SelectorError; +use crate::planner_selection::models::{ + ModelSelectorNames, SelectorExpansion, UnitTestSelectorSplit, +}; + +pub(super) struct UnitTestSplitTestCase { + pub(super) description: &'static str, + pub(super) select: Vec, + pub(super) exclude: Vec, + pub(super) accepts_unit_tests: bool, + pub(super) expected_split: Result, +} + +pub(super) struct ExpansionTestCase { + pub(super) description: &'static str, + pub(super) raw: &'static str, + pub(super) expected_expansion: Result, +} + +pub(super) struct ModelSelectorTestCase { + pub(super) description: &'static str, + pub(super) term: &'static str, + pub(super) expected_names: ModelSelectorNames, +} diff --git a/crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs b/crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs new file mode 100644 index 0000000000..87c481a3a7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod path_text; diff --git a/crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs b/crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs new file mode 100644 index 0000000000..40e5daa881 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs @@ -0,0 +1,21 @@ +//! `pathlib` suffix handling on POSIX path text. + +/// `PurePath.with_suffix(suffix)`: the last suffix of the final component replaced or added. +pub(crate) fn with_suffix(path: &str, suffix: &str) -> String { + let start: usize = path.rfind('/').map_or(0, |index| index + 1); + let name: &str = &path[start..]; + let stem_length: usize = match name.rfind('.') { + Some(index) if index > 0 && index < name.len() - 1 => index, + _ => name.len(), + }; + format!("{}{}{suffix}", &path[..start], &name[..stem_length]) +} + +/// The first `max_bytes` bytes of `value`, without a trailing partial character. +pub(crate) fn truncate_utf8(value: &str, max_bytes: usize) -> &str { + let mut end: usize = max_bytes.min(value.len()); + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} diff --git a/crates/sqlbuild-analysis/src/target_paths/constants.rs b/crates/sqlbuild-analysis/src/target_paths/constants.rs new file mode 100644 index 0000000000..99f4861abc --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/constants.rs @@ -0,0 +1,11 @@ +//! Directory names and limits of the paths SQLBuild writes under `target/`. + +pub(crate) const CHAIN_DIR: &str = "_chain_"; +pub(crate) const SQL_FILE_SUFFIX: &str = ".sql"; +/// A model chain directory name longer than this many bytes is shortened to a digest. +pub(crate) const MAX_CHAIN_COMPONENT_BYTES: usize = 200; +pub(crate) const CHAIN_DIGEST_LENGTH: usize = 12; +/// Bytes kept from the first and last model names of a shortened chain directory. +pub(crate) const CHAIN_EDGE_NAME_BYTES: usize = 80; +pub(crate) const MODELS_DIR: &str = "models"; +pub(crate) const FUNCTIONS_DIR: &str = "functions"; diff --git a/crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs b/crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs new file mode 100644 index 0000000000..6b46b65f1b --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs @@ -0,0 +1,16 @@ +//! Where a function's executed SQL is written under `target/run`. + +use crate::target_paths::_helpers::path_text::with_suffix; +use crate::target_paths::constants::{FUNCTIONS_DIR, SQL_FILE_SUFFIX}; + +/// The function's path under `functions//`, with a `.sql` suffix. +#[must_use] +pub fn function_run_path(relative_path: &str, language: &str) -> String { + let mut parts = relative_path.split('/'); + let path: String = if parts.next() == Some(FUNCTIONS_DIR) && parts.next() == Some(language) { + relative_path.to_owned() + } else { + format!("{FUNCTIONS_DIR}/{language}/{relative_path}") + }; + with_suffix(&path, SQL_FILE_SUFFIX) +} diff --git a/crates/sqlbuild-analysis/src/target_paths/main/mod.rs b/crates/sqlbuild-analysis/src/target_paths/main/mod.rs new file mode 100644 index 0000000000..43cc006909 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/mod.rs @@ -0,0 +1,4 @@ +pub mod function_run_path; +pub mod model_run_path; +pub mod runtime_sql_text; +pub mod sql_test_output_path; diff --git a/crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs b/crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs new file mode 100644 index 0000000000..1b65f25dd1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs @@ -0,0 +1,12 @@ +//! Where a model's executed SQL is written under `target/run`. + +use crate::target_paths::constants::MODELS_DIR; + +/// The model's relative path, under `models/` unless it already starts there. +#[must_use] +pub fn model_run_path(relative_path: &str) -> String { + if relative_path.split('/').next() == Some(MODELS_DIR) { + return relative_path.to_owned(); + } + format!("{MODELS_DIR}/{relative_path}") +} diff --git a/crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs b/crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs new file mode 100644 index 0000000000..2d0a3d51f1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs @@ -0,0 +1,23 @@ +//! The text of a runtime SQL artifact: its statements, each terminated. + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +/// The statements joined by blank lines, each ending in `;`, with one final newline. +#[must_use] +pub fn runtime_sql_text(statements: &[String]) -> String { + let joined: String = statements + .iter() + .map(|statement| { + let stripped: &str = statement.trim_end_matches(python_isspace); + if stripped.is_empty() { + statement.clone() + } else if stripped.ends_with(';') { + stripped.to_owned() + } else { + format!("{stripped};") + } + }) + .collect::>() + .join("\n\n"); + format!("{}\n", joined.trim_end_matches(python_isspace)) +} diff --git a/crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs b/crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs new file mode 100644 index 0000000000..d615afc011 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs @@ -0,0 +1,54 @@ +//! The readable, filesystem-safe path of one SQL test artifact. + +use sha2::{Digest, Sha256}; + +use crate::target_paths::_helpers::path_text::{truncate_utf8, with_suffix}; +use crate::target_paths::constants::{ + CHAIN_DIGEST_LENGTH, CHAIN_DIR, CHAIN_EDGE_NAME_BYTES, MAX_CHAIN_COMPONENT_BYTES, + SQL_FILE_SUFFIX, +}; +use crate::target_paths::models::SqlTestPathFacts; + +fn chain_folder(name: &str, model_names: &[String]) -> String { + let mut unique: Vec<&str> = model_names.iter().map(String::as_str).collect(); + unique.sort_unstable(); + unique.dedup(); + match unique.as_slice() { + [] => return name.to_owned(), + [only] => return (*only).to_owned(), + _ => {} + } + let chain: String = unique.join("__"); + if chain.len() <= MAX_CHAIN_COMPONENT_BYTES { + return format!("{CHAIN_DIR}/{chain}"); + } + let digest: String = Sha256::digest(chain.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::()[..CHAIN_DIGEST_LENGTH] + .to_owned(); + format!( + "{CHAIN_DIR}/{}__{}__{digest}", + truncate_utf8(unique[0], CHAIN_EDGE_NAME_BYTES), + truncate_utf8(unique[unique.len() - 1], CHAIN_EDGE_NAME_BYTES) + ) +} + +/// The test's chain folder and name, or its source stem and case for an expanded case. +#[must_use] +pub fn sql_test_output_path(facts: &SqlTestPathFacts) -> String { + match (&facts.case_name, &facts.source_path) { + (Some(case_name), Some(source_path)) => format!( + "{}/block_{}__{case_name}{SQL_FILE_SUFFIX}", + with_suffix(source_path, ""), + facts + .block_index + .map_or_else(|| "None".to_owned(), |index| index.to_string()) + ), + _ => format!( + "{}/{}{SQL_FILE_SUFFIX}", + chain_folder(&facts.name, &facts.model_names), + facts.name + ), + } +} diff --git a/crates/sqlbuild-analysis/src/target_paths/mod.rs b/crates/sqlbuild-analysis/src/target_paths/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/target_paths/models.rs b/crates/sqlbuild-analysis/src/target_paths/models.rs new file mode 100644 index 0000000000..3b5ce2205d --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/models.rs @@ -0,0 +1,13 @@ +//! The SQL test facts its target path is built from. + +/// One SQL test or expanded case, as its target path needs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqlTestPathFacts { + pub name: String, + /// The authoring file, relative and in POSIX form. + pub source_path: Option, + pub block_index: Option, + pub case_name: Option, + /// The models of the test's chain, in any order and possibly repeated. + pub model_names: Vec, +} diff --git a/crates/sqlbuild-analysis/src/target_paths/tests.rs b/crates/sqlbuild-analysis/src/target_paths/tests.rs new file mode 100644 index 0000000000..a27406ab34 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_target_paths.rs"] +mod target_paths; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs b/crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs new file mode 100644 index 0000000000..f6f1ea0e23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs @@ -0,0 +1,15 @@ +use crate::target_paths::models::SqlTestPathFacts; + +pub(super) fn facts( + name: &str, + case: Option<(&str, Option, &str)>, + models: &[&str], +) -> SqlTestPathFacts { + SqlTestPathFacts { + name: name.to_owned(), + source_path: case.map(|(source, _, _)| source.to_owned()), + block_index: case.and_then(|(_, block, _)| block), + case_name: case.map(|(_, _, case_name)| case_name.to_owned()), + model_names: models.iter().map(|model| (*model).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs b/crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs new file mode 100644 index 0000000000..2d8f49a8c8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs @@ -0,0 +1,129 @@ +use crate::target_paths::main::function_run_path::function_run_path; +use crate::target_paths::main::model_run_path::model_run_path; +use crate::target_paths::main::runtime_sql_text::runtime_sql_text; +use crate::target_paths::main::sql_test_output_path::sql_test_output_path; +use crate::target_paths::tests::helpers::facts; +use crate::target_paths::tests::test_types::{ + RunPathTestCase, RuntimeSqlTestCase, SqlTestPathTestCase, +}; + +/// Expectations recorded from the retired Python path builders. +#[test] +fn given_sql_test_when_building_output_path_then_matches_python() { + let long: Vec = (0..9) + .map(|index| format!("m\u{e9}tric_{index:02}_xxxxxxxxxxxxxxxxxxxx")) + .collect(); + let last: String = format!("{}a", "\u{e9}".repeat(50)); + let mut long_chain: Vec<&str> = vec![last.as_str()]; + long_chain.extend(long.iter().map(String::as_str)); + let test_cases = [ + SqlTestPathTestCase { + description: "test without models uses its own name", + facts: facts("t1", None, &[]), + expected_path: "t1/t1.sql", + }, + SqlTestPathTestCase { + description: "one repeated model names the folder", + facts: facts("t2", None, &["orders", "orders"]), + expected_path: "orders/t2.sql", + }, + SqlTestPathTestCase { + description: "a chain is sorted and deduplicated", + facts: facts("t3", None, &["zeta", "alpha", "zeta"]), + expected_path: "_chain_/alpha__zeta/t3.sql", + }, + SqlTestPathTestCase { + description: "a long chain keeps truncated edge names and a digest", + facts: facts("t4", None, &long_chain), + expected_path: "_chain_/m\u{e9}tric_00_xxxxxxxxxxxxxxxxxxxx__\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}__132c41e73f1c/t4.sql", + }, + SqlTestPathTestCase { + description: "an expanded case lives under its source stem", + facts: facts( + "t5", + Some(("tests/orders.sql", Some(2), "happy")), + &["orders"], + ), + expected_path: "tests/orders/block_2__happy.sql", + }, + SqlTestPathTestCase { + description: "a dot file keeps its name and a missing block prints None", + facts: facts("t6", Some(("tests/.hidden", None, "c")), &[]), + expected_path: "tests/.hidden/block_None__c.sql", + }, + ]; + + for test_case in test_cases { + assert_eq!( + sql_test_output_path(&test_case.facts), + test_case.expected_path, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_path_when_building_run_paths_then_matches_python() { + let test_cases = [ + RunPathTestCase { + description: "paths already under their directory are kept", + relative_path: "models/a/b.sql", + language: "sql", + expected_model_path: "models/a/b.sql", + expected_function_path: "functions/sql/models/a/b.sql", + }, + RunPathTestCase { + description: "other paths are nested and functions get a sql suffix", + relative_path: "udfs/f.py", + language: "python", + expected_model_path: "models/udfs/f.py", + expected_function_path: "functions/python/udfs/f.sql", + }, + RunPathTestCase { + description: "a function under its language directory is kept", + relative_path: "functions/sql/f.sql", + language: "sql", + expected_model_path: "models/functions/sql/f.sql", + expected_function_path: "functions/sql/f.sql", + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + model_run_path(test_case.relative_path), + function_run_path(test_case.relative_path, test_case.language) + ), + ( + test_case.expected_model_path.to_owned(), + test_case.expected_function_path.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_statements_when_rendering_runtime_sql_then_matches_python() { + let test_cases = [RuntimeSqlTestCase { + description: "statements are terminated and blank ones kept", + statements: vec![ + "select 1 ".to_owned(), + "create table x as select 2;\n".to_owned(), + " ".to_owned(), + "drop view y\u{1f}".to_owned(), + ], + expected_text: "select 1;\n\ncreate table x as select 2;\n\n \n\ndrop view y;\n", + }]; + + for test_case in test_cases { + assert_eq!( + runtime_sql_text(&test_case.statements), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs b/crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs new file mode 100644 index 0000000000..0a90265759 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs @@ -0,0 +1,21 @@ +use crate::target_paths::models::SqlTestPathFacts; + +pub(super) struct SqlTestPathTestCase { + pub(super) description: &'static str, + pub(super) facts: SqlTestPathFacts, + pub(super) expected_path: &'static str, +} + +pub(super) struct RunPathTestCase { + pub(super) description: &'static str, + pub(super) relative_path: &'static str, + pub(super) language: &'static str, + pub(super) expected_model_path: &'static str, + pub(super) expected_function_path: &'static str, +} + +pub(super) struct RuntimeSqlTestCase { + pub(super) description: &'static str, + pub(super) statements: Vec, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 725f0cea3a..ad78844d34 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -143,9 +143,9 @@ const PLAN_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const PLAN_ROUTE: Route = routed( "run_plan", - ("sqlbuild.cli.commands.main.project._plan", "run_plan"), + ("sqlbuild._native", "run_plan_command"), &PLAN_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const BUILD_FIELDS: [Field; 34] = [ @@ -699,12 +699,9 @@ const RULES_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const RULES_ROUTE: Route = routed( "run_rules", - ( - "sqlbuild.cli.commands.main.project._rules", - "run_rules_command", - ), + ("sqlbuild._native", "run_rules_command"), &RULES_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const DBT_INIT_FIELDS: [Field; 9] = [ @@ -802,3 +799,6 @@ pub(crate) const SCENARIO_CAPTURE_ROUTE: Route = routed( &SCENARIO_CAPTURE_ARGUMENTS, REQUEST_KEYWORD_SHAPE, ); + +/// The setting a user raises when lifecycle event export runs out of time at exit. +pub(crate) const SHUTDOWN_TIMEOUT_SETTING: &str = "sinks.lifecycle.shutdown_timeout"; diff --git a/crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs b/crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs new file mode 100644 index 0000000000..365528aa8f --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs @@ -0,0 +1,38 @@ +//! The stderr warning for lifecycle events dropped or failed at command exit. + +use crate::dispatch::constants::SHUTDOWN_TIMEOUT_SETTING; +use crate::dispatch::models::{ExportSummary, ExporterLoss}; + +/// One warning line when lifecycle events were dropped or failed, else `None`. +#[must_use] +pub fn event_export_warning(summary: &ExportSummary, command_succeeded: bool) -> Option { + if summary.dropped == 0 && summary.failed == 0 { + return None; + } + let affected: Vec<&ExporterLoss> = summary + .per_exporter + .iter() + .filter(|exporter| exporter.dropped > 0 || exporter.failed > 0) + .collect(); + let names: String = affected + .iter() + .map(|exporter| format!("'{}'", exporter.exporter_name)) + .collect::>() + .join(", "); + let sink_label: &str = if affected.len() == 1 { "sink" } else { "sinks" }; + let unit: &str = if summary.per_exporter.len() == 1 { + "events" + } else { + "event deliveries" + }; + let prefix: &str = if command_succeeded { + "Warning: command completed successfully, but lifecycle event export was incomplete" + } else { + "Warning: lifecycle event export was incomplete" + }; + Some(format!( + "{prefix}: {} of {} {unit} dropped, {} failed ({sink_label} {names}). \ + Increase {SHUTDOWN_TIMEOUT_SETTING} or check sink health.", + summary.dropped, summary.accepted, summary.failed + )) +} diff --git a/crates/sqlbuild-cli/src/dispatch/main/mod.rs b/crates/sqlbuild-cli/src/dispatch/main/mod.rs index 73ddbae24f..d96f4ca211 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/mod.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/mod.rs @@ -1,4 +1,5 @@ pub mod command_dispatch; +pub mod event_export_warning; pub mod failure_report; pub mod invocation_policy; pub mod unavailable_handler; diff --git a/crates/sqlbuild-cli/src/dispatch/models.rs b/crates/sqlbuild-cli/src/dispatch/models.rs index eafaee9839..87230c9bf4 100644 --- a/crates/sqlbuild-cli/src/dispatch/models.rs +++ b/crates/sqlbuild-cli/src/dispatch/models.rs @@ -142,3 +142,20 @@ pub enum Dispatch { /// Commands with no handler exit successfully. Nothing, } + +/// Lifecycle event export accounting of one sink at command exit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExporterLoss { + pub exporter_name: String, + pub dropped: u64, + pub failed: u64, +} + +/// Lifecycle event export accounting of every sink at command exit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExportSummary { + pub accepted: u64, + pub dropped: u64, + pub failed: u64, + pub per_exporter: Vec, +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests.rs b/crates/sqlbuild-cli/src/dispatch/tests.rs index a857d5eccd..2844cc4653 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_command_dispatch.rs"] mod command_dispatch; +#[path = "tests/test_event_export_warning.rs"] +mod event_export_warning; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_invocation_policy.rs"] diff --git a/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs b/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs index ac0c7c9e38..dddc39e942 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs @@ -1,4 +1,6 @@ -use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; +use crate::dispatch::models::{ + CommandFacts, Dispatch, DispatchFailure, ExportSummary, ExporterLoss, Route, +}; /// Facts of `command` and `subcommand` with every flag off and every required value present. pub(super) fn facts(command: &str, subcommand: Option<&str>) -> CommandFacts { @@ -25,3 +27,20 @@ pub(super) fn failure(message: &str, code: &'static str) -> DispatchFailure { help: None, } } + +/// An export summary of `(name, accepted, delivered, failed, dropped)` sinks. +pub(super) fn export_summary(exporters: &[(&str, u64, u64, u64, u64)]) -> ExportSummary { + ExportSummary { + accepted: exporters.iter().map(|exporter| exporter.1).sum(), + dropped: exporters.iter().map(|exporter| exporter.4).sum(), + failed: exporters.iter().map(|exporter| exporter.3).sum(), + per_exporter: exporters + .iter() + .map(|exporter| ExporterLoss { + exporter_name: exporter.0.to_owned(), + dropped: exporter.4, + failed: exporter.3, + }) + .collect(), + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs new file mode 100644 index 0000000000..c799b4b25a --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs @@ -0,0 +1,56 @@ +use crate::dispatch::main::event_export_warning::event_export_warning; +use crate::dispatch::tests::helpers::export_summary; +use crate::dispatch::tests::test_types::EventExportWarningTestCase; + +#[test] +fn given_final_export_summary_when_formatting_warning_then_reports_loss_only() { + let test_cases = [ + EventExportWarningTestCase { + description: "all delivered", + summary: export_summary(&[("orders_events", 248, 248, 0, 0)]), + command_succeeded: true, + expected_warning: None, + }, + EventExportWarningTestCase { + description: "successful command with drops", + summary: export_summary(&[("orders_events", 248, 108, 0, 140)]), + command_succeeded: true, + expected_warning: Some( + "Warning: command completed successfully, but lifecycle event export was \ + incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). \ + Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + }, + EventExportWarningTestCase { + description: "failed command with failures", + summary: export_summary(&[("orders_events", 10, 9, 1, 0)]), + command_succeeded: false, + expected_warning: Some( + "Warning: lifecycle event export was incomplete: 0 of 10 events dropped, \ + 1 failed (sink 'orders_events'). \ + Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + }, + EventExportWarningTestCase { + description: "multiple sinks names only affected sinks", + summary: export_summary(&[ + ("customer_events", 20, 20, 0, 0), + ("orders_events", 20, 15, 1, 4), + ]), + command_succeeded: true, + expected_warning: Some( + "Warning: command completed successfully, but lifecycle event export was \ + incomplete: 4 of 40 event deliveries dropped, 1 failed (sink 'orders_events'). \ + Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + event_export_warning(&test_case.summary, test_case.command_succeeded).as_deref(), + test_case.expected_warning, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs index 2a20ac2140..a075ba8d89 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::dispatch::models::{CommandFacts, Dispatch, InvocationPolicy}; +use crate::dispatch::models::{CommandFacts, Dispatch, ExportSummary, InvocationPolicy}; pub(super) struct CommandDispatchTestCase { pub(super) description: &'static str, @@ -11,3 +11,10 @@ pub(super) struct InvocationPolicyTestCase { pub(super) facts: CommandFacts, pub(super) expected_policy: InvocationPolicy, } + +pub(super) struct EventExportWarningTestCase { + pub(super) description: &'static str, + pub(super) summary: ExportSummary, + pub(super) command_succeeded: bool, + pub(super) expected_warning: Option<&'static str>, +} diff --git a/crates/sqlbuild-cli/src/replay/_helpers/files.rs b/crates/sqlbuild-cli/src/replay/_helpers/files.rs new file mode 100644 index 0000000000..2f3dca3d61 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/_helpers/files.rs @@ -0,0 +1,61 @@ +//! Atomic launch record writes and pruning of the oldest records. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::replay::constants::{LAUNCH_RECORD_LIMIT, LAUNCH_RECORD_SUFFIX}; + +/// 32 random hex digits, as Python's `uuid.uuid4().hex`. +fn uuid4_hex() -> io::Result { + let mut bytes = [0_u8; 16]; + getrandom::fill(&mut bytes).map_err(|error| io::Error::other(error.to_string()))?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Ok(bytes.iter().map(|byte| format!("{byte:02x}")).collect()) +} + +/// Write `text` to `path` through a hidden temporary sibling replaced into place. +pub(crate) fn replace_file(path: &Path, text: &str) -> io::Result<()> { + let Some(parent) = path.parent() else { + return Err(io::Error::other("a launch record path has no directory")); + }; + fs::create_dir_all(parent)?; + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let temporary: PathBuf = path.with_file_name(format!(".{name}.{}.tmp", uuid4_hex()?)); + let written: io::Result<()> = + fs::write(&temporary, text.as_bytes()).and_then(|()| fs::rename(&temporary, path)); + let _ = fs::remove_file(&temporary); + written +} + +/// Delete all but the newest `LAUNCH_RECORD_LIMIT` records in `directory`. +pub(crate) fn prune_records(directory: &Path) -> io::Result<()> { + let mut records: Vec<(i128, String, PathBuf)> = Vec::new(); + for entry in fs::read_dir(directory)? { + let entry: fs::DirEntry = entry?; + let name: String = entry.file_name().to_string_lossy().into_owned(); + if !name.ends_with(LAUNCH_RECORD_SUFFIX) { + continue; + } + let metadata: fs::Metadata = fs::metadata(entry.path())?; + records.push((modified_ns(&metadata)?, name, entry.path())); + } + records.sort_by(|left, right| right.cmp(left)); + for (_, _, stale) in records.into_iter().skip(LAUNCH_RECORD_LIMIT) { + let _ = fs::remove_file(stale); + } + Ok(()) +} + +/// `os.stat(path).st_mtime_ns`. +fn modified_ns(metadata: &fs::Metadata) -> io::Result { + let modified: std::time::SystemTime = metadata.modified()?; + Ok(match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(after) => i128::try_from(after.as_nanos()).unwrap_or(i128::MAX), + Err(before) => -i128::try_from(before.duration().as_nanos()).unwrap_or(i128::MAX), + }) +} diff --git a/crates/sqlbuild-cli/src/replay/_helpers/mod.rs b/crates/sqlbuild-cli/src/replay/_helpers/mod.rs index 07f5c4f195..97182491a2 100644 --- a/crates/sqlbuild-cli/src/replay/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/replay/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod console; +pub(crate) mod files; pub(crate) mod records; pub(crate) mod serving; diff --git a/crates/sqlbuild-cli/src/replay/constants.rs b/crates/sqlbuild-cli/src/replay/constants.rs index 2ee2e8da61..7c34dbf8a1 100644 --- a/crates/sqlbuild-cli/src/replay/constants.rs +++ b/crates/sqlbuild-cli/src/replay/constants.rs @@ -6,3 +6,5 @@ pub const LAUNCH_RECORD_FORMAT: i64 = 1; pub const LAUNCH_RECORD_DIRECTORY: [&str; 3] = ["target", "cache", "launcher-v1"]; pub const LAUNCH_RECORD_SUFFIX: &str = ".json"; pub const OUTPUT_FAILURE_EXIT_CODE: i64 = 1; +/// How many launch records one project keeps; older ones are pruned after each write. +pub const LAUNCH_RECORD_LIMIT: usize = 32; diff --git a/crates/sqlbuild-cli/src/replay/main/mod.rs b/crates/sqlbuild-cli/src/replay/main/mod.rs index 6f3a35f1ad..f377c949dc 100644 --- a/crates/sqlbuild-cli/src/replay/main/mod.rs +++ b/crates/sqlbuild-cli/src/replay/main/mod.rs @@ -1 +1,2 @@ pub mod serve_reused_compile; +pub mod write_launch_record; diff --git a/crates/sqlbuild-cli/src/replay/main/write_launch_record.rs b/crates/sqlbuild-cli/src/replay/main/write_launch_record.rs new file mode 100644 index 0000000000..4eba983e99 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/main/write_launch_record.rs @@ -0,0 +1,15 @@ +//! Leave the launch record a later `sqb` reads to serve an unchanged compile natively. + +use std::io; +use std::path::Path; + +use crate::replay::_helpers::files::{prune_records, replace_file}; + +/// Atomically replace `path` with the record `text`, then prune the oldest records beside it. +pub fn write_launch_record(path: &Path, text: &str) -> io::Result<()> { + replace_file(path, text)?; + match path.parent() { + Some(directory) => prune_records(directory), + None => Ok(()), + } +} diff --git a/crates/sqlbuild-cli/src/replay/tests.rs b/crates/sqlbuild-cli/src/replay/tests.rs index 46c3c1617c..c91f6b3796 100644 --- a/crates/sqlbuild-cli/src/replay/tests.rs +++ b/crates/sqlbuild-cli/src/replay/tests.rs @@ -1,4 +1,8 @@ #[path = "tests/test_console.rs"] mod console; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_write_launch_record.rs"] +mod write_launch_record; diff --git a/crates/sqlbuild-cli/src/replay/tests/helpers.rs b/crates/sqlbuild-cli/src/replay/tests/helpers.rs new file mode 100644 index 0000000000..fffc994443 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests/helpers.rs @@ -0,0 +1,32 @@ +use std::fs; +use std::path::Path; +use std::time::{Duration, SystemTime}; + +/// `count` records named `000.json`, `001.json`, ... with increasing modification times. +pub(super) fn existing_records(directory: &Path, count: usize) { + fs::create_dir_all(directory).expect("record directory"); + let start: SystemTime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + for index in 0..count { + let path = directory.join(format!("{index:03}.json")); + fs::write(&path, b"{}").expect("record"); + let file = fs::File::options().write(true).open(&path).expect("open"); + file.set_modified(start + Duration::from_secs(index as u64)) + .expect("mtime"); + } +} + +/// The file names in `directory`, sorted. +pub(super) fn names(directory: &Path) -> Vec { + let mut names: Vec = fs::read_dir(directory) + .expect("listing") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + names.sort(); + names +} diff --git a/crates/sqlbuild-cli/src/replay/tests/test_types.rs b/crates/sqlbuild-cli/src/replay/tests/test_types.rs index b8145c3236..72ea3d4416 100644 --- a/crates/sqlbuild-cli/src/replay/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/replay/tests/test_types.rs @@ -4,3 +4,10 @@ pub(crate) struct TenthsTestCase { pub(crate) milliseconds: i64, pub(crate) expected_text: &'static str, } + +/// Existing records in a launch record directory and which survive one more write. +pub(crate) struct LaunchRecordWriteTestCase { + pub(crate) description: &'static str, + pub(crate) existing_records: usize, + pub(crate) expected_names: Vec, +} diff --git a/crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs b/crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs new file mode 100644 index 0000000000..88d4caf8eb --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs @@ -0,0 +1,45 @@ +use std::fs; + +use crate::replay::main::write_launch_record::write_launch_record; +use crate::replay::tests::helpers::{existing_records, names}; +use crate::replay::tests::test_types::LaunchRecordWriteTestCase; + +#[test] +fn given_record_directory_when_writing_then_record_replaces_atomically_and_oldest_are_pruned() { + let test_cases = [ + LaunchRecordWriteTestCase { + description: "first record creates its directory", + existing_records: 0, + expected_names: vec!["new.json".to_owned()], + }, + LaunchRecordWriteTestCase { + description: "the oldest record beyond the limit is pruned", + existing_records: 32, + expected_names: (1..32) + .map(|index| format!("{index:03}.json")) + .chain(["new.json".to_owned()]) + .collect(), + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let directory = root.path().join("target/cache/launcher-v1"); + existing_records(&directory, test_case.existing_records); + let path = directory.join("new.json"); + + write_launch_record(&path, "{\"format\": 1}").expect("write"); + + assert_eq!( + fs::read_to_string(&path).expect("read"), + "{\"format\": 1}", + "{}", + test_case.description + ); + assert_eq!( + names(&directory), + test_case.expected_names, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index ddac7563c2..3e2632fd59 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -1,4 +1,5 @@ use crate::cli_text::_helpers::diagnostics::diagnostics_text; +use crate::cli_text::main::format_coded_error::format_coded_error; use crate::cli_text::main::format_compile_text::format_compile_text; use crate::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; use crate::cli_text::main::format_plan_completion::format_plan_completion; @@ -10,8 +11,8 @@ use crate::cli_text::models::{ }; use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ - CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, - PlanTextTestCase, + CodedErrorTestCase, CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, + NoticeTestCase, PlanTextTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -232,3 +233,79 @@ fn given_dbt_interop_plan_when_formatting_then_matches_python_text() { ); } } + +/// Expectations recorded from the retired Python coded-error formatter. +#[test] +fn given_coded_error_when_formatting_then_renders_consistently() { + let plain: CliStyle = CliStyle { use_color: false }; + let color: CliStyle = CliStyle { use_color: true }; + let test_cases = [ + CodedErrorTestCase { + description: "renders plain coded error", + parts: ("R002", "materialization failed", None), + include_error_label: true, + style: plain, + expected_rendered: "error[R002]: materialization failed", + }, + CodedErrorTestCase { + description: "renders coded detail beneath an existing error label", + parts: ("T002", "test execution failed", None), + include_error_label: false, + style: plain, + expected_rendered: "[T002] test execution failed", + }, + CodedErrorTestCase { + description: "renders plain coded error with help", + parts: ( + "K011", + "contract requires staged promotion", + Some("set table_promotion_mode to staged"), + ), + include_error_label: true, + style: plain, + expected_rendered: "error[K011]: contract requires staged promotion\n = help: set table_promotion_mode to staged", + }, + CodedErrorTestCase { + description: "renders colorized coded error with help", + parts: ("R006", "audit failed", Some("fix failing audit rows")), + include_error_label: true, + style: color, + expected_rendered: "\x1b[38;5;167m\x1b[1merror[R006]:\x1b[0m audit failed\n \x1b[2m= help:\x1b[0m fix failing audit rows", + }, + CodedErrorTestCase { + description: "renders multiline coded error without color unchanged", + parts: ( + "D012", + "Provider 'slack_provider' in providers/slack.py has invalid settings:\n1 validation error for SlackProvider\nSLACK_TOKEN", + None, + ), + include_error_label: true, + style: plain, + expected_rendered: "error[D012]: Provider 'slack_provider' in providers/slack.py has invalid settings:\n1 validation error for SlackProvider\nSLACK_TOKEN", + }, + CodedErrorTestCase { + description: "renders multiline coded error with muted continuation lines", + parts: ( + "D012", + "Provider 'slack_provider' in providers/slack.py has invalid settings:\n1 validation error for SlackProvider\nSLACK_TOKEN", + None, + ), + include_error_label: true, + style: color, + expected_rendered: "\x1b[38;5;167m\x1b[1merror[D012]:\x1b[0m Provider 'slack_provider' in providers/slack.py has invalid settings:\n\x1b[2m1 validation error for SlackProvider\x1b[0m\n\x1b[2mSLACK_TOKEN\x1b[0m", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_coded_error( + test_case.parts, + test_case.include_error_label, + test_case.style + ), + test_case.expected_rendered, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index ed807492b8..65a373b900 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -32,3 +32,11 @@ pub(super) struct DbtPlanTextTestCase { pub(super) max_entries: Option, pub(super) expected_text: &'static str, } + +pub(super) struct CodedErrorTestCase { + pub(super) description: &'static str, + pub(super) parts: (&'static str, &'static str, Option<&'static str>), + pub(super) include_error_label: bool, + pub(super) style: CliStyle, + pub(super) expected_rendered: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index a438a7c984..67cdf32b3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -11,11 +11,11 @@ use sqlbuild_core::audit_command::main::render_audit_summary::render_audit_summa use sqlbuild_core::audit_command::models::AuditCounts; use sqlbuild_core::cli_text::models::CliStyle; +use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -482,15 +482,10 @@ fn complete( #[pyfunction] #[pyo3(signature = (*, request))] fn run_audit_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "AuditInvocation")?, - ), - ], + let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( + request, + &imported(py, COMMAND_MODELS, "AuditInvocation")?, + true, )?; let pipeline: Bound<'_, PyAny> = compile_audits(request, &invocation)?; let run: AuditRun<'_> = prepare_run(request, &invocation, &pipeline)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 7dd3ddb644..34b0529700 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -18,6 +18,9 @@ use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, none, text, utc_now, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_outputs::runtime_target::{ + write_python_check_runtime_target, write_runtime_target, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; @@ -25,9 +28,6 @@ const COST_COLLECTION_MODULE: &str = "sqlbuild.cli.commands._helpers.cost.collec const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; -const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_runtime_target"; -const CHECK_TARGET_MODULE: &str = - "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; const SUCCESS_STATUS: &str = "success"; const PENDING_MESSAGE: &str = "Build cost collection has not completed."; const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; @@ -326,21 +326,8 @@ pub(crate) fn complete_direct_build<'py>( let exit_code: i32 = build_exit_code(succeeded, (python_failed, checks_failed)); let target_dir: Bound<'py, PyAny> = joined(&context.get("effective_project_dir")?, &["target"])?; - call_with( - &imported(py, RUNTIME_TARGET_MODULE, "write_runtime_target")?, - &[ - ("target_dir", target_dir.clone()), - ("plan_output", plan.clone()), - ("result", result.clone()), - ], - )?; - call_with( - &imported(py, CHECK_TARGET_MODULE, "write_python_check_runtime_target")?, - &[ - ("target_dir", target_dir), - ("results", check_results.clone()), - ], - )?; + write_runtime_target(&target_dir, &plan, result)?; + write_python_check_runtime_target(&target_dir, check_results)?; let cost_started: Instant = Instant::now(); let record: PyResult> = finalize_build_cost( context, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 61bbfd3bce..eaad8e76b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::build_command::support::{ boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; @@ -186,13 +186,11 @@ fn cursors<'py>( if !overrides.is_truthy()? { overrides = imported(py, PLANNER_MODELS, "CursorOverrides")?.call0()?; } - let timestamp: Bound<'py, PyAny> = imported(py, PARSING_MODULE, "parse_cursor_timestamp")?; - let integer: Bound<'py, PyAny> = imported(py, PARSING_MODULE, "parse_cursor_integer")?; Ok([ - timestamp.call1((overrides.getattr("start_ts")?,))?, - timestamp.call1((overrides.getattr("end_ts")?,))?, - integer.call1((overrides.getattr("start_int")?,))?, - integer.call1((overrides.getattr("end_int")?,))?, + cursor_timestamp(&overrides.getattr("start_ts")?)?, + cursor_timestamp(&overrides.getattr("end_ts")?)?, + cursor_integer(&overrides.getattr("start_int")?)?, + cursor_integer(&overrides.getattr("end_int")?)?, ]) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 3d109f203a..f50c97829f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -9,14 +9,13 @@ use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const REPORTERS_MODULE: &str = "sqlbuild.cli.progress.main._build_command_progress_reporters"; const SETTINGS_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; @@ -148,13 +147,7 @@ pub(crate) fn resolve_build_context<'py>( ], )? .extract()?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", text(py, &adapter_name)), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&text(py, &adapter_name), &project_dir)?; let limits: Bound<'py, PyAny> = execution_limits(&discovered, &target_name, (&adapter_name, &adapter))?; let connection_config: Bound<'py, PyAny> = call_with( @@ -168,24 +161,28 @@ pub(crate) fn resolve_build_context<'py>( )?; let machine_output: bool = request.getattr("json_output")?.is_truthy()?; let debug: bool = request.getattr("debug")?.is_truthy()?; - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && !machine_output - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !machine_output && stdout_supports_color(py)?; let stream: Bound<'py, PyAny> = py.import("sys")?.getattr(if debug || machine_output { "stderr" } else { "stdout" })?; - let reporters: Bound<'py, PyAny> = call_with( - &imported(py, REPORTERS_MODULE, "build_command_progress_reporters")?, + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, &[ ("adapter_name", text(py, &adapter_name)), ("stream", stream.clone()), ("use_color", boolean(py, use_color)), ], )?; + let planning_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", boolean(py, use_color)), + ], + )?; let load_sources: Bound<'py, PyAny> = request.getattr("load_sources")?; let should_load_sources: bool = request.getattr("reload_sources")?.is_truthy()? || if load_sources.is_none() { @@ -209,8 +206,8 @@ pub(crate) fn resolve_build_context<'py>( ("connection_config", connection_config), ("use_color", boolean(py, use_color)), ("progress_stream", stream.clone()), - ("connection_progress", reporters.getattr("connection")?), - ("planning_progress", reporters.getattr("planning")?), + ("connection_progress", connection_progress), + ("planning_progress", planning_progress), ("should_load_sources", boolean(py, should_load_sources)), ("effective_target_name", target_name), ("execution_limits", limits), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index ad9c071ce9..122d228eeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; -const LOADER_DAG_MODULE: &str = "sqlbuild.compiler.planner.main.selection._loader_dag"; +const LOADER_DAG_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 02bcdfc731..e486e528b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -8,13 +8,13 @@ use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, keyword_relay, none, }; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const DEFER_CLONE_MODULE: &str = "sqlbuild.cli.commands._helpers.build_planning.defer_clone"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const VERBOSE_INSPECTION_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; @@ -119,11 +119,8 @@ pub(crate) fn compile_build_plan<'py>(context: &BuildContext<'py>) -> PyResult = call_with( - &imported(py, VERBOSE_INSPECTION_MODULE, "verbose_inspection_output")?, - &[("enabled", context.request.getattr("verbose")?)], - )?; - within(&manager, |_| { + let verbose: bool = context.request.getattr("verbose")?.is_truthy()?; + with_verbose_inspection(py, verbose, || { call_with( &imported(py, NATIVE_MODULE, "run_compile_pipeline")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 591e3c48e3..27abb010e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, transient_status, + audit_progress, build_callbacks, footer, nested_progress, terminal, transient_status, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -11,5 +11,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { build_callbacks::register(module)?; footer::register(module)?; nested_progress::register(module)?; + terminal::register(module)?; transient_status::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs index 65a1bb16ac..c0ddf2193d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs @@ -1,13 +1,17 @@ //! Terminal width, the shared transient line coordinator and spinner threads of progress reporters. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const COLUMNS_MODULE: &str = "sqlbuild.presentation.main.terminal_columns"; const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SPINNER_JOIN_SECONDS: f64 = 0.2; +const EXIT_JOIN_SECONDS: f64 = 1.0; +const SPINNER_THREAD_NAME: &str = "sqlbuild-progress-spinner"; +const STOP_EVENT_ATTRIBUTE: &str = "_sqlbuild_stop_event"; pub(crate) const SPINNER_TICK_SECONDS: f64 = 0.1; pub(crate) const CLEAR_LINE: &str = "\r\u{1b}[K"; pub(crate) const HIDE_CURSOR: &str = "\u{1b}[?25l"; @@ -89,6 +93,32 @@ pub(crate) fn flush(stream: &Bound<'_, PyAny>) -> PyResult<()> { Ok(()) } +/// Stop and join live spinners at exit: finalization ending one inside its native wait aborts. +#[pyfunction] +fn stop_progress_spinners(py: Python<'_>) -> PyResult<()> { + let mut spinners: Vec> = Vec::new(); + for thread in imported(py, "threading", "enumerate")? + .call0()? + .try_iter()? + { + let thread: Bound<'_, PyAny> = thread?; + if thread.getattr("name")?.eq(SPINNER_THREAD_NAME)? + && thread.hasattr(STOP_EVENT_ATTRIBUTE)? + { + thread.getattr(STOP_EVENT_ATTRIBUTE)?.call_method0("set")?; + spinners.push(thread); + } + } + for thread in spinners { + if thread.call_method0("is_alive")?.is_truthy()? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("timeout", EXIT_JOIN_SECONDS)?; + thread.call_method("join", (), Some(&options))?; + } + } + Ok(()) +} + /// Start a daemon thread running `owner._spin_until_stopped(event)`; return the event and thread. pub(crate) fn start_spinner<'py>( owner: &Bound<'py, PyAny>, @@ -101,8 +131,10 @@ pub(crate) fn start_spinner<'py>( ("target", owner.getattr("_spin_until_stopped")?), ("args", PyTuple::new(py, [&event])?.into_any()), ("daemon", PyBool::new(py, true).to_owned().into_any()), + ("name", SPINNER_THREAD_NAME.into_bound_py_any(py)?), ], )?; + thread.setattr(STOP_EVENT_ATTRIBUTE, &event)?; thread.call_method0("start")?; Ok((event, thread)) } @@ -116,3 +148,11 @@ pub(crate) fn join_spinner(thread: &Bound<'_, PyAny>) -> PyResult<()> { } Ok(()) } + +/// Add `stop_progress_spinners` and run it at exit, before finalization ends daemon threads. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let hook = wrap_pyfunction!(stop_progress_spinners, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index bbf3d306bf..c255be2ec5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -14,7 +14,8 @@ use crate::bindings::_helpers::build_progress::terminal::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{imported, within}; -const SPINNERS_DISABLED_MODULE: &str = "sqlbuild.presentation.main._progress_spinners_disabled"; +const NO_PROGRESS_VARIABLE: &str = "SQLBUILD_NO_PROGRESS"; +const PROGRESS_DISABLED_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; const PREFLIGHT_COMPLETION_PREFIX: &str = "Prepared "; /// Mutable reporter state; only held briefly and never across Python calls. @@ -149,12 +150,8 @@ impl TransientStatusReporter { enabled: bool, ) -> PyResult { let lines: Bound<'_, PyAny> = shared_lines(py)?; - let spinners_disabled: bool = - imported(py, SPINNERS_DISABLED_MODULE, "progress_spinners_disabled")? - .call0()? - .is_truthy()?; Ok(Self { - enabled: enabled && !spinners_disabled && tty_flag(stream)?, + enabled: enabled && !progress_spinners_disabled() && tty_flag(stream)?, stream: stream.clone().unbind(), lock: lines.getattr("lock")?.unbind(), lines: lines.unbind(), @@ -293,6 +290,30 @@ impl TransientStatusReporter { } } +/// Whether `SQLBUILD_NO_PROGRESS` turns transient spinners off. +pub(crate) fn progress_spinners_disabled() -> bool { + std::env::var(NO_PROGRESS_VARIABLE) + .is_ok_and(|value| PROGRESS_DISABLED_VALUES.contains(&value.trim().to_lowercase().as_str())) +} + +/// Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. +pub(crate) fn maybe_status<'py, T>( + py: Python<'py>, + (message, enabled): (&str, bool), + body: impl FnOnce() -> PyResult, +) -> PyResult { + let stderr: Bound<'py, PyAny> = py.import("sys")?.getattr("stderr")?; + if !enabled || progress_spinners_disabled() || !tty_flag(&stderr)? { + return body(); + } + let reporter: Bound<'py, TransientStatusReporter> = + Bound::new(py, TransientStatusReporter::new(py, &stderr, false, true)?)?; + TransientStatusReporter::start(&reporter, message.to_owned())?; + let result: PyResult = body(); + TransientStatusReporter::close(&reporter)?; + result +} + /// A native status reporter writing to `stream`. pub(crate) fn status_reporter<'py>( stream: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 609fbd9225..09b1c82ae6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -15,20 +15,21 @@ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; +use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; -const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; @@ -39,8 +40,6 @@ const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; -const RUNTIME_TARGET_MODULE: &str = - "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ select checks directly without '+', '~', or path-between expansion"; @@ -658,21 +657,11 @@ fn complete<'py>( stream.call_method1("write", (render_check_summary(&texts, style),))?; stream.call_method0("flush")?; let tuple: Bound<'py, PyAny> = PyTuple::new(py, results)?.into_any(); - call_with( - &imported( - py, - RUNTIME_TARGET_MODULE, - "write_python_check_runtime_target", - )?, - &[ - ( - "target_dir", - invocation - .getattr("effective_project_dir")? - .call_method1("__truediv__", ("target",))?, - ), - ("results", tuple), - ], + write_python_check_runtime_target( + &invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + &tuple, )?; call_with( &imported(py, NATIVE_MODULE, "write_execution_json_output")?, @@ -692,15 +681,10 @@ fn complete<'py>( #[pyfunction] #[pyo3(signature = (*, request))] fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "CheckInvocation")?, - ), - ], + let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( + request, + &imported(py, COMMAND_MODELS, "CheckInvocation")?, + true, )?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; stream.call_method1("write", ("\n",))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs index 901810863c..3a9e4d51d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs @@ -5,14 +5,17 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, }; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::argv::main::parse_cli_arguments::parse_cli_arguments as parse_arguments; use sqlbuild_cli::argv::models::{ArgvOutcome, ArgvSettings, Stream, TypeFailure, Value}; use sqlbuild_cli::argv::types::ArgvHost; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::bindings::_helpers::cli::surrogates::{python_text, rust_text, uses_placeholders}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NAMESPACE_MODULE: &str = "sqlbuild.cli.commands.classes.cli_namespace"; +const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const INT_CONVERTER: &str = "int"; const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; const VARS_CONVERTER: &str = "parse_cli_vars"; @@ -171,20 +174,17 @@ fn object_of(py: Python<'_>, host: &PythonHost<'_>, value: Value) -> PyResult>, + argv: &[Bound<'_, PyString>], defaults: &Bound<'_, PyList>, - use_color: bool, - columns: i64, + settings: ArgvSettings, ) -> PyResult { let py: Python<'_> = defaults.py(); let mut arguments: Vec = Vec::with_capacity(argv.len()); let mut plain_placeholders: bool = false; let mut surrogates: bool = false; - for arg in &argv { + for arg in argv { let (text, mapped) = rust_text(arg)?; surrogates |= mapped; plain_placeholders |= !mapped && uses_placeholders(&text); @@ -205,7 +205,6 @@ fn parse_cli_arguments( let (name, object): (String, Bound<'_, PyAny>) = item.extract()?; values.push((name, host.value_of(&object)?)); } - let settings: ArgvSettings = ArgvSettings { columns, use_color }; let outcome: ArgvOutcome = parse_arguments(&arguments, values, settings, &mut host); let sys: Bound<'_, PyModule> = py.import("sys")?; for (stream, text) in outcome.writes { @@ -236,7 +235,46 @@ fn parse_cli_arguments( } } +/// Parse CLI arguments into a `CliNamespace`; help, version and errors print and set the exit code. +#[pyfunction] +#[pyo3(signature = (argv, *, use_color))] +pub(crate) fn parse_cli_invocation<'py>( + py: Python<'py>, + argv: Vec>, + use_color: bool, +) -> PyResult> { + let args: Bound<'py, PyAny> = imported(py, NAMESPACE_MODULE, "CliNamespace")?.call0()?; + let defaults: Bound<'py, PyList> = py + .get_type::() + .call1((args.getattr("__dict__")?.call_method0("items")?,))? + .downcast_into::()?; + let columns: i64 = py + .import("shutil")? + .call_method0("get_terminal_size")? + .getattr("columns")? + .extract()?; + let settings: ArgvSettings = ArgvSettings { columns, use_color }; + let (rows, exit_code): ParsedRows = parse_cli_arguments(&argv, &defaults, settings)?; + let parsed: Bound<'py, PyAny> = imported(py, ENTRY_MODELS_MODULE, "ParsedCliInvocation")?; + let Some(rows) = rows else { + return call_with( + &parsed, + &[ + ("args", py.None().into_bound(py)), + ("exit_code", exit_code.into_bound_py_any(py)?), + ], + ); + }; + for (name, value) in rows { + args.setattr(name.as_str(), value)?; + } + call_with( + &parsed, + &[("args", args), ("exit_code", py.None().into_bound(py))], + ) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_cli_arguments, module)?)?; + module.add_function(wrap_pyfunction!(parse_cli_invocation, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 90df2f52a4..b20ec28c02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; +use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::diff_report::support::cli_error; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -21,7 +22,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; -const WAREHOUSE_SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; @@ -354,14 +354,15 @@ pub(crate) fn dispatch_command<'py>( ], )?; } - let scope: Bound<'py, PyAny> = call_with( - &imported(py, WAREHOUSE_SCOPE_MODULE, "command_warehouse_scope")?, - &[ - ("command", args.getattr("command")?), - ("cli_warehouse", warehouse), - ("cli_vars", args.getattr("vars")?), - ], - )?; + let scope: Bound<'py, PyAny> = Bound::new( + py, + command_warehouse_scope( + &args.getattr("command")?, + &warehouse, + Some(&args.getattr("vars")?), + )?, + )? + .into_any(); within(&scope, |_| routed_command(args, handlers.as_ref())) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index c947ab597f..d57da3604a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -12,15 +12,18 @@ use sqlbuild_cli::dispatch::main::failure_report::failure_report; use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPolicy}; +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; +use crate::bindings::_helpers::cli::observability::cli_observability_scope; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; +use sqlbuild_core::cli_text::models::CliStyle; const NO_COLOR_OPTION: &str = "--no-color"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.entry.parsing"; const COMPUTE_LOGS_MODULE: &str = "sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"; -const OBSERVABILITY_MODULE: &str = "sqlbuild.cli.commands._helpers.entry.observability"; const LIFECYCLE_EVENT_MODULE: &str = "sqlbuild.runtime.observability.main.create_lifecycle_event"; const INVOCATION_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.invocation_scope"; const EXTERNAL_SCOPE_MODULE: &str = @@ -28,8 +31,6 @@ const EXTERNAL_SCOPE_MODULE: &str = const INVOCATION_CONTEXT_MODULE: &str = "sqlbuild.runtime.output_capture.main.invocation_context"; const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exceptions"; const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const CODED_ERROR_MODULE: &str = "sqlbuild.presentation.main.coded_error_text"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; @@ -142,19 +143,27 @@ fn print_expected_error( ) -> PyResult<()> { let py: Python<'_> = invocation.py(); let (code, message, help) = error_parts(error, fallback_code)?; - let text: Bound<'_, PyAny> = call_with( - &imported(py, CODED_ERROR_MODULE, "format_coded_error")?, - &[ - ("code", code), - ("message", message), - ("help", help), - ( - "use_color", - PyBool::new(py, invocation.use_color).to_owned().into_any(), - ), - ], - )?; - print_object(&text.add("\n")?, Some("stderr"), None) + let help: Option = if help.is_none() { + None + } else { + Some(help.str()?.extract()?) + }; + let text: String = format_coded_error( + ( + &code.extract::()?, + &message.extract::()?, + help.as_deref(), + ), + true, + CliStyle { + use_color: invocation.use_color, + }, + ); + print_object( + &PyString::new(py, &format!("{text}\n")).into_any(), + Some("stderr"), + None, + ) } /// The compile JSON error document `sqb compile --json` prints for a failure. @@ -408,13 +417,14 @@ fn observed_dispatch<'py>( .unwrap_or(u64::MAX) .into_bound_py_any(py) }; - let scope: Bound<'py, PyAny> = call_with( - &imported(py, OBSERVABILITY_MODULE, "cli_observability_scope")?, - &[ - ("args", invocation.args.clone()), - ("project_dir", invocation.project_dir()?), - ], - )?; + let scope: Bound<'py, PyAny> = Bound::new( + py, + cli_observability_scope( + invocation.args.clone().unbind(), + invocation.project_dir()?.unbind(), + ), + )? + .into_any(); within(&scope, |dispatcher| { let publish = |event: Bound<'py, PyAny>| -> PyResult<()> { dispatcher.call_method1("publish_lifecycle", (event,))?; @@ -509,20 +519,8 @@ fn parsed_invocation<'py>( identity: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = argv.py(); - let use_color: bool = !argv.contains(NO_COLOR_OPTION)? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; - let parsed: Bound<'py, PyAny> = call_with( - &imported(py, PARSING_MODULE, "parse_cli_invocation")?, - &[ - ("argv", argv.clone()), - ( - "use_color", - PyBool::new(py, use_color).to_owned().into_any(), - ), - ], - )?; + let use_color: bool = !argv.contains(NO_COLOR_OPTION)? && stdout_supports_color(py)?; + let parsed: Bound<'py, PyAny> = parse_cli_invocation(py, argv.extract()?, use_color)?; let args: Bound<'py, PyAny> = parsed.getattr("args")?; if args.is_none() { let exit_code: Bound<'py, PyAny> = parsed.getattr("exit_code")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 4f49628d00..101172f2be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod argv; pub(crate) mod dispatch; pub(crate) mod entry; +pub(crate) mod observability; pub(crate) mod progress; pub(crate) mod progress_reporters; pub(crate) mod progress_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs new file mode 100644 index 0000000000..c51f83098a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -0,0 +1,572 @@ +//! The observed CLI invocation: lifecycle dispatcher, native progress, terminal index and sinks. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, MutexGuard}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyString, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_cli::dispatch::main::event_export_warning::event_export_warning; +use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EVENT_DISPATCHER_MODULE: &str = "sqlbuild.runtime.observability.classes.event_dispatcher"; +const DISPATCHER_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.dispatcher_scope"; +const IDENTITY_MODULE: &str = "sqlbuild.runtime.observability.main.current_execution_identity"; +const EXTENSIONS_MODULE: &str = "sqlbuild.compiler.discovery.main.runtime_extensions"; +const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; +const EXPORTER_DISPATCHER_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.dispatcher"; +const EXPORTER_SCOPE_CLASS_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.command_scope"; +const EXPORTER_SCOPE_MODULE: &str = + "sqlbuild.runtime.event_exporting.main._event_exporter_command_scope"; +const EXPORTING_CONSTANTS_MODULE: &str = "sqlbuild.runtime.event_exporting.constants"; +const OUTPUT_CAPTURE_MODULE: &str = + "sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture"; +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const OBSERVABILITY_LOGGER: &str = "sqlbuild.cli.observability"; +const DEBUG_COMMAND: &str = "debug"; +const LINEAGE_COMMAND: &str = "lineage"; +const INVOCATION_COMPLETED: &str = "invocation_completed"; +const INVOCATION_FAILED: &str = "invocation_failed"; +const SUMMARY_FIELDS: [&str; 3] = ["queue_depth", "queue_capacity", "flush_complete"]; +const COUNT_FIELDS: [&str; 5] = ["accepted", "filtered", "delivered", "failed", "dropped"]; + +/// `log_debug_event(logger=, message=message, **fields)`. +fn log_debug<'py>( + py: Python<'py>, + message: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult<()> { + let logger: Bound<'py, PyAny> = + imported(py, "logging", "getLogger")?.call1((OBSERVABILITY_LOGGER,))?; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("logger", logger), + ("message", PyString::new(py, message).into_any()), + ]; + pairs.extend(fields.iter().cloned()); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &pairs, + )?; + Ok(()) +} + +fn error_type_name<'py>(py: Python<'py>, error: &PyErr) -> PyResult> { + error.get_type(py).getattr("__name__") +} + +/// Log a failed lifecycle subscriber. +#[pyfunction] +fn log_dispatch_failure(failure: &Bound<'_, PyAny>) -> PyResult<()> { + log_debug( + failure.py(), + "Lifecycle event subscriber failed", + &[ + ("error_type", failure.getattr("error_type")?), + ("channel", failure.getattr("channel")?), + ("subscriber", failure.getattr("subscriber")?), + ], + ) +} + +/// Log a failed event exporter delivery. +#[pyfunction] +fn log_exporter_failure(failure: &Bound<'_, PyAny>) -> PyResult<()> { + log_debug( + failure.py(), + "Event exporter delivery failed", + &[ + ("exporter_name", failure.getattr("exporter_name")?), + ("error_type", failure.getattr("error_type")?), + ("event_kind", failure.getattr("event_kind")?), + ("event_severity", failure.getattr("event_severity")?), + ], + ) +} + +/// Log a failed command-output sink delivery. +#[pyfunction] +fn log_command_output_failure(error: &Bound<'_, PyAny>) -> PyResult<()> { + log_debug( + error.py(), + "Command-output sink delivery failed", + &[("error_type", error.get_type().getattr("__name__")?)], + ) +} + +/// Log the event exporter delivery summary and each exporter's accounting. +#[pyfunction] +fn log_exporter_summary(summary: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = summary.py(); + let mut fields: Vec<(&str, Bound<'_, PyAny>)> = Vec::new(); + for name in COUNT_FIELDS.into_iter().chain(SUMMARY_FIELDS) { + fields.push((name, summary.getattr(name)?)); + } + log_debug(py, "Event exporter delivery summary", &fields)?; + for exporter in summary.getattr("per_exporter")?.try_iter()? { + let exporter: Bound<'_, PyAny> = exporter?; + let counts: Bound<'_, PyAny> = exporter.getattr("counts")?; + let mut fields: Vec<(&str, Bound<'_, PyAny>)> = + vec![("exporter_name", exporter.getattr("exporter_name")?)]; + for name in COUNT_FIELDS { + fields.push((name, counts.getattr(name)?)); + } + for name in SUMMARY_FIELDS { + fields.push((name, summary.getattr(name)?)); + } + log_debug(py, "Event exporter accounting", &fields)?; + } + Ok(()) +} + +/// Records whether the invocation published a successful terminal fact. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct InvocationOutcome { + succeeded: AtomicBool, +} + +#[pymethods] +impl InvocationOutcome { + fn consume(&self, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event_type: String = event.getattr("event_type")?.extract()?; + if event_type == INVOCATION_COMPLETED || event_type == INVOCATION_FAILED { + self.succeeded + .store(event_type == INVOCATION_COMPLETED, Ordering::SeqCst); + } + Ok(()) + } +} + +/// What an observed invocation installed, released in reverse when it ends. +struct Observed { + dispatcher: Py, + projector: Py, + projector_token: Py, + unsubscribe_terminal_index: Py, + unsubscribe_progress: Py, + outcome: Py, + exporter_scope: Option>, + unsubscribe_exporters: Option>, + unsubscribe_outcome: Option>, + /// Context managers entered, innermost last. + entered: Vec>, +} + +/// Log a cleanup failure without letting logging fail the command. +fn report_cleanup_failure(py: Python<'_>, error: &PyErr, phase: &str) { + let reported = error_type_name(py, error).and_then(|error_type| { + log_debug( + py, + "CLI observability cleanup failed", + &[ + ("error_type", error_type), + ("phase", PyString::new(py, phase).into_any()), + ], + ) + }); + drop(reported); +} + +fn run_cleanup(py: Python<'_>, result: PyResult, phase: &str) { + if let Err(error) = result { + report_cleanup_failure(py, &error, phase); + } +} + +/// Native export accounting of a Python `EventExportSummary`. +fn export_summary(summary: &Bound<'_, PyAny>) -> PyResult { + let mut per_exporter: Vec = Vec::new(); + for exporter in summary.getattr("per_exporter")?.try_iter()? { + let exporter: Bound<'_, PyAny> = exporter?; + let counts: Bound<'_, PyAny> = exporter.getattr("counts")?; + per_exporter.push(ExporterLoss { + exporter_name: exporter.getattr("exporter_name")?.extract()?, + dropped: counts.getattr("dropped")?.extract()?, + failed: counts.getattr("failed")?.extract()?, + }); + } + Ok(ExportSummary { + accepted: summary.getattr("accepted")?.extract()?, + dropped: summary.getattr("dropped")?.extract()?, + failed: summary.getattr("failed")?.extract()?, + per_exporter, + }) +} + +/// Print the incomplete-export warning to stderr when events were dropped or failed. +fn warn_incomplete_export(summary: &Bound<'_, PyAny>, command_succeeded: bool) -> PyResult<()> { + let py: Python<'_> = summary.py(); + let Some(message) = event_export_warning(&export_summary(summary)?, command_succeeded) else { + return Ok(()); + }; + let options = pyo3::types::PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&options, "file", py.import("sys")?.getattr("stderr")?)?; + pyo3::types::PyDictMethods::set_item(&options, "flush", true)?; + py.import("builtins")? + .getattr("print")? + .call((message,), Some(&options))?; + Ok(()) +} + +impl Observed { + /// Leave entered contexts innermost first; the exception raised by one, and whether any suppressed. + fn exit_contexts( + &mut self, + py: Python<'_>, + mut pending: Option, + ) -> (Option, bool) { + let mut suppressed: bool = false; + while let Some(manager) = self.entered.pop() { + let arguments: (Py, Py, Py) = pending.as_ref().map_or_else( + || (py.None(), py.None(), py.None()), + |error| { + ( + error.get_type(py).into_any().unbind(), + error.value(py).clone().into_any().unbind(), + error + .traceback(py) + .map_or_else(|| py.None(), |traceback| traceback.into_any().unbind()), + ) + }, + ); + match manager.bind(py).call_method1("__exit__", arguments) { + Ok(result) => { + if pending.is_some() && result.is_truthy().unwrap_or(false) { + pending = None; + suppressed = true; + } + } + Err(error) => { + pending = Some(error); + suppressed = false; + } + } + } + (pending, suppressed) + } + + /// Release exporters, progress and the terminal index, then warn about lost events. + fn release(self, py: Python<'_>) { + if let Some(unsubscribe) = &self.unsubscribe_exporters { + run_cleanup( + py, + unsubscribe.bind(py).call0(), + "event_exporter_unsubscribe", + ); + } + if let Some(unsubscribe) = &self.unsubscribe_outcome { + run_cleanup( + py, + unsubscribe.bind(py).call0(), + "invocation_outcome_unsubscribe", + ); + } + let summary: Option> = self.exporter_scope.as_ref().and_then(|scope| { + match scope.bind(py).call_method0("close") { + Ok(summary) => Some(summary), + Err(error) => { + report_cleanup_failure(py, &error, "event_exporter_shutdown"); + None + } + } + }); + run_cleanup( + py, + self.unsubscribe_terminal_index.bind(py).call0(), + "terminal_index_unsubscribe", + ); + run_cleanup( + py, + self.unsubscribe_progress.bind(py).call0(), + "progress_unsubscribe", + ); + let projector: &Bound<'_, PyAny> = self.projector.bind(py); + run_cleanup(py, projector.call_method0("close"), "progress_close"); + run_cleanup( + py, + projector.call_method1("restore", (&self.projector_token,)), + "progress_context_restore", + ); + if let Some(summary) = summary.filter(|summary| !summary.is_none()) { + let succeeded: bool = self.outcome.get().succeeded.load(Ordering::SeqCst); + run_cleanup( + py, + warn_incomplete_export(&summary, succeeded), + "event_exporter_warning", + ); + } + } + + fn subscribe(&self, py: Python<'_>, subscriber: Bound<'_, PyAny>) -> PyResult> { + Ok(call_with( + &self.dispatcher.bind(py).getattr("subscribe_lifecycle")?, + &[ + ("subscriber", subscriber), + ( + "accepts_opaque", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )? + .unbind()) + } + + fn enter(&mut self, manager: Bound<'_, PyAny>) -> PyResult<()> { + manager.call_method0("__enter__")?; + self.entered.push(manager.unbind()); + Ok(()) + } + + /// Bind configured project event exporters and command-output sinks, then install scopes. + fn install( + &mut self, + py: Python<'_>, + (args, project_dir): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + terminal_index: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let mut command_output_sinks: bool = false; + if args.getattr("command")?.ne(LINEAGE_COMMAND)? { + let extensions: Bound<'_, PyAny> = call_with( + &imported(py, EXTENSIONS_MODULE, "discover_runtime_extensions")?, + &[("project_dir", project_dir.clone())], + )?; + let event_exporters: Bound<'_, PyAny> = extensions.getattr("event_exporters")?; + let sinks: Bound<'_, PyAny> = extensions.getattr("command_output_sinks")?; + command_output_sinks = sinks.is_truthy()?; + if event_exporters.is_truthy()? || command_output_sinks { + imported(py, LAUNCH_RECORD_MODULE, "withhold_launch_record")?.call0()?; + let timeout: Bound<'_, PyAny> = + extensions.getattr("lifecycle_shutdown_timeout_seconds")?; + let timeout: Bound<'_, PyAny> = if timeout.is_none() { + imported( + py, + EXPORTING_CONSTANTS_MODULE, + "DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS", + )? + } else { + timeout.extract::()?.into_bound_py_any(py)? + }; + let delivery: Bound<'_, PyAny> = call_with( + &imported(py, EXPORTER_DISPATCHER_MODULE, "EventExporterDispatcher")?, + &[ + ("shutdown_timeout_seconds", timeout), + ( + "failure_callback", + wrap_pyfunction!(log_exporter_failure, py)?.into_any(), + ), + ( + "summary_callback", + wrap_pyfunction!(log_exporter_summary, py)?.into_any(), + ), + ], + )?; + let scope: Bound<'_, PyAny> = call_with( + &imported(py, EXPORTER_SCOPE_CLASS_MODULE, "EventExporterCommandScope")?, + &[("dispatcher", delivery.clone())], + )?; + self.exporter_scope = Some(scope.clone().unbind()); + call_with( + &scope.getattr("configure_extensions")?, + &[ + ("project_dir", project_dir.clone()), + ("providers", extensions.getattr("providers")?), + ("event_exporters", event_exporters), + ("command_output_sinks", sinks), + ], + )?; + self.unsubscribe_exporters = + Some(self.subscribe(py, delivery.getattr("enqueue")?)?); + let outcome: Bound<'_, PyAny> = self.outcome.bind(py).clone().into_any(); + self.unsubscribe_outcome = Some(self.subscribe(py, outcome.getattr("consume")?)?); + } + } + self.enter( + imported(py, DISPATCHER_SCOPE_MODULE, "dispatcher_scope")? + .call1((self.dispatcher.bind(py),))?, + )?; + self.enter( + imported(py, NATIVE_MODULE, "terminal_event_index_scope")?.call1((terminal_index,))?, + )?; + let Some(scope) = self + .exporter_scope + .as_ref() + .map(|scope| scope.bind(py).clone()) + else { + return Ok(()); + }; + self.enter( + imported(py, EXPORTER_SCOPE_MODULE, "event_exporter_command_scope")? + .call1((&scope,))?, + )?; + let identity: Bound<'_, PyAny> = + imported(py, IDENTITY_MODULE, "current_execution_identity")?.call0()?; + if !identity.is_none() && command_output_sinks { + self.enter(call_with( + &imported(py, OUTPUT_CAPTURE_MODULE, "configured_output_capture_scope")?, + &[ + ("exporter_scope", scope), + ("identity", identity), + ( + "failure_callback", + wrap_pyfunction!(log_command_output_failure, py)?.into_any(), + ), + ], + )?)?; + } + Ok(()) + } +} + +/// Installs native progress, terminal indexing and configured event exporters for one command. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CliObservabilityScope { + args: Py, + project_dir: Py, + observed: Mutex>, +} + +impl CliObservabilityScope { + fn locked(&self) -> PyResult>> { + self.observed + .lock() + .map_err(|_| PyRuntimeError::new_err("observability scope lock is poisoned")) + } +} + +#[pymethods] +impl CliObservabilityScope { + fn __enter__(&self, py: Python<'_>) -> PyResult> { + let args: &Bound<'_, PyAny> = self.args.bind(py); + let dispatcher: Bound<'_, PyAny> = call_with( + &imported(py, EVENT_DISPATCHER_MODULE, "EventDispatcher")?, + &[( + "health_callback", + wrap_pyfunction!(log_dispatch_failure, py)?.into_any(), + )], + )?; + let terminal_index: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "TerminalEventIndex")?.call0()?; + let subscribe = |subscriber: Bound<'_, PyAny>| -> PyResult> { + Ok(call_with( + &dispatcher.getattr("subscribe_lifecycle")?, + &[ + ("subscriber", subscriber), + ( + "accepts_opaque", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )? + .unbind()) + }; + let unsubscribe_terminal_index: Py = subscribe(terminal_index.getattr("consume")?)?; + let machine_output: bool = imported(py, NATIVE_MODULE, "machine_output_mode")? + .call1((args,))? + .is_truthy()?; + let debug: Bound<'_, PyAny> = args.getattr("debug")?; + let progress_on_stderr: bool = + machine_output || debug.is_truthy()? || args.getattr("command")?.eq(DEBUG_COMMAND)?; + let use_color: bool = !machine_output + && !args.getattr("no_color")?.is_truthy()? + && stdout_supports_color(py)?; + let projector: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "NativeProgressProjector")?, + &[ + ( + "stream", + py.import("sys")?.getattr(if progress_on_stderr { + "stderr" + } else { + "stdout" + })?, + ), + ( + "use_color", + PyBool::new(py, use_color).to_owned().into_any(), + ), + ("debug", debug), + ], + )?; + let unsubscribe_progress: Py = subscribe(projector.getattr("consume")?)?; + let projector_token: Bound<'_, PyAny> = projector.call_method0("install")?; + let mut observed: Observed = Observed { + dispatcher: dispatcher.clone().unbind(), + projector: projector.unbind(), + projector_token: projector_token.unbind(), + unsubscribe_terminal_index, + unsubscribe_progress, + outcome: Py::new( + py, + InvocationOutcome { + succeeded: AtomicBool::new(false), + }, + )?, + exporter_scope: None, + unsubscribe_exporters: None, + unsubscribe_outcome: None, + entered: Vec::new(), + }; + if let Err(error) = observed.install(py, (args, self.project_dir.bind(py)), &terminal_index) + { + let (raised, _) = observed.exit_contexts(py, Some(error)); + observed.release(py); + return Err(raised + .unwrap_or_else(|| PyRuntimeError::new_err("observability scope setup failed"))); + } + *self.locked()? = Some(observed); + Ok(dispatcher.unbind()) + } + + #[pyo3(signature = (*exc_info))] + fn __exit__(&self, py: Python<'_>, exc_info: &Bound<'_, PyTuple>) -> PyResult { + let Some(mut observed) = self.locked()?.take() else { + return Ok(false); + }; + let value: Bound<'_, PyAny> = exc_info.get_item(1)?; + let pending: Option = if value.is_none() { + None + } else { + Some(PyErr::from_value(value)) + }; + let had_error: bool = pending.is_some(); + let (raised, suppressed) = observed.exit_contexts(py, pending); + observed.release(py); + match raised { + Some(error) if had_error => { + let original: Bound<'_, PyAny> = exc_info.get_item(1)?; + if error.value(py).is(&original) { + Ok(false) + } else { + Err(error) + } + } + Some(error) => Err(error), + None => Ok(suppressed), + } + } +} + +/// Install native progress, terminal indexing and configured event exporters for one command. +#[pyfunction] +#[pyo3(signature = (*, args, project_dir))] +pub(crate) fn cli_observability_scope( + args: Py, + project_dir: Py, +) -> CliObservabilityScope { + CliObservabilityScope { + args, + project_dir, + observed: Mutex::new(None), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(cli_observability_scope, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs index 1df6d8f56d..6fb23a1220 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -16,7 +16,9 @@ use sqlbuild_core::progress::models::{ExpectationStep, InspectionRecord}; use sqlbuild_core::progress::types::DifferenceSample; use crate::bindings::_helpers::cli::progress::write_persistent; -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; + +const INSPECTION_SINK_MODULE: &str = "sqlbuild.adapter.relations.main.inspection_query_sink"; const HEADER_OPTIONS: (&[&str], &[&str]) = ( &["stream", "command", "target", "concurrency", "use_color"], @@ -161,6 +163,24 @@ impl VerboseInspectionWriter { } } +/// Run `body`, writing each inspection read and then a total to stderr while `enabled`. +pub(crate) fn with_verbose_inspection<'py, T>( + py: Python<'py>, + enabled: bool, + body: impl FnOnce() -> PyResult, +) -> PyResult { + if !enabled { + return body(); + } + let writer: Bound<'py, VerboseInspectionWriter> = + Bound::new(py, VerboseInspectionWriter::new())?; + let sink: Bound<'py, PyAny> = imported(py, INSPECTION_SINK_MODULE, "inspection_query_sink")? + .call1((writer.getattr("write")?,))?; + let value: T = within(&sink, |_| body())?; + writer.get().write_total(py)?; + Ok(value) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(format_execution_header, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 7e12aa65a4..3bf2d20a93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -2,21 +2,26 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; +use crate::bindings::_helpers::cli::{ + argv, entry, observability, progress, progress_reporters, progress_text, +}; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, - format_command, freshness, init_command, janitor, load, presentation_text, query_command, - result_output, scenario_command, seed_command, sql_test_command, workspace_commands, + format_command, freshness, init_command, janitor, load, plan_command, presentation_text, + query_command, result_output, scenario_command, seed_command, sql_test_command, + workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; entry::register(module)?; + observability::register(module)?; progress::register(module)?; progress_reporters::register(module)?; progress_text::register(module)?; janitor::registration::register(module)?; + plan_command::registration::register(module)?; freshness::registration::register(module)?; clone::registration::register(module)?; load::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index d131b3336b..0a41321cf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -23,16 +23,16 @@ use crate::bindings::_helpers::clone::clone_execution::{ }; use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; @@ -127,10 +127,8 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let use_color: bool = !request.getattr("no_color")?.extract::()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let style: CliStyle = CliStyle { use_color }; let stream: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; let progress: Bound<'py, PyAny> = progress_reporter(&stream, style)?; @@ -179,13 +177,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; Ok(Invocation { project_dir, discovered, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs new file mode 100644 index 0000000000..c1ba684318 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs @@ -0,0 +1,70 @@ +//! Adapter resolution for CLI commands, with adapter configuration failures as `C601`. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const REGISTRY_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; +const ADAPTER_ERRORS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; +const CLI_ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ADAPTER_CONFIGURATION_CODE: &str = "C601"; + +/// `CliUserError(error.message, code="C601", help=error.help)` raised from `error`. +fn cli_adapter_error(py: Python<'_>, error: &PyErr) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", ADAPTER_CONFIGURATION_CODE)?; + keywords.set_item("help", value.getattr("help")?)?; + let raised: PyErr = PyErr::from_value( + imported(py, CLI_ERRORS_MODULE, "CliUserError")? + .call((value.getattr("message")?,), Some(&keywords))?, + ); + raised.set_cause(py, Some(error.clone_ref(py))); + Ok(raised) +} + +/// The registered adapter for `adapter_name`, resolved through the adapter registry. +pub(crate) fn resolve_cli_adapter<'py>( + adapter_name: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter_name.py(); + let resolved: PyResult> = call_with( + &imported(py, REGISTRY_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + ); + match resolved { + Err(error) + if error.is_instance( + py, + &imported(py, ADAPTER_ERRORS_MODULE, "AdapterUserError")?, + ) => + { + Err(cli_adapter_error(py, &error)?) + } + resolved => resolved, + } +} + +/// Resolve an adapter name from project config to an adapter instance. +#[pyfunction] +#[pyo3(signature = (*, adapter_name, project_dir = None))] +fn resolve_adapter<'py>( + adapter_name: &Bound<'py, PyAny>, + project_dir: Option>, +) -> PyResult> { + let project_dir: Bound<'py, PyAny> = + project_dir.unwrap_or_else(|| adapter_name.py().None().into_bound(adapter_name.py())); + resolve_cli_adapter(adapter_name, &project_dir) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_adapter, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs new file mode 100644 index 0000000000..83936d55a5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -0,0 +1,155 @@ +//! Discovery, adapter, connection and output context for project commands that connect. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; + +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; + +/// The project directory, discovered inputs, adapter context, colour and progress stream. +type InvocationContext<'py> = ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + bool, + Bound<'py, PyAny>, +); + +fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +/// The effective adapter and project connection configuration as an `AdapterConnectionContext`. +pub(crate) fn resolve_adapter_connection_context<'py>( + discovered: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + (selected_target, cli_vars): (Bound<'py, PyAny>, Bound<'py, PyAny>), + require_connection: bool, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, project_dir)?; + let connection_config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("project_dir", project_dir.clone()), + ("selected_target", selected_target), + ("cli_vars", cli_vars), + ("require_connection", boolean(py, require_connection)), + ], + )?; + call_with( + &imported(py, COMMAND_MODELS, "AdapterConnectionContext")?, + &[ + ("adapter_name", adapter_name), + ("adapter", adapter), + ("connection_config", connection_config), + ], + ) +} + +fn invocation_context<'py>( + request: &Bound<'py, PyAny>, + require_connection: bool, +) -> PyResult> { + let py: Python<'py> = request.py(); + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = if requested.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( + &discovered, + &project_dir, + ( + request.getattr("selected_target")?, + request.getattr("cli_vars")?, + ), + require_connection, + )?; + let machine_output: bool = request.getattr("json_output")?.is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !machine_output && stdout_supports_color(py)?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if machine_output { "stderr" } else { "stdout" })?; + Ok((project_dir, discovered, adapter_context, use_color, stream)) +} + +fn context_fields<'py>( + context: &InvocationContext<'py>, +) -> PyResult)>> { + let (project_dir, discovered, adapter_context, use_color, stream) = context; + let py: Python<'py> = project_dir.py(); + Ok(vec![ + ("effective_project_dir", project_dir.clone()), + ("discovered_inputs", discovered.clone()), + ("adapter_name", adapter_context.getattr("adapter_name")?), + ("adapter", adapter_context.getattr("adapter")?), + ( + "connection_config", + adapter_context.getattr("connection_config")?, + ), + ("use_color", boolean(py, *use_color)), + ("progress_stream", stream.clone()), + ]) +} + +/// `invocation_type(...)` with discovery, adapter, connection and output context. +pub(crate) fn resolve_command_invocation<'py>( + request: &Bound<'py, PyAny>, + invocation_type: &Bound<'py, PyAny>, +) -> PyResult> { + let context: InvocationContext<'py> = invocation_context(request, true)?; + call_with(invocation_type, &context_fields(&context)?) +} + +/// `invocation_type(...)` with command context plus connection and planning progress reporters. +pub(crate) fn resolve_reported_command_invocation<'py>( + request: &Bound<'py, PyAny>, + invocation_type: &Bound<'py, PyAny>, + require_connection: bool, +) -> PyResult> { + let py: Python<'py> = request.py(); + let context: InvocationContext<'py> = invocation_context(request, require_connection)?; + let (_, _, adapter_context, use_color, stream) = &context; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = context_fields(&context)?; + fields.push(( + "connection_progress", + call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", adapter_context.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", boolean(py, *use_color)), + ], + )?, + )); + fields.push(( + "planning_progress", + call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", boolean(py, *use_color)), + ], + )?, + )); + call_with(invocation_type, &fields) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index df3675c30b..2d06334d13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,5 +1,8 @@ +pub(crate) mod adapters; pub(crate) mod connection; +pub(crate) mod invocation; pub(crate) mod preview_target; pub(crate) mod registration; pub(crate) mod runtime_errors; +pub(crate) mod warehouse_scope; pub(crate) mod warehouses; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs index 9aa9b25ee4..c6d17197ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs @@ -2,9 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::command_runtime::{connection, preview_target, warehouses}; +use crate::bindings::_helpers::command_runtime::{ + adapters, connection, preview_target, warehouse_scope, warehouses, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + adapters::register(module)?; + warehouse_scope::register(module)?; warehouses::register(module)?; connection::register(module)?; preview_target::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs new file mode 100644 index 0000000000..ae3c52c241 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -0,0 +1,117 @@ +//! The active command's warehouse group, `--warehouse` override and CLI vars. + +use std::sync::Mutex; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; +const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; +const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; +const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; + +fn context_variable(py: Python<'_>) -> PyResult> { + py.import(NATIVE_MODULE)?.getattr(CONTEXT_ATTRIBUTE) +} + +/// Installs one command's warehouse scope for the duration of a `with` block. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CommandWarehouseScopeInstall { + scope: Py, + token: Mutex>>, +} + +#[pymethods] +impl CommandWarehouseScopeInstall { + fn __enter__(&self, py: Python<'_>) -> PyResult> { + let token: Bound<'_, PyAny> = context_variable(py)?.call_method1("set", (&self.scope,))?; + *self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("warehouse scope lock is poisoned"))? = + Some(token.unbind()); + Ok(self.scope.clone_ref(py)) + } + + #[pyo3(signature = (*_exc_info))] + fn __exit__(&self, py: Python<'_>, _exc_info: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("warehouse scope lock is poisoned"))? + .take(); + if let Some(token) = token { + context_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// Install the warehouse group, CLI override, and CLI vars for one command invocation. +#[pyfunction] +#[pyo3(signature = (*, command, cli_warehouse, cli_vars = None))] +pub(crate) fn command_warehouse_scope<'py>( + command: &Bound<'py, PyAny>, + cli_warehouse: &Bound<'py, PyAny>, + cli_vars: Option<&Bound<'py, PyAny>>, +) -> PyResult { + let py: Python<'py> = command.py(); + let group: Bound<'py, PyAny> = if command.is_none() { + py.None().into_bound(py) + } else { + imported(py, COMMAND_CONSTANTS, "COMMAND_WAREHOUSE_GROUPS")?.call_method1( + "get", + (imported(py, COMMAND_TYPES, "CliCommand")?.call1((command,))?,), + )? + }; + let vars: Bound<'py, PyAny> = match cli_vars.filter(|cli_vars| !cli_vars.is_none()) { + Some(cli_vars) => py.get_type::().call1((cli_vars,))?, + None => PyDict::new(py).into_any(), + }; + let scope: Bound<'py, PyAny> = call_with( + &imported(py, ENTRY_MODELS, "CommandWarehouseScope")?, + &[ + ("group", group), + ("cli_warehouse", cli_warehouse.clone()), + ("cli_vars", vars), + ], + )?; + Ok(CommandWarehouseScopeInstall { + scope: scope.unbind(), + token: Mutex::new(None), + }) +} + +/// Return the active command's warehouse scope, or an empty one outside the CLI. +#[pyfunction] +pub(crate) fn current_command_warehouse_scope(py: Python<'_>) -> PyResult> { + let scope: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; + if !scope.is_none() { + return Ok(scope); + } + call_with( + &imported(py, ENTRY_MODELS, "CommandWarehouseScope")?, + &[("group", py.None().into_bound(py))], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + module.add( + CONTEXT_ATTRIBUTE, + py.import("contextvars")? + .getattr("ContextVar")? + .call((CONTEXT_NAME,), Some(&keywords))?, + )?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(command_warehouse_scope, module)?)?; + module.add_function(wrap_pyfunction!(current_command_warehouse_scope, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index e02035283c..d9b469c8ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -21,6 +21,7 @@ use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, raised, }; +use crate::bindings::_helpers::command_runtime::warehouse_scope::current_command_warehouse_scope; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -28,7 +29,6 @@ use crate::bindings::_helpers::discovery::configuration::discover_project_config use crate::bindings::_helpers::model_config::template_data::expanded_template_data; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; -const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; @@ -86,8 +86,7 @@ fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { } pub(crate) fn command_scope(py: Python<'_>) -> PyResult> { - let scope: Bound<'_, PyAny> = - imported(py, SCOPE_MODULE, "current_command_warehouse_scope")?.call0()?; + let scope: Bound<'_, PyAny> = current_command_warehouse_scope(py)?; Ok(CommandScope { group: scope.getattr("group")?, cli_warehouse: optional_text(&scope.getattr("cli_warehouse")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index eb1ab460a8..a89f776184 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod artifact_batch; pub(crate) mod prepared_artifacts; pub(crate) mod registration; pub(crate) mod report; +pub(crate) mod reuse_attempt; pub(crate) mod reuse_recording; pub(crate) mod surrogates; pub(crate) mod target_writer; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index afaea3f886..dfc638391e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_recording, target_writer, + prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { report::register(module)?; + reuse_attempt::register(module)?; reuse_recording::register(module)?; target_writer::register(module)?; prepared_artifacts::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs new file mode 100644 index 0000000000..2d8b81a517 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -0,0 +1,454 @@ +//! Check whether a compile can replay the stored result, replay it on a hit and leave a launch record. + +use std::path::PathBuf; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; + +use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const REUSE_CONSTANTS: &str = "sqlbuild.cli.compile_reuse.constants"; +const REUSE_MODELS: &str = "sqlbuild.cli.compile_reuse.models"; +const REUSE_TYPES: &str = "sqlbuild.cli.compile_reuse.types"; +const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; +const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; +const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; +const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; +const LAUNCH_RECORD_PATH_MODULE: &str = "sqlbuild.cli.entry.main._launch_record_path"; +const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; +const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; +const SECONDS_FIELD: &str = "{seconds:.1f}"; +const HIT: &str = "HIT"; +const CHECKED_HIT: &str = "hit"; +const CHECKED_BYPASS: &str = "bypass"; +const MISS: &str = "MISS"; +const BYPASS: &str = "BYPASS"; + +fn constant<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, REUSE_CONSTANTS, name) +} + +fn sorted<'py>(value: Bound<'py, PyAny>) -> PyResult> { + imported(value.py(), "builtins", "sorted")?.call1((value,)) +} + +fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) +} + +fn reuse_logger(py: Python<'_>) -> PyResult> { + imported(py, "logging", "getLogger")?.call1((constant(py, "REUSE_LOGGER_NAME")?,)) +} + +/// The reuse constants the native check, the stored compile and the launch record share. +pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult> { + let rules: Bound<'_, PyDict> = PyDict::new(py); + rules.set_item("excluded", sorted(constant(py, "EXCLUDED_DIRECTORIES")?)?)?; + rules.set_item( + "excluded_root", + sorted(constant(py, "EXCLUDED_ROOT_DIRECTORIES")?)?, + )?; + rules.set_item( + "presence_suffixes", + listed(constant(py, "PRESENCE_ONLY_FILE_SUFFIXES")?)?, + )?; + let output_files: Bound<'_, PyAny> = + imported(py, RUNTIME_IDENTITY_MODULE, "redirected_output_files")?.call0()?; + if with_output_files { + rules.set_item("output_files", output_files)?; + } + rules.set_item( + "tracked_prefixes", + listed(constant(py, "TRACKED_ENVIRONMENT_PREFIXES")?)?, + )?; + rules.set_item( + "untracked_names", + sorted(constant(py, "UNTRACKED_ENVIRONMENT_NAMES")?)?, + )?; + rules.set_item( + "disabling_variables", + PyList::new( + py, + [ + ( + imported(py, COMPILE_CONSTANTS, "COMPILE_CACHE_DISABLE_ENV_VAR")?, + imported(py, COMPILE_CONSTANTS, "COMPILE_CACHE_DISABLE_VALUE")?, + ), + ( + constant(py, "REUSE_DISABLE_ENV_VAR")?, + constant(py, "REUSE_DISABLE_VALUE")?, + ), + ], + )?, + )?; + rules.set_item( + "config_filenames", + listed(constant(py, "PROJECT_CONFIG_FILENAMES")?)?, + )?; + for (key, name) in [ + ("racy_window_ns", "RACY_WINDOW_NS"), + ("max_stored_entries", "REUSE_MAX_STORED_ENTRIES"), + ("max_entry_bytes", "REUSE_MAX_ENTRY_BYTES"), + ("missing_environment_value", "MISSING_ENVIRONMENT_VALUE"), + ("missing_file_digest", "MISSING_FILE_DIGEST"), + ("missing_path_mtime_ns", "MISSING_PATH_MTIME_NS"), + ("project_root_path_mtime_ns", "PROJECT_ROOT_PATH_MTIME_NS"), + ] { + rules.set_item(key, constant(py, name)?)?; + } + rules.set_item( + "compiled_root", + imported(py, "os.path", "join")?.call1(( + constant(py, "TARGET_DIRECTORY_NAME")?, + constant(py, "COMPILED_DIRECTORY_NAME")?, + ))?, + )?; + rules.set_item( + "retired_directories", + PyList::new(py, [constant(py, "RETIRED_REUSE_DIRECTORY_NAME")?])?, + )?; + Ok(rules) +} + +/// The launch record `sqb` replays this reusable compile from, as a JSON object. +fn launch_payload<'py>( + attempt: &Bound<'py, PyAny>, + native: &Bound<'py, PyAny>, + key: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = attempt.py(); + let (invocation_digest, runtime, search_path): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = native.call_method0("launch_identity")?.extract()?; + let project_dir: Bound<'py, PyAny> = attempt.getattr("project_dir")?; + let resolved: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; + let capture: Bound<'py, PyAny> = if py + .import("sys")? + .getattr("stdout")? + .is_instance(&imported(py, COMPUTE_LOG_TEE_MODULE, "TextComputeLogTee")?)? + { + let logs: Bound<'py, PyAny> = resolved + .call_method1( + "__truediv__", + (constant(py, "COMPUTE_LOG_DIRECTORY_NAME")?,), + )? + .call_method0("resolve")?; + let capture: Bound<'py, PyDict> = PyDict::new(py); + capture.set_item("root", logs.str()?)?; + capture.set_item("command", constant(py, "COMPILE_COMMAND_NAME")?)?; + capture.set_item("project_dir", resolved.str()?)?; + capture.set_item("target", attempt.getattr("selected_target")?)?; + capture.into_any() + } else { + py.None().into_bound(py) + }; + let hit_message: Bound<'py, PyAny> = + constant(py, "REUSE_HIT_MESSAGE")?.call_method1("split", (SECONDS_FIELD,))?; + let timings: Bound<'py, PyDict> = PyDict::new(py); + for (key, name) in [ + ("hit", "REUSE_HIT_TIMING"), + ("miss", "REUSE_MISS_TIMING"), + ("bypass", "REUSE_BYPASS_TIMING"), + ("check", "REUSE_CHECK_TIMING"), + ("total", "TOTAL_TIMING"), + ] { + timings.set_item(key, constant(py, name)?)?; + } + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("format", constant(py, "LAUNCH_RECORD_FORMAT")?)?; + payload.set_item("key", key)?; + payload.set_item("project_dir", project_dir.str()?)?; + payload.set_item("store_path", native.call_method0("store_path")?)?; + payload.set_item("invocation_digest", invocation_digest)?; + payload.set_item("runtime", runtime)?; + payload.set_item("search_path", search_path)?; + payload.set_item( + "startup_inputs", + imported(py, STARTUP_INPUTS_MODULE, "startup_inputs")?.call0()?, + )?; + payload.set_item("json_output", attempt.getattr("json_output")?)?; + payload.set_item("rules", reuse_rules(py, false)?)?; + payload.set_item("capture", capture)?; + payload.set_item("hit_message", listed(hit_message)?)?; + payload.set_item("timings", timings)?; + Ok(payload) +} + +fn write_record( + attempt: &Bound<'_, PyAny>, + native: &Bound<'_, PyAny>, + path: &Bound<'_, PyAny>, +) -> PyResult<()> { + let payload: Bound<'_, PyDict> = launch_payload(attempt, native, path.getattr("stem")?)?; + let text: String = plain_json(payload.as_any())?; + let path: PathBuf = path.extract()?; + write_record_file(&path, &text).map_err(PyErr::from) +} + +/// Record this reusable compile for `sqb` when nothing it cannot replay took part. +pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = attempt.py(); + let path: Bound<'_, PyAny> = + imported(py, LAUNCH_RECORD_PATH_MODULE, "launch_record_path")?.call0()?; + let native: Bound<'_, PyAny> = attempt.getattr("native")?; + if path.is_none() || native.is_none() { + return Ok(()); + } + let environment: Bound<'_, PyAny> = imported(py, "os", "environ")?; + if environment.contains(imported( + py, + OUTPUT_CAPTURE_CONSTANTS, + "INVOCATION_CONTEXT_ENV", + )?)? { + return Ok(()); + } + let mut directory: Bound<'_, PyAny> = attempt.getattr("project_dir")?; + for part in constant(py, "LAUNCH_RECORD_DIRECTORY")?.try_iter()? { + directory = directory.call_method1("__truediv__", (part?,))?; + } + if path + .getattr("parent")? + .call_method0("resolve")? + .ne(directory.call_method0("resolve")?)? + { + return Ok(()); + } + match write_record(attempt, &native, &path) { + Err(error) if error.is_instance_of::(py) => { + reuse_logger(py)?.call_method( + "debug", + ("Compile reuse did not leave a launch record",), + Some(&keywords( + py, + &[("exc_info", error.value(py).clone().into_any())], + )?), + )?; + Ok(()) + } + written => written, + } +} + +fn outcome<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, REUSE_TYPES, "CompileReuseOutcome")?.getattr(name) +} + +/// A `CompileReuseAttempt` with `fields`. +fn reuse_attempt<'py>( + py: Python<'py>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, REUSE_MODELS, "CompileReuseAttempt")?, fields) +} + +/// The native reuse check request for this invocation. +fn reuse_check<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let flag = |name: &str| -> PyResult { request.getattr(name)?.is_truthy() }; + let use_color: bool = !flag("no_color")? && stdout_supports_color(py)?; + let check: Bound<'py, PyDict> = PyDict::new(py); + check.set_item("project_dir", project_dir)?; + check.set_item( + "store_directory", + imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? + .call1((imported(py, "pathlib", "Path")?.call1((project_dir,))?,))? + .call_method1( + "__truediv__", + (constant(py, "NATIVE_REUSE_DIRECTORY_NAME")?,), + )?, + )?; + check.set_item("selected_target", request.getattr("selected_target")?)?; + check.set_item( + "invocation", + call_with( + &imported(py, RUNTIME_IDENTITY_MODULE, "invocation_identity")?, + &[ + ("request", request.clone()), + ("project_dir", project_dir.clone()), + ("use_color", use_color.into_bound_py_any(py)?), + ], + )?, + )?; + check.set_item( + "runtime", + imported(py, RUNTIME_IDENTITY_MODULE, "runtime_identity")?.call0()?, + )?; + check.set_item("search_path", listed(py.import("sys")?.getattr("path")?)?)?; + check.set_item( + "bypass_requested", + flag("no_cache")? || flag("manifest")? || flag("profiling")? || flag("debug")?, + )?; + check.set_item("json_output", request.getattr("json_output")?)?; + check.set_item("rules", reuse_rules(py, true)?)?; + Ok(check) +} + +/// `print(text, file=sys.stderr)`. +fn print_stderr(py: Python<'_>, text: &Bound<'_, PyAny>) -> PyResult<()> { + py.import("builtins")?.getattr("print")?.call( + (text,), + Some(&keywords( + py, + &[("file", py.import("sys")?.getattr("stderr")?)], + )?), + )?; + Ok(()) +} + +/// `fields` with each override replacing the field of the same name, as `dataclasses.replace`. +fn overridden<'py>( + mut fields: Vec<(&'static str, Bound<'py, PyAny>)>, + overrides: Vec<(&'static str, Bound<'py, PyAny>)>, +) -> Vec<(&'static str, Bound<'py, PyAny>)> { + for (name, value) in overrides { + match fields.iter_mut().find(|field| field.0 == name) { + Some(field) => field.1 = value, + None => fields.push((name, value)), + } + } + fields +} + +/// Replay the stored compile on a hit; the missed attempt when it cannot be replayed. +fn replayed<'py>( + native: &Bound<'py, PyAny>, + missed: Vec<(&'static str, Bound<'py, PyAny>)>, + (started, json_output): (f64, bool), +) -> PyResult> { + let py: Python<'py> = native.py(); + let check_ms: i64 = elapsed_ms(py, started)?; + let mut timings: Vec<(Bound<'py, PyAny>, i64)> = Vec::new(); + for (name, counted) in [ + ("REUSE_HIT_TIMING", 1), + ("REUSE_MISS_TIMING", 0), + ("REUSE_BYPASS_TIMING", 0), + ("REUSE_CHECK_TIMING", check_ms), + ] { + timings.push((constant(py, name)?, counted)); + } + timings.push((constant(py, "TOTAL_TIMING")?, elapsed_ms(py, started)?)); + let rows: Bound<'py, PyList> = if json_output { + PyList::new(py, timings)? + } else { + PyList::empty(py) + }; + let replay: Bound<'py, PyAny> = native.call_method1("replay", (rows,))?; + if replay.is_none() { + return reuse_attempt( + py, + &overridden(missed, vec![("check_ms", 0_i64.into_bound_py_any(py)?)]), + ); + } + let (stdout, stderr_lines, exit_code): ( + Bound<'py, PyString>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = replay.extract()?; + let seconds: Bound<'py, PyAny> = check_ms + .into_bound_py_any(py)? + .call_method1("__truediv__", (constant(py, "MILLISECONDS_PER_SECOND")?,))?; + print_stderr( + py, + &constant(py, "REUSE_HIT_MESSAGE")?.call_method( + "format", + (), + Some(&keywords(py, &[("seconds", seconds)])?), + )?, + )?; + for line in stderr_lines.try_iter()? { + print_stderr(py, &line?)?; + } + let output: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; + output.call_method1("write", (stdout,))?; + output.call_method0("flush")?; + reuse_attempt( + py, + &overridden( + missed, + vec![ + ("outcome", outcome(py, HIT)?), + ("check_ms", check_ms.into_bound_py_any(py)?), + ("exit_code", exit_code), + ], + ), + ) +} + +/// Replay the stored compile when every input matches, otherwise prepare to store one. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let started: f64 = monotonic(py)?; + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = + imported(py, "os.path", "abspath")?.call1((if requested.is_none() { + imported(py, "os", "getcwd")?.call0()? + } else { + requested + },))?; + let native: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "check_compile_reuse")? + .call1((reuse_check(request, &project_dir)?,))?; + let checked: String = native.call_method0("outcome")?.extract()?; + let json_output: Bound<'py, PyAny> = request.getattr("json_output")?; + let mut base: Vec<(&'static str, Bound<'py, PyAny>)> = vec![ + ( + "project_dir", + imported(py, "pathlib", "Path")?.call1((&project_dir,))?, + ), + ("started", started.into_bound_py_any(py)?), + ("selected_target", request.getattr("selected_target")?), + ("json_output", json_output.clone()), + ]; + if checked == CHECKED_BYPASS { + base.push(("outcome", outcome(py, BYPASS)?)); + return reuse_attempt(py, &base); + } + reuse_logger(py)?.call_method1( + "debug", + ( + "Compile reuse check digested %d project files", + native.call_method0("digested_files")?, + ), + )?; + let check_ms: i64 = if native.call_method0("replay_failed")?.is_truthy()? { + 0 + } else { + elapsed_ms(py, started)? + }; + base.extend([ + ("outcome", outcome(py, MISS)?), + ( + "entry_path", + imported(py, "pathlib", "Path")?.call1((native.call_method0("store_path")?,))?, + ), + ("native", native.clone()), + ("check_ms", check_ms.into_bound_py_any(py)?), + ]); + if checked != CHECKED_HIT { + return reuse_attempt(py, &base); + } + let attempt: Bound<'py, PyAny> = replayed(&native, base, (started, json_output.is_truthy()?))?; + if attempt.getattr("outcome")?.eq(outcome(py, HIT)?)? { + write_launch_record(&attempt)?; + } + Ok(attempt) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index aa64f0de47..9908bd9144 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -8,15 +8,14 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::bindings::_helpers::compile_command::reuse_attempt::{reuse_rules, write_launch_record}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::compile_outputs::reuse::artifact_digest; const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; -const NATIVE_REUSE_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.native_reuse"; const PROVIDER_SETTINGS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.provider_settings"; -const LAUNCH_RECORDS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.launch_records"; const MACRO_STORE_PATHS_MODULE: &str = "sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"; const MISS_OUTCOME: &str = "miss"; @@ -466,10 +465,7 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { ), ("dag_path", dag_path), ("json_output", facts.json_output.clone()), - ( - "rules", - imported(py, NATIVE_REUSE_MODULE, "reuse_rules")?.call0()?, - ), + ("rules", reuse_rules(py, true)?.into_any()), ], )?; let (outcome, digested): (String, Bound<'_, PyAny>) = @@ -562,10 +558,7 @@ pub(crate) fn write_reusable_compile( Err(error) => return Err(error), }; if stored { - let _ = call_with( - &imported(py, LAUNCH_RECORDS_MODULE, "write_launch_record")?, - &[("attempt", attempt)], - )?; + write_launch_record(&attempt)?; } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 00eb66a67c..9e6bffcfb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -27,6 +27,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::compile_outputs::artifacts::{ publish_staged_artifacts, remove_stale_artifacts, }; +use crate::bindings::_helpers::compile_outputs::runtime_target::{ + compiled_sql_test_path, planned_sql_test_path, +}; const COMPILED_DIR: &str = "compiled"; const MODELS_DIR: &str = "models"; @@ -43,8 +46,6 @@ const PUBLISHED_STAGING_CHANGED: &str = "changed"; const PUBLISHED_TREE: &str = "tree"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COMPILED_TEST_PATH_MODULE: &str = "sqlbuild.cli.paths.main._compiled_sql_test_output_path"; -const TEST_PATH_MODULE: &str = "sqlbuild.cli.paths.main._sql_test_output_path"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const MODEL_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.execution.sql_test_model_chain"; @@ -269,7 +270,7 @@ fn write_planned_target<'py>( let entry: Bound<'py, PyAny> = entry?; let path: Bound<'py, PyAny> = compiled .div(TESTS_DIR)? - .div(imported(py, TEST_PATH_MODULE, "sql_test_output_path")?.call1((&entry,))?)?; + .div(planned_sql_test_path(&entry)?)?; let comparison_sql: String = timed(py, "comparison_render_ms", || { call_with( &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, @@ -537,16 +538,9 @@ fn write_static_tests<'py>( let mut test_paths: Vec> = Vec::new(); for ((test, _, _), artifact) in zipped(planned)? { let artifact: &Bound<'py, PyAny> = artifact.bind(py); - let path: Bound<'py, PyAny> = tests_root.div(call_with( - &imported( - py, - COMPILED_TEST_PATH_MODULE, - "compiled_sql_test_output_path", - )?, - &[ - ("test", test.bind(py).clone()), - ("model_names", artifact.getattr("model_names")?), - ], + let path: Bound<'py, PyAny> = tests_root.div(compiled_sql_test_path( + test.bind(py), + &artifact.getattr("model_names")?, )?)?; batch.queue_sql(&path, &artifact.getattr("sql")?.extract::()?)?; managed.push(path.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 31a965b2b0..b39370a296 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -6,13 +6,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; +use crate::bindings::_helpers::rules::results::rules_progress; +use crate::bindings::_helpers::rules::rules_config::load_rules_config; const DISCOVER_MODULE: &str = "sqlbuild._native"; -const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const REJECT_SELECTORS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; @@ -24,13 +27,10 @@ const FAST_COLUMN_LINEAGE: &str = "FAST"; const CONTRACTS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; -const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; -const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; -const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; -const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const SELECTION_MODULE: &str = "sqlbuild._native"; +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RULE_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const EARLY_LINT_MODULE: &str = "sqlbuild.rule_engine.classes.early_sql_lint"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild.cli.compile.models"; @@ -141,11 +141,7 @@ pub(crate) fn analyze<'py>( let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(selection.getattr("select")?.is_truthy()? || selection.getattr("exclude")?.is_truthy()?); - let early_lint_class: Bound<'py, PyAny> = imported(py, EARLY_LINT_MODULE, "EarlySqlLint")?; - let manager: Bound<'py, PyAny> = call_with( - &early_lint_class, - &[("enabled", PyBool::new(py, enabled).to_owned().into_any())], - )?; + let manager: Bound<'py, PyAny> = Bound::new(py, EarlySqlLint::new(py, enabled)?)?.into_any(); within(&manager, |early_lint| analyzed(py, &options, early_lint)) } @@ -330,13 +326,7 @@ impl<'py> Driver<'_, 'py> { ("local_config", discovered.getattr("local_config")?), ], )?; - call_with( - &imported(self.py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", self.options.project_dir.clone()), - ], - ) + resolve_cli_adapter(&adapter_name, &self.options.project_dir) } /// Compile the project graph, starting the early SQL lint once inputs are ready. @@ -474,10 +464,8 @@ impl<'py> Driver<'_, 'py> { let py: Python<'py> = self.py; let options: &AnalysisOptions<'py> = self.options; let project_dir: &Bound<'py, PyAny> = &options.project_dir; - let rules_config: Bound<'py, PyAny> = call_with( - &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, - &[("project_dir", project_dir.clone())], - )?; + let rules_config: Bound<'py, PyAny> = + load_rules_config(py, project_dir.extract()?)?.into_bound(py); if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; @@ -505,7 +493,7 @@ impl<'py> Driver<'_, 'py> { py.None().into_bound(py) }; let result: Bound<'py, PyAny> = call_with( - &imported(py, RUN_RULES_MODULE, "run_rules")?, + &imported(py, RULE_THREADS_MODULE, "evaluate_rules")?, &[ ("graph", graph.clone()), ("discovered_inputs", discovered.clone()), @@ -522,17 +510,7 @@ impl<'py> Driver<'_, 'py> { ], )?; let ended: f64 = monotonic.call0()?.extract()?; - let progress: String = call_with( - &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, - &[ - ( - "elapsed_seconds", - (ended - started).into_pyobject(py)?.into_any(), - ), - ("result", result.clone()), - ], - )? - .extract()?; + let progress: String = rules_progress(ended - started, &result, false)?; self.status.complete(&progress)?; Ok(result) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index d0d189dce9..30f86b80b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -1,7 +1,7 @@ //! The `sqb compile` command: analysis, output phases, reuse recording and printed output. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; @@ -19,6 +19,8 @@ use crate::bindings::_helpers::compile_driver::phases::{ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; @@ -27,8 +29,6 @@ const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; -const SKIPPED_RULES_MODULE: &str = "sqlbuild.rule_engine.main.render_skipped_rules"; -const SUPPORTS_COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; /// What one full compile reports for reuse recording. struct CommandOutcome<'py> { @@ -297,15 +297,10 @@ fn compiled_with_status<'py>( } } close_status(status)?; - let skipped_note: Bound<'py, PyAny> = call_with( - &imported(py, SKIPPED_RULES_MODULE, "format_skipped_type_proof_rules")?, - &[("codes", analysis.getattr("skipped_type_proof_rules")?)], - )?; - if !skipped_note.is_none() { - output.call_method1( - "print_stderr_line", - (format!("note: {}", skipped_note.str()?.to_cow()?),), - )?; + if let Some(skipped_note) = + format_skipped_type_proof_rules(analysis.getattr("skipped_type_proof_rules")?.extract()?) + { + output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } let dag_artifact_path: Bound<'py, PyAny> = if rules_failed || request.getattr("dag_path")?.is_none() { @@ -453,10 +448,8 @@ fn record_output<'py>( &format_compile_json(py, Some(keywords(py, &arguments)?))?, ) } else { - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && imported(py, SUPPORTS_COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(py)?; arguments.push(( "use_color", PyBool::new(py, use_color).to_owned().into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index fbe1f63570..2a71c1c096 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -12,6 +12,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; +use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; +use crate::bindings::_helpers::rules::rules_config::load_rules_config; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -21,10 +23,7 @@ const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; -const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; -const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; -const RULES_RESULT_MODULE: &str = "sqlbuild.rule_engine.main.render_result"; -const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; @@ -259,10 +258,8 @@ fn run_rules_gate<'py>( rules_error.call1(("compiler rules require a resolved project directory",))?, )); } - let mut config: Bound<'py, PyAny> = call_with( - &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, - &[("project_dir", project_dir.clone())], - )?; + let mut config: Bound<'py, PyAny> = + load_rules_config(py, project_dir.extract()?)?.into_bound(py); if options.getattr("no_cache")?.is_truthy()? { let cache: Bound<'py, PyAny> = call_with( &imported(py, RULE_MODELS_MODULE, "RulesCacheConfig")?, @@ -292,7 +289,7 @@ fn run_rules_gate<'py>( ], )?; let result: Bound<'py, PyAny> = call_with( - &imported(py, RUN_RULES_MODULE, "run_rules")?, + &imported(py, RULE_THREADS_MODULE, "evaluate_rules")?, &[ ("graph", graph.clone()), ("discovered_inputs", discovered.clone()), @@ -305,17 +302,7 @@ fn run_rules_gate<'py>( )?; if !on_progress.is_none() { let elapsed: f64 = monotonic(py)? - started; - let message: Bound<'py, PyAny> = call_with( - &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, - &[ - ("elapsed_seconds", elapsed.into_pyobject(py)?.into_any()), - ("result", result.clone()), - ( - "note_skipped_rules", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], - )?; + let message: String = rules_progress(elapsed, &result, true)?; on_progress.call1((message,))?; } let findings: Bound<'py, PyAny> = result.getattr("findings")?; @@ -335,19 +322,9 @@ fn run_rules_gate<'py>( ("cache_misses", result.getattr("cache_misses")?), ], )?; - let rendered: Bound<'py, PyAny> = call_with( - &imported(py, RULES_RESULT_MODULE, "format_result")?, - &[ - ("result", summary), - ( - "json_output", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ], - )?; + let rendered: String = format_rules_result(&summary, false)?; Err(PyErr::from_value(rules_error.call1((format!( - "configured compiler rules failed:\n{}", - rendered.str()? + "configured compiler rules failed:\n{rendered}" ),))?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 29e2b31dd0..5234f47d92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -10,8 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; -const SPLIT_SELECTORS_MODULE: &str = - "sqlbuild.compiler.planner.main.selection._split_unit_test_selectors"; +const SPLIT_SELECTORS_MODULE: &str = "sqlbuild._native"; const PLANNER_SCOPES_MODULE: &str = "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; const STATIC_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._scope"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index 1e499b85fe..b1280c561e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -104,6 +104,19 @@ pub(crate) fn indented_json(value: &Bound<'_, PyAny>) -> PyResult { ) } +/// `json.dumps(value)`, raising what `json.dumps` raises. +pub(crate) fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { + let mut conversion: Conversion = Conversion::default(); + let converted: JsonValue = conversion.json_value(value)?; + if let Some(error) = conversion.stdlib_error { + return Err(error); + } + emitted( + &converted, + &JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + ) +} + /// `json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True)`. pub(crate) fn compact_sorted_json(value: &Bound<'_, PyAny>) -> PyResult { let mut conversion: Conversion = Conversion::default(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs index 0afc71b85d..6e37e35303 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod artifacts; pub(crate) mod json_report; pub(crate) mod registration; pub(crate) mod reuse; +pub(crate) mod runtime_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs index 604ca7f3d2..26fea3bc26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_outputs::{json_report, reuse}; +use crate::bindings::_helpers::compile_outputs::{json_report, reuse, runtime_target}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { json_report::register(module)?; - reuse::register(module) + reuse::register(module)?; + runtime_target::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs new file mode 100644 index 0000000000..8e8784396f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -0,0 +1,274 @@ +//! SQL test artifact paths, and the executed SQL and Python check results written to `target/run`. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use pyo3::wrap_pyfunction; +use sqlbuild_analysis::target_paths::main::function_run_path::function_run_path; +use sqlbuild_analysis::target_paths::main::model_run_path::model_run_path; +use sqlbuild_analysis::target_paths::main::runtime_sql_text::runtime_sql_text; +use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output_path; +use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; +const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; +const RUN_DIR: &str = "run"; +const TESTS_DIR: &str = "tests"; +const CHECKS_DIR: &str = "checks"; +const PYTHON_CHECKS_FILE: &str = "python_checks.json"; +const PYTHON_CHECK_KIND: &str = "python_check"; +const JSON_INDENT: i64 = 2; + +/// One test's identity as the test runtime target matches entries to results. +type TestKey = (String, Option, Option, Option); + +fn posix(path: &Bound<'_, PyAny>) -> PyResult> { + if path.is_none() { + return Ok(None); + } + path.call_method0("as_posix")?.extract().map(Some) +} + +fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> { + imported(py, "pathlib", "Path")?.call1((text,)) +} + +fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + Ok(( + value.getattr(name_field)?.extract()?, + posix(&value.getattr("source_path")?)?, + value.getattr("block_index")?.extract()?, + value.getattr("case_name")?.extract()?, + )) +} + +fn path_facts(test: &Bound<'_, PyAny>, model_names: Vec) -> PyResult { + let (name, source_path, block_index, case_name) = test_key(test, "name")?; + Ok(SqlTestPathFacts { + name, + source_path, + block_index, + case_name, + model_names, + }) +} + +/// The target path of one planned SQL test, relative to its artifact directory. +pub(crate) fn planned_sql_test_path<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let mut model_names: Vec = Vec::new(); + for step in entry.getattr("chain")?.try_iter()? { + model_names.push(step?.getattr("model_name")?.extract()?); + } + path_object( + entry.py(), + &sql_test_output_path(&path_facts(entry, model_names)?), + ) +} + +/// The target path of one compiled SQL test from its chain's model names. +pub(crate) fn compiled_sql_test_path<'py>( + test: &Bound<'py, PyAny>, + model_names: &Bound<'py, PyAny>, +) -> PyResult> { + path_object( + test.py(), + &sql_test_output_path(&path_facts(test, model_names.extract()?)?), + ) +} + +fn write_text(path: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + let py: Python<'_> = path.py(); + let mkdir: Bound<'_, PyDict> = PyDict::new(py); + mkdir.set_item("parents", true)?; + mkdir.set_item("exist_ok", true)?; + path.getattr("parent")? + .call_method("mkdir", (), Some(&mkdir))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("encoding", "utf-8")?; + path.call_method("write_text", (text,), Some(&options))?; + Ok(()) +} + +fn sql_statements(result: &Bound<'_, PyAny>) -> PyResult> { + let sql_kind: Bound<'_, PyAny> = + imported(result.py(), LIFECYCLE_TYPES, "LifeCycleEventKind")?.getattr("SQL")?; + let mut statements: Vec = Vec::new(); + for event in result.getattr("lifecycle_events")?.try_iter()? { + let event: Bound<'_, PyAny> = event?; + if event.getattr("kind")?.eq(&sql_kind)? { + statements.push(event.getattr("content")?.extract()?); + } + } + Ok(statements) +} + +fn entry_paths( + entries: &Bound<'_, PyAny>, + path_of: impl Fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult> { + let mut paths: HashMap = HashMap::new(); + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + paths.insert(entry.getattr("name")?.extract()?, path_of(&entry)?); + } + Ok(paths) +} + +fn relative_posix(entry: &Bound<'_, PyAny>) -> PyResult { + entry + .getattr("relative_path")? + .call_method0("as_posix")? + .extract() +} + +/// Write executed model and function lifecycle SQL under `target/run`. +pub(crate) fn write_runtime_target( + target_dir: &Bound<'_, PyAny>, + plan: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + let run_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?; + let models: HashMap = entry_paths(&plan.getattr("model_entries")?, |entry| { + relative_posix(entry).map(|path| model_run_path(&path)) + })?; + let functions: HashMap = + entry_paths(&plan.getattr("function_entries")?, |entry| { + Ok(function_run_path( + &relative_posix(entry)?, + &entry + .getattr("language")? + .getattr("value")? + .extract::()?, + )) + })?; + for (results, name_field, paths) in [ + ("model_results", "model_name", &models), + ("function_results", "function_name", &functions), + ] { + for item in result.getattr(results)?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let statements: Vec = sql_statements(&item)?; + if statements.is_empty() { + continue; + } + let name: String = item.getattr(name_field)?.extract()?; + if let Some(path) = paths.get(&name) { + write_text(&run_dir.div(path)?, &runtime_sql_text(&statements))?; + } + } + } + Ok(()) +} + +/// Write the executed SQL unit-test comparison statements under `target/run/tests`. +pub(crate) fn write_test_runtime_target( + (target_dir, adapter): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + plan: &Bound<'_, PyAny>, + results: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = target_dir.py(); + let mut executed: HashSet = HashSet::new(); + for result in results.try_iter()? { + executed.insert(test_key(&result?, "test_name")?); + } + let tests_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?.div(TESTS_DIR)?; + for entry in plan.getattr("test_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if !executed.contains(&test_key(&entry, "name")?) { + continue; + } + let sql: String = call_with( + &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, + &[ + ("test_entry", entry.clone()), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ], + )? + .extract()?; + write_text( + &tests_dir.div(planned_sql_test_path(&entry)?)?, + &format!("{}\n", sql.trim_end_matches(python_isspace)), + )?; + } + Ok(()) +} + +fn check_status(result: &Bound<'_, PyAny>) -> PyResult<&'static str> { + if result.getattr("passed")?.is_truthy()? { + return Ok("pass"); + } + if result.getattr("warned")?.is_truthy()? { + return Ok("warn"); + } + Ok("fail") +} + +/// Write Python check results under `target/run/checks` when any ran. +pub(crate) fn write_python_check_runtime_target( + target_dir: &Bound<'_, PyAny>, + results: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = target_dir.py(); + let checks: Bound<'_, PyList> = PyList::empty(py); + for result in results.try_iter()? { + let result: Bound<'_, PyAny> = result?; + let name: Bound<'_, PyAny> = result.getattr("node_name")?; + let check: Bound<'_, PyDict> = PyDict::new(py); + check.set_item("kind", PYTHON_CHECK_KIND)?; + check.set_item("name", &name)?; + check.set_item("display_name", &name)?; + check.set_item("check_id", format!("{PYTHON_CHECK_KIND}:{}", name.str()?))?; + check.set_item("status", check_status(&result)?)?; + check.set_item("passed", result.getattr("passed")?)?; + check.set_item("severity", result.getattr("severity")?.getattr("value")?)?; + check.set_item("message", result.getattr("message")?)?; + check.set_item("error_message", result.getattr("error_message")?)?; + check.set_item("metadata", result.getattr("metadata")?)?; + checks.append(check)?; + } + if checks.is_empty() { + return Ok(()); + } + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("checks", checks)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("indent", JSON_INDENT)?; + let text: String = imported(py, "json", "dumps")? + .call((payload,), Some(&options))? + .extract()?; + write_text( + &target_dir + .div(RUN_DIR)? + .div(CHECKS_DIR)? + .div(PYTHON_CHECKS_FILE)?, + &format!("{text}\n"), + ) +} + +/// Write executed model and function lifecycle SQL under `target/run`. +#[pyfunction(name = "write_runtime_target")] +#[pyo3(signature = (*, target_dir, plan_output, result))] +fn write_runtime_target_binding( + target_dir: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + write_runtime_target(target_dir, plan_output, result) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_runtime_target_binding, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index d81bf43958..78a94477a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -19,15 +19,15 @@ use sqlbuild_core::contract_command::models::{ }; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; const GENERATE_ACTION: &str = "generate"; @@ -348,11 +348,8 @@ fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && !json_output - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !json_output && stdout_supports_color(py)?; Ok(Progress { stream, use_color }) } @@ -400,18 +397,11 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< "C472", )); } - let context: Bound<'_, PyAny> = call_with( - &imported( - py, - ADAPTER_CONTEXT_MODULE, - "resolve_adapter_connection_context", - )?, - &[ - ("discovered_inputs", discovered.clone()), - ("effective_project_dir", project_dir.clone()), - ("selected_target", py.None().into_bound(py)), - ("cli_vars", request.getattr("cli_vars")?), - ], + let context: Bound<'_, PyAny> = resolve_adapter_connection_context( + &discovered, + &project_dir, + (py.None().into_bound(py), request.getattr("cli_vars")?), + true, )?; let selection: Selection<'_> = select_resources(request, &discovered, &context)?; let progress: Progress<'_> = progress_for(request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs index 5c8a4f55a3..64d9d392b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs @@ -17,9 +17,9 @@ use crate::bindings::_helpers::cost_command::bounds::{parse_bound, parse_since}; use crate::bindings::_helpers::cost_command::refresh::refresh_pending_cost_run; use crate::bindings::_helpers::cost_command::refusal::cost_error; use crate::bindings::_helpers::cost_command::sorting::{sort_history, sort_resources}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; /// The request fields every cost view reads. struct CostRequest<'py> { @@ -67,9 +67,7 @@ impl<'py> CostRequest<'py> { if self.request.getattr("no_color")?.is_truthy()? { return Ok(false); } - imported(self.request.py(), COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy() + stdout_supports_color(self.request.py()) } /// Write the JSON file when asked, then the JSON or the rendered text to stdout. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 073bca79ba..cfee0e63c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -9,12 +9,12 @@ use sqlbuild_core::cost_command::constants::{ use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; use sqlbuild_core::cost_command::models::CostCoverage; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const COST_TYPES: &str = "sqlbuild.cost.types"; fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { @@ -120,13 +120,7 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'_, PyAny> = call_with( - &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; if !adapter.is_instance(&imported(py, COST_TYPES, "CostAwareAdapter")?)? { return Ok(()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs index dec9f2ffdf..a1ea62d802 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod conversion; pub(crate) mod durations; pub(crate) mod intervals; +pub(crate) mod overrides; pub(crate) mod registration; pub(crate) mod scalars; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs new file mode 100644 index 0000000000..268983ed83 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs @@ -0,0 +1,59 @@ +//! Executor cursor bounds from `--start-cursor`/`--end-cursor` overrides. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyInt, PyString, PyStringMethods}; +use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::conversion::{MODELS_MODULE, algebra_error}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; + +const DECIMAL_MODULE: &str = "decimal"; +const INTEGER_AS_TIMESTAMP: &str = "an integer cursor cannot be used as a timestamp"; +const TIMESTAMP_AS_INTEGER: &str = "a timestamp cursor cannot be used as an integer"; + +fn instance_of(value: &Bound<'_, PyAny>, class: &str) -> PyResult { + value.is_instance(&imported(value.py(), MODELS_MODULE, class)?) +} + +/// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. +pub(crate) fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, "TimestampValue")? { + return value.getattr("value"); + } + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + if instance_of(value, "DateValue")? { + let midnight: Bound<'py, PyAny> = datetime.getattr("min")?.call_method0("time")?; + return datetime.call_method1("combine", (value.getattr("value")?, midnight)); + } + if instance_of(value, "IntegerValue")? { + return Err(algebra_error(py, INTEGER_AS_TIMESTAMP)); + } + datetime.call_method1("fromisoformat", (value,)) +} + +/// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. +pub(crate) fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, "IntegerValue")? { + return value.getattr("value"); + } + if instance_of(value, "DateValue")? || instance_of(value, "TimestampValue")? { + return Err(algebra_error(py, TIMESTAMP_AS_INTEGER)); + } + if let Ok(text) = value.downcast::() + && let Some(parsed) = parse_integer_text(text.to_str()?) + { + return Ok(parsed.into_pyobject(py)?.into_any()); + } + let decimal: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Decimal")?.call1((value,))?; + py.get_type::().call1((decimal,)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs index ea7f60a7c9..f92b519062 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs @@ -5,8 +5,7 @@ use pyo3::types::PyBool; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::imported; - -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; /// `sys.stdout` or `sys.stderr`, looked up when written. pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -15,9 +14,7 @@ pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult) -> PyResult { - imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy() + stdout_supports_color(py) } pub(crate) fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index 89c30819de..d0d82f18f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -26,10 +26,8 @@ use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; -const SELECTOR_EXPANSION_MODULE: &str = - "sqlbuild.compiler.planner.main.selection.selector_expansion"; -const MODEL_SELECTORS_MODULE: &str = - "sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors"; +const SELECTOR_EXPANSION_MODULE: &str = "sqlbuild._native"; +const MODEL_SELECTORS_MODULE: &str = "sqlbuild._native"; const MODEL_RESOURCE_TYPE: &str = "model"; const SELECTION_OPTIONS: (&[&str], &[&str]) = ( &["project", "graph", "select"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 069ca675fc..76a82c12f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -21,23 +21,23 @@ use sqlbuild_core::debug_report::models::{ DebugReport, DebugRuntime, WarehouseCheck, }; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, resolution_message, }; +use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( &[], &[ @@ -272,13 +272,7 @@ fn build_report( ("local_config", discovered_inputs.getattr("local_config")?), ], )?; - let adapter: Bound<'_, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", project_dir_object.clone()), - ], - )?; + let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir_object)?; let context: DebugContext<'_> = DebugContext { project_dir: project_dir_object, discovered_inputs, @@ -286,13 +280,9 @@ fn build_report( adapter, }; let command_scope: CommandScope<'_> = command_scope(py)?; - let unscoped: Bound<'_, PyAny> = call_with( - &imported(py, SCOPE_MODULE, "command_warehouse_scope")?, - &[ - ("command", py.None().into_bound(py)), - ("cli_warehouse", py.None().into_bound(py)), - ], - )?; + let none: Bound<'_, PyAny> = py.None().into_bound(py); + let unscoped: Bound<'_, PyAny> = + Bound::new(py, command_warehouse_scope(&none, &none, None)?)?.into_any(); let base_connection: Bound<'_, PyAny> = within(&unscoped, |_| { project_connection_config( &context.discovered_inputs, @@ -395,10 +385,7 @@ fn run_debug_command(py: Python<'_>, options: Option>) -> PyRe let text: String = if arguments.flag("json_output")? { format_debug_json(&report) + "\n" } else { - let use_color: bool = !arguments.flag("no_color")? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; format_debug_text(&report, CliStyle { use_color }) }; let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 7b92d284e8..7860259577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -30,7 +30,7 @@ use crate::bindings::_helpers::discovery::configuration::discover_project_config use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index ad318afc21..fb2d82986a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -12,6 +12,7 @@ use sqlbuild_cli::diff_report::main::read_only_query_problem::read_only_query_pr use sqlbuild_cli::diff_report::models::QueryShapeFacts; use sqlbuild_cli::diff_report::types::PhaseSeconds; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::diff_report::support::{ call, cli_error, error_text, none, object, progress_line, @@ -19,7 +20,6 @@ use crate::bindings::_helpers::diff_report::support::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const NAMESPACE_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODULE: &str = "sqlbuild._native"; const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; @@ -61,10 +61,7 @@ pub(crate) fn project_adapter<'py>( ("local_config", &discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[("adapter_name", &name), ("project_dir", project_dir)], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&name, project_dir)?; Ok((name.extract()?, adapter)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs index 644245c366..78fa537525 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -18,10 +18,10 @@ use crate::bindings::_helpers::diff_report::command::{ }; use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_cli::diff_report::types::PhaseSeconds; const DEFAULT_MAX_VALUE_LENGTH: usize = 160; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const TEXT_OPTIONS: (&[&str], &[&str]) = ( &[ "result", @@ -195,10 +195,7 @@ pub(crate) fn publish_diff_output( if request.getattr("json_output")?.is_truthy()? { return write_stdout(py, &json_text(models, &document)?); } - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(py)?; write_stdout( py, &render_diff_text( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index c32f91ad2d..d72d384f86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -25,9 +25,9 @@ use crate::bindings::_helpers::format_command::report::lint_report; use crate::bindings::_helpers::format_command::run::{FormatOutcome, FormatRequest, run_format}; use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_lint_inputs}; use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; const START_MESSAGE: &str = "Formatting SQL START"; @@ -285,10 +285,7 @@ fn run_format_command<'py>(py: Python<'py>, options: Option>) } else { raw_project_dir }; - let use_color: bool = !arguments.flag("no_color")? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; let flags: FormatFlags = FormatFlags { check: arguments.flag("check")?, diff: arguments.flag("diff")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 01508c0bdf..e74a2b9a2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -14,17 +14,16 @@ use sqlbuild_core::format_command::main::resolve_lint_paths::resolve_lint_paths; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const UNIT_TEST_SELECTORS_MODULE: &str = - "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SELECTION_CODE: &str = "S007"; /// The adapter, the selected files when a selector scopes the run, and the discovered inputs. @@ -84,13 +83,7 @@ fn discovered_adapter<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - call_with( - &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - ) + resolve_cli_adapter(&adapter_name, project_dir) } fn selectors<'py>(py: Python<'py>, values: &[String]) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index a6f4a96e63..a71149e344 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -14,19 +14,19 @@ use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness_text; use sqlbuild_core::freshness_text::models::FreshnessSourceResult; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::freshness::freshness_observe::{ Observer, PreviousState, observe_freshness, }; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const CONNECTION_MODULE: &str = "sqlbuild._native"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; -const UPSTREAM_MODULE: &str = "sqlbuild.compiler.planner.main.selection._upstream"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; +const UPSTREAM_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const STATE_READ_MODULE: &str = "sqlbuild.compiler.source_freshness.main.read"; @@ -325,16 +325,7 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult ], )?), )?; - let adapter: Bound<'_, PyAny> = imported(py, ADAPTERS_MODULE, "resolve_adapter")?.call( - (), - Some(&keywords( - py, - &[ - ("adapter_name", &adapter_name), - ("project_dir", &project_dir), - ], - )?), - )?; + let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; let graph: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "build_project_graph")?.call( (), Some(&keywords( @@ -368,10 +359,8 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult if !json_output_path.is_none() { write_json_output(py, &payload, (false, &json_output_path))?; } - let use_color: bool = !request.getattr("no_color")?.extract::()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; write(py, &render_freshness_text(&results, CliStyle { use_color }))?; } Ok(freshness_exit_code( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs index 54c8e6f8f7..dbeb9c3956 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs @@ -69,7 +69,7 @@ fn transitive_closure<'py>( } /// Reversed edges: every source key first, then each new target in first-seen order. -fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = edges.py(); let inverted: Bound<'py, PyDict> = PyDict::new(py); for key in edges.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index ee70abfb5b..c4415b453d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyclass, pymethods}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; @@ -12,20 +12,15 @@ use sqlbuild_analysis::graph::main::edge_keys::edge_keys; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::main::match_selector::match_selector; use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; -use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; -use sqlbuild_analysis::graph::main::selector_name_help::selector_name_help; use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; use sqlbuild_analysis::graph::models::{ - BuildResources, EdgeDirection, GraphIndexes, GraphResource, ParsedSelector, ProjectGraph, + BuildResources, EdgeDirection, GraphIndexes, GraphResource, ProjectGraph, }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const KIND_SELECTOR: &str = "kind"; -const PATH_SELECTOR: &str = "path"; - /// `(key, deps, tags, model folder)`. pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); type Edges = Vec<(ObjectKey, Vec)>; @@ -43,8 +38,6 @@ type UniqueIdMap = Vec<(String, Vec)>; type FailureRow = (&'static str, String, Option); /// The keys, or the `PlannerInputError` Python raises instead. type KeysOutcome = (Option>, Option); -/// `("kind", kind, value, upstream, downstream)` or `("path", start, end, upstream, downstream)`. -type ParsedRow = (&'static str, String, String, bool, bool); /// A project graph: lineage edges, tag, path and name indexes, and selector resolution. #[pyclass(module = "sqlbuild._native", frozen)] @@ -226,40 +219,6 @@ impl NativeProjectGraph { } } -/// One selector token parsed, or the `PlannerInputError` Python raises. -#[pyfunction] -fn parse_project_selector(raw: &str) -> (Option, Option) { - match parse_selector(raw) { - Ok(ParsedSelector::Kind { - kind, - value, - upstream, - downstream, - }) => ( - Some((KIND_SELECTOR, kind, value, upstream, downstream)), - None, - ), - Ok(ParsedSelector::Path { - start, - end, - upstream, - downstream, - }) => ( - Some((PATH_SELECTOR, start, end, upstream, downstream)), - None, - ), - Err(error) => (None, Some(failure(error))), - } -} - -/// The "did you mean" help for an unknown selector name, if any. -#[pyfunction] -fn project_selector_name_help(value: &str, candidates: Vec) -> Option { - selector_name_help(value, candidates.iter().map(String::as_str).collect()) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(parse_project_selector, module)?)?; - module.add_function(wrap_pyfunction!(project_selector_name_help, module)?) + module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 3f3ce3f1e5..01218702a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -51,7 +51,7 @@ fn lineage_graph_names<'py>( Ok(names) } -fn edge_dict<'py>( +pub(crate) fn edge_dict<'py>( cache: &KeyCache<'py>, edges: &[(ObjectKey, Vec)], ) -> PyResult> { @@ -137,25 +137,47 @@ fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult { + pub(crate) all_keys: &'a Bound<'py, PyAny>, + pub(crate) upstream: &'a Bound<'py, PyAny>, + pub(crate) downstream: &'a Bound<'py, PyAny>, + pub(crate) tag_index: &'a Bound<'py, PyAny>, + pub(crate) path_index: &'a Bound<'py, PyAny>, +} + +/// A native graph over Python's index mappings, keeping their order. +pub(crate) fn graph_from_view_mappings( + mappings: GraphViewMappings<'_, '_>, +) -> PyResult { let mut names: Vec<(String, ObjectKey)> = Vec::new(); - for item in field("all_keys")?.call_method0("items")?.try_iter()? { + for item in mappings.all_keys.call_method0("items")?.try_iter()? { let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; names.push((name, native_key(&key)?)); } let mut tags: Vec<(String, Vec)> = Vec::new(); - for item in field("tag_index")?.call_method0("items")?.try_iter()? { + for item in mappings.tag_index.call_method0("items")?.try_iter()? { let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; tags.push((tag, native_graph_keys(&keys)?)); } let mut paths: Vec<(ObjectKey, String)> = Vec::new(); - for item in field("path_index")?.call_method0("items")?.try_iter()? { + for item in mappings.path_index.call_method0("items")?.try_iter()? { let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; paths.push((native_key(&key)?, folder)); } graph_from_dicts(GraphIndexes { names, - upstream: native_edges(&field("upstream")?)?, - downstream: native_edges(&field("downstream")?)?, + upstream: native_edges(mappings.upstream)?, + downstream: native_edges(mappings.downstream)?, tags, paths, }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs index 464164b09f..dad2d061af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -15,10 +15,10 @@ use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::init_command::scaffold::scaffold_blank_project; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const EXISTS_MESSAGE: &str = "sqlbuild_project.toml already exists in this directory"; fn exists_error(py: Python<'_>) -> PyErr { @@ -56,9 +56,7 @@ fn run_init_command(py: Python<'_>, project_dir: Option>) -> Py &[("project_dir", base_dir)], )?; let style: CliStyle = CliStyle { - use_color: imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?, + use_color: stdout_supports_color(py)?, }; py.import("sys")?.getattr("stdout")?.call_method1( "write", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 6d3bc6bb69..578e2994a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -19,19 +19,19 @@ use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; use sqlbuild_core::janitor_text::models::JanitorPlanText; +use crate::bindings::_helpers::janitor::janitor_phases::{ + JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, + compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, +}; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; -const PLANNING_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.planning"; -const EXECUTION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.execution"; -const COMPILATION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.compilation"; -const CONNECTIONS_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.connections"; const COMMAND_NAME: &str = "janitor"; fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { @@ -104,10 +104,8 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let options: Bound<'py, PyDict> = keywords( py, &[ @@ -185,25 +183,14 @@ fn clean( py: Python<'_>, request: &Bound<'_, PyAny>, (invocation, settings): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (compile_context, connection_context): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), ) -> PyResult { let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.extract()?, }; - let planning: Bound<'_, PyAny> = imported(py, PLANNING_MODULE, "build_janitor_execution_plan")? - .call( - (), - Some(&keywords( - py, - &[ - ("invocation", invocation), - ("settings", settings), - ("compile_context", compile_context), - ("connection_context", connection_context), - ], - )?), - )?; - let plan: JanitorPlanText = janitor_plan_text(&planning.getattr("plan")?)?; + let planning: Bound<'_, PyAny> = + build_janitor_execution_plan((invocation, settings), compiled, connection)?; + let plan: JanitorPlanText = janitor_plan_text(&planning)?; write(py, &render_janitor_plan(&plan, style))?; let blocked: i32 = i32::from(!plan.blocked_schemas.is_empty()); let as_target: Option = invocation.getattr("as_target")?.extract()?; @@ -225,18 +212,7 @@ fn clean( write(py, "Janitor cancelled.\n")?; return Ok(1); } - let result: Bound<'_, PyAny> = imported(py, EXECUTION_MODULE, "execute_janitor_cleanup")? - .call( - (), - Some(&keywords( - py, - &[ - ("compile_context", compile_context), - ("connection_context", connection_context), - ("planning_result", &planning), - ], - )?), - )?; + let result: Bound<'_, PyAny> = execute_janitor_cleanup(compiled, connection, &planning)?; write( py, &janitor_completion_text(&janitor_result_counts(&result)?, style), @@ -270,39 +246,15 @@ fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = - imported(py, COMPILATION_MODULE, "compile_janitor_project")? - .call((), Some(&keywords(py, &[("invocation", &invocation)])?))?; - let connection_context: Bound<'_, PyAny> = - imported(py, CONNECTIONS_MODULE, "connect_janitor_warehouse")?.call( - (), - Some(&keywords( - py, - &[ - ("invocation", &invocation), - ("compile_context", &compile_context), - ], - )?), - )?; + let compiled: JanitorCompiled<'_> = compile_janitor_project(&invocation)?; + let connection: Bound<'_, PyAny> = connect_janitor_warehouse(&invocation, &compiled)?; let outcome: PyResult = clean( py, request, (&invocation, &settings), - (&compile_context, &connection_context), + (&compiled, &connection), ); - let closed: PyResult> = - imported(py, CONNECTIONS_MODULE, "close_janitor_warehouse").and_then(|close| { - close.call( - (), - Some(&keywords( - py, - &[ - ("compile_context", &compile_context), - ("connection_context", &connection_context), - ], - )?), - ) - }); + let closed: PyResult<()> = close_janitor_warehouse(&compiled, &connection); let code: i32 = outcome?; closed?; Ok(code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs new file mode 100644 index 0000000000..06deeaaf9b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -0,0 +1,293 @@ +//! `sqb janitor` phases: compile, connect, inspect and plan, clean up and close. + +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; + +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const JANITOR_PLAN_MODULE: &str = "sqlbuild.executor.janitor.main.plan"; +const JANITOR_EXECUTE_MODULE: &str = "sqlbuild.executor.janitor.main.execute"; +const JANITOR_MODELS: &str = "sqlbuild.executor.janitor.models"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const UNKNOWN_VIEW_CODE: &str = "C503"; +const UNKNOWN_VIEW_HELP: &str = + "Run sqb janitor without --drop-old-name-view to list compatibility views."; +const CANDIDATE_FIELDS: [&str; 4] = [ + "candidates", + "archive_deletion_candidates", + "query_diff_artifact_candidates", + "direct_state_prune_candidates", +]; + +/// The compiled project, its adapter and the active target connection settings. +pub(crate) struct JanitorCompiled<'py> { + pub(crate) adapter_name: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) project: Bound<'py, PyAny>, + pub(crate) connection_config: Bound<'py, PyAny>, +} + +fn stdout(py: Python<'_>) -> PyResult> { + py.import("sys")?.getattr("stdout") +} + +fn use_color(invocation: &Bound<'_, PyAny>) -> PyResult { + invocation.getattr("use_color")?.extract() +} + +/// `OperationLifecycle(operation_kind=..., operation_name=...)`. +fn lifecycle<'py>(py: Python<'py>, (kind, name): (&str, &str)) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("operation_kind", kind)?; + options.set_item("operation_name", name)?; + imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?.call((), Some(&options)) +} + +/// Compile for the janitor or `--as` target and resolve the active target connection. +pub(crate) fn compile_janitor_project<'py>( + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let context: Bound<'py, PyAny> = resolve_adapter_connection_context( + &invocation.getattr("discovered_inputs")?, + &invocation.getattr("effective_project_dir")?, + ( + invocation.getattr("selected_target")?, + py.None().into_bound(py), + ), + true, + )?; + let started: Instant = Instant::now(); + let status: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stdout(py)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?; + let as_target: Bound<'py, PyAny> = invocation.getattr("as_target")?; + let target: Bound<'py, PyAny> = if as_target.is_none() { + invocation.getattr("selected_target")? + } else { + as_target + }; + let project: Bound<'py, PyAny> = + within(&lifecycle(py, ("project", "project_compile"))?, |_| { + status.call_method1("on_progress", ("Compiling project...",))?; + let project: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("adapter", context.getattr("adapter")?), + ("selected_target", target), + ("resolved_connection", context.getattr("connection_config")?), + ], + )?; + status.call_method1( + "on_progress", + (format!( + "Compiled project. ({:.2}s)", + started.elapsed().as_secs_f64() + ),), + )?; + Ok(project) + })?; + Ok(JanitorCompiled { + adapter_name: context.getattr("adapter_name")?, + adapter: context.getattr("adapter")?, + project, + connection_config: context.getattr("connection_config")?, + }) +} + +fn connection_timing(progress: &Bound<'_, PyAny>, method: &str, started: Instant) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(progress.py()); + options.set_item("connection_count", 1)?; + options.set_item("elapsed_seconds", started.elapsed().as_secs_f64())?; + progress.call_method(method, (), Some(&options))?; + Ok(()) +} + +/// Connect to the janitor target warehouse. +pub(crate) fn connect_janitor_warehouse<'py>( + invocation: &Bound<'py, PyAny>, + compiled: &JanitorCompiled<'py>, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", compiled.adapter_name.clone()), + ("stream", stdout(py)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?; + let started: Instant = Instant::now(); + progress.call_method1("on_connection_start", (1,))?; + let connected: PyResult> = within( + &lifecycle(py, ("janitor", "janitor_target_connection"))?, + |_| { + compiled + .adapter + .call_method1("connect", (&compiled.connection_config,)) + }, + ); + match connected { + Ok(connection) => { + connection_timing(&progress, "on_connection_complete", started)?; + Ok(connection) + } + Err(error) => { + connection_timing(&progress, "on_connection_error", started)?; + Err(error) + } + } +} + +/// Close the janitor warehouse connection. +pub(crate) fn close_janitor_warehouse( + compiled: &JanitorCompiled<'_>, + connection: &Bound<'_, PyAny>, +) -> PyResult<()> { + compiled.adapter.call_method1("close", (connection,))?; + Ok(()) +} + +fn candidate_count(plan: &Bound<'_, PyAny>) -> PyResult { + let mut count: usize = plan.getattr("old_name_views")?.getattr("drops")?.len()?; + for field in CANDIDATE_FIELDS { + count += plan.getattr(field)?.len()?; + } + Ok(count) +} + +fn unknown_view_error(py: Python<'_>, requests: &Bound<'_, PyAny>) -> PyResult { + let names: Vec = requests.extract()?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", UNKNOWN_VIEW_CODE)?; + options.set_item("help", UNKNOWN_VIEW_HELP)?; + let error: Bound<'_, PyAny> = imported(py, ERRORS_MODULE, "CliUserError")?.call( + (format!( + "no compatibility view is recorded at {}", + names.join(", ") + ),), + Some(&options), + )?; + Ok(PyErr::from_value(error)) +} + +/// Build the janitor execution plan from warehouse facts. +pub(crate) fn build_janitor_execution_plan<'py>( + (invocation, settings): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + compiled: &JanitorCompiled<'py>, + connection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let started: Instant = Instant::now(); + let status: Bound<'py, PyAny> = status_reporter(&stdout(py)?, use_color(invocation)?)?; + status.call_method1("start", ("Inspecting warehouse state...",))?; + let janitor: Bound<'py, PyAny> = invocation + .getattr("discovered_inputs")? + .getattr("project_config")? + .getattr("janitor")?; + let plan: Bound<'py, PyAny> = within( + &lifecycle(py, ("janitor", "janitor_candidate_planning"))?, + |lifecycle| { + let direct_settings: Bound<'py, PyAny> = call_with( + &imported(py, JANITOR_MODELS, "JanitorDirectModeSettings")?, + &[ + ("enabled", PyBool::new(py, true).to_owned().into_any()), + ( + "state_history_versions", + settings.getattr("direct_state_history_versions")?, + ), + ( + "archive_retention_days", + settings.getattr("archive_retention_days")?, + ), + ], + )?; + let plan: Bound<'py, PyAny> = call_with( + &imported(py, JANITOR_PLAN_MODULE, "build_janitor_plan")?, + &[ + ("project", compiled.project.clone()), + ("adapter", compiled.adapter.clone()), + ("connection", connection.clone()), + ("retention_days", settings.getattr("retention_days")?), + ( + "delete_tracked_only", + janitor.getattr("delete_tracked_only")?, + ), + ("exclude_patterns", janitor.getattr("exclude_patterns")?), + ("direct_settings", direct_settings), + ( + "early_old_name_view_drops", + settings.getattr("drop_old_name_views")?, + ), + ], + )?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", candidate_count(&plan)?)?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("metadata", metadata)?; + lifecycle.call_method("completed", (), Some(&options))?; + Ok(plan) + }, + )?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "message", + format!( + "Inspected warehouse state. ({:.2}s)", + started.elapsed().as_secs_f64() + ), + )?; + options.set_item("blank_line_after", true)?; + status.call_method("complete", (), Some(&options))?; + let unknown: Bound<'py, PyAny> = plan + .getattr("old_name_views")? + .getattr("unknown_requests")?; + if unknown.is_truthy()? { + return Err(unknown_view_error(py, &unknown)?); + } + Ok(plan) +} + +/// Execute the warehouse cleanup plan. +pub(crate) fn execute_janitor_cleanup<'py>( + compiled: &JanitorCompiled<'py>, + connection: &Bound<'py, PyAny>, + plan: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + compiled.adapter.py(), + JANITOR_EXECUTE_MODULE, + "execute_janitor_plan", + )?, + &[ + ("plan", plan.clone()), + ("adapter", compiled.adapter.clone()), + ("connection", connection.clone()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index af44f2ecdc..b3e8c36dc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod janitor_command; +pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 0296e7af8b..a8cfdc1f67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -31,23 +31,23 @@ use sqlbuild_analysis::lineage_command::models::{ }; use sqlbuild_core::cli_text::models::CliStyle; +use crate::bindings::_helpers::build_progress::transient_status::maybe_status; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_MODULE: &str = "sqlbuild.presentation.main._maybe_status"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; -const SELECTION_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selection"; -const SELECTOR_TOKENS_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_tokens"; -const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; -const SELECTOR_MATCH_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_match"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; +const SELECTION_MODULE: &str = "sqlbuild._native"; +const SELECTOR_TOKENS_MODULE: &str = "sqlbuild._native"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; +const SELECTOR_MATCH_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; @@ -200,16 +200,7 @@ fn prepare_source<'py>( ], )?), )?; - let adapter: Bound<'py, PyAny> = imported(py, ADAPTERS_MODULE, "resolve_adapter")?.call( - (), - Some(&keywords( - py, - &[ - ("adapter_name", &adapter_name), - ("project_dir", project_dir), - ], - )?), - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, project_dir)?; let no_validation: bool = request.getattr("no_sql_validation")?.extract::()? || !compiled; let options: Bound<'py, PyDict> = keywords( py, @@ -384,7 +375,7 @@ fn traced_edges( Ok(result) } -/// `with maybe_status(...)` around building column lineage. +/// Build column lineage under a stderr spinner for large rich-mode requests. fn column_lineage<'py>( py: Python<'py>, (graph, mode): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -392,26 +383,19 @@ fn column_lineage<'py>( ) -> PyResult> { let count: usize = candidates.model_names.len(); let rich: bool = mode.str()?.extract::()? == RICH_MODE; - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item( - "message", - format!("Analyzing column lineage for {count} models..."), - )?; - options.set_item("enabled", rich && count >= RICH_STATUS_MODEL_THRESHOLD)?; - let status: Bound<'py, PyAny> = - imported(py, STATUS_MODULE, "maybe_status")?.call((), Some(&options))?; - status.call_method0("__enter__")?; - let built: PyResult> = - imported(py, NATIVE_MODULE, "build_project_column_lineage").and_then(|build| { - build.call1(( + let message: String = format!("Analyzing column lineage for {count} models..."); + maybe_status( + py, + (&message, rich && count >= RICH_STATUS_MODEL_THRESHOLD), + || { + imported(py, NATIVE_MODULE, "build_project_column_lineage")?.call1(( graph.getattr("project")?, py.None(), mode, PyFrozenSet::new(py, &candidates.model_names)?, )) - }); - status.call_method1("__exit__", (py.None(), py.None(), py.None()))?; - built + }, + ) } /// The column trace of a `model.column` target, or `None` when the target is not a model column. @@ -693,9 +677,7 @@ fn run_lineage_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult()? && !machine_output - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + && stdout_supports_color(py)?; let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if machine_output { "stderr" } else { "stdout" })?; @@ -286,13 +284,7 @@ fn prepare<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", invocation.project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, &invocation.project_dir)?; let connection_config: Bound<'py, PyAny> = call_with( &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, &[ @@ -356,13 +348,6 @@ fn prepare<'py>( }) } -fn cursor<'py>( - overrides: &Bound<'py, PyAny>, - (field, parser): (&str, &str), -) -> PyResult> { - imported(overrides.py(), CURSOR_PARSING_MODULE, parser)?.call1((overrides.getattr(field)?,)) -} - /// The executor's runtime parameters for this load. fn runtime_params<'py>( request: &Bound<'py, PyAny>, @@ -387,19 +372,19 @@ fn runtime_params<'py>( ("is_reload", request.getattr("reload")?), ( "start_cursor_ts", - cursor(overrides, ("start_ts", "parse_cursor_timestamp"))?, + cursor_timestamp(&overrides.getattr("start_ts")?)?, ), ( "end_cursor_ts", - cursor(overrides, ("end_ts", "parse_cursor_timestamp"))?, + cursor_timestamp(&overrides.getattr("end_ts")?)?, ), ( "start_cursor_int", - cursor(overrides, ("start_int", "parse_cursor_integer"))?, + cursor_integer(&overrides.getattr("start_int")?)?, ), ( "end_cursor_int", - cursor(overrides, ("end_int", "parse_cursor_integer"))?, + cursor_integer(&overrides.getattr("end_int")?)?, ), ( "use_color", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs index 021b9ce147..506e5e996c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -4,16 +4,15 @@ use std::collections::HashMap; use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use pyo3::{pyclass, pymethods}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::load_command::main::render_load_result::render_load_result; use sqlbuild_core::load_command::main::render_load_spinner_line::render_load_spinner_line; use sqlbuild_core::load_command::models::{LoadResultText, LoadSpinnerRow, LoadStatus}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, within, -}; +use crate::bindings::_helpers::build_progress::terminal::start_spinner; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; @@ -177,21 +176,10 @@ impl LoadProgressReporter { fn start_spinner(slf: &Bound<'_, Self>) -> PyResult<()> { let py: Python<'_> = slf.py(); slf.get().stop_spinner(py)?; - let event: Bound<'_, PyAny> = imported(py, "threading", "Event")?.call0()?; - let thread: Bound<'_, PyAny> = call_with( - &imported(py, "threading", "Thread")?, - &[ - ("target", slf.getattr("_spin_until_stopped")?), - ("args", PyTuple::new(py, [&event])?.into_any()), - ("daemon", PyBool::new(py, true).to_owned().into_any()), - ], - )?; - { - let mut state = slf.get().state(); - state.stop_event = Some(event.unbind()); - state.thread = Some(thread.clone().unbind()); - } - thread.call_method0("start")?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = slf.get().state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); Ok(()) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 71b7bb0094..08ef0a6c01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -43,8 +43,12 @@ pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; +pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; +pub(crate) mod planner_identity; +pub(crate) mod planner_scope; +pub(crate) mod planner_selection; pub(crate) mod planner_strategy; pub(crate) mod presentation_text; pub(crate) mod project_assembly; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs new file mode 100644 index 0000000000..5e6119a24d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod plan; +pub(crate) mod registration; +pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs new file mode 100644 index 0000000000..2f51f18c81 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -0,0 +1,290 @@ +//! `sqb plan`: compile the plan pipeline and print the plan as text or JSON. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::build_command::support::keyword_relay; +use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; +const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const COMMAND_NAME: &str = "plan"; +const DEFAULT_SECTION_ENTRIES: i64 = 50; +const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; +/// Request fields passed to the compile pipeline under the same name. +const PIPELINE_FIELDS: [&str; 11] = [ + "no_sql_validation", + "no_cache", + "defer_to", + "defer_sources_to", + "cursor_overrides", + "full_refresh", + "select", + "exclude", + "cli_vars", + "max_microbatches", + "selection_diagnostics", +]; + +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +fn print_line(py: Python<'_>, text: &str) -> PyResult<()> { + imported(py, "builtins", "print")?.call1((text,))?; + Ok(()) +} + +/// The discovered project, adapter context, output stream and progress reporters of one plan. +struct PlanInvocation<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, + stream: Bound<'py, PyAny>, + use_color: bool, + connection_progress: Bound<'py, PyAny>, + planning_progress: Bound<'py, PyAny>, + load_sources: Bound<'py, PyAny>, +} + +fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let requested_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = if requested_dir.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested_dir + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?; + let preview: Bound<'py, PyDict> = PyDict::new(py); + preview.set_item("discovered_inputs", &discovered)?; + preview.set_item("as_target", request.getattr("as_target")?)?; + preview.set_item("command_name", COMMAND_NAME)?; + imported(py, NATIVE_MODULE, "validate_preview_target")?.call((), Some(&preview))?; + let context: Bound<'py, PyAny> = resolve_adapter_connection_context( + &discovered, + &project_dir, + ( + request.getattr("selected_target")?, + request.getattr("cli_vars")?, + ), + true, + )?; + let json_output: bool = request.getattr("json_output")?.is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !json_output && stdout_supports_color(py)?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if json_output { "stderr" } else { "stdout" })?; + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", context.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", flag(py, use_color)), + ], + )?; + let planning_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", flag(py, use_color)), + ], + )?; + let requested_load: Bound<'py, PyAny> = request.getattr("load_sources")?; + let load_sources: Bound<'py, PyAny> = if requested_load.is_none() { + call_with( + &imported(py, NATIVE_MODULE, "build_effective_settings_config")?, + &[("discovered_inputs", discovered.clone())], + )? + .getattr("auto_load_sources")? + } else { + requested_load + }; + Ok(PlanInvocation { + project_dir, + discovered, + context, + stream, + use_color, + connection_progress, + planning_progress, + load_sources, + }) +} + +/// A `ConnectionHooks` forwarding to the plan's progress reporters. +fn connection_hooks<'py>(invocation: &PlanInvocation<'py>) -> PyResult> { + let py: Python<'py> = invocation.stream.py(); + let connection: &Bound<'py, PyAny> = &invocation.connection_progress; + call_with( + &imported(py, CONTRACT_MODELS, "ConnectionHooks")?, + &[ + ( + "on_progress", + invocation.planning_progress.getattr("on_progress")?, + ), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + keyword_relay( + connection.getattr("on_connection_complete")?, + &CONNECTION_TIMING, + )?, + ), + ( + "on_connection_error", + keyword_relay( + connection.getattr("on_connection_error")?, + &CONNECTION_TIMING, + )?, + ), + ], + ) +} + +fn compile_pipeline<'py>( + request: &Bound<'py, PyAny>, + invocation: &PlanInvocation<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.project_dir.clone()), + ("discovered_inputs", invocation.discovered.clone()), + ], + )?; + let as_target: Bound<'py, PyAny> = request.getattr("as_target")?; + if !as_target.is_none() { + invocation.stream.call_method1( + "write", + (format!( + "Previewing plan as target '{}' through the active target's connection (inspection only).\n", + as_target.str()? + ),), + )?; + invocation.stream.call_method0("flush")?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "selected_target", + if as_target.is_none() { + request.getattr("selected_target")? + } else { + as_target + }, + )?; + for field in PIPELINE_FIELDS { + options.set_item(field, request.getattr(field)?)?; + } + options.set_item("auto_load_sources", &invocation.load_sources)?; + options.set_item( + "connection_config", + invocation.context.getattr("connection_config")?, + )?; + options.set_item("external_sql_reference_resolver", resolver)?; + options.set_item( + "resolve_python_run_selectors", + request.getattr("include_python")?, + )?; + options.set_item("record_migration_fingerprints", false)?; + let pipeline_options: Bound<'py, PyAny> = + imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?.call((), Some(&options))?; + call_with( + &imported(py, NATIVE_MODULE, "run_compile_pipeline")?, + &[ + ("discovered_inputs", invocation.discovered.clone()), + ("adapter", invocation.context.getattr("adapter")?), + ("options", pipeline_options), + ("hooks", connection_hooks(invocation)?), + ], + ) +} + +fn write_output( + request: &Bound<'_, PyAny>, + invocation: &PlanInvocation<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = request.py(); + let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; + let python_entries: Bound<'_, PyAny> = pipeline.getattr("python_plan_entries")?; + if request.getattr("json_output")?.is_truthy()? { + let text: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_plan_json")?, + &[("plan", plan), ("python_plan_entries", python_entries)], + )?; + imported(py, "builtins", "print")?.call1((text,))?; + return Ok(()); + } + let display: Bound<'_, PyDict> = PyDict::new(py); + display.set_item( + "max_entries_per_section", + if request.getattr("verbose")?.is_truthy()? { + None + } else { + Some(DEFAULT_SECTION_ENTRIES) + }, + )?; + let text: String = call_with( + &imported(py, NATIVE_MODULE, "format_plan")?, + &[ + ("plan", plan), + ("full_refresh", request.getattr("full_refresh")?), + ("use_color", flag(py, invocation.use_color)), + ( + "display_options", + imported(py, PRESENTATION_MODELS, "DisplayOptions")?.call((), Some(&display))?, + ), + ("python_plan_entries", python_entries), + ("include_direct_freshness_diagnostics", flag(py, false)), + ("include_completion", flag(py, true)), + ], + )? + .extract()?; + print_line(py, &format!("\n{text}")) +} + +/// Run one `sqb plan` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_plan_command(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let invocation: PlanInvocation<'_> = resolve_invocation(request)?; + if !request.getattr("json_output")?.is_truthy()? { + invocation.stream.call_method1("write", ("\n",))?; + invocation.stream.call_method0("flush")?; + } + let verbose: bool = request.getattr("verbose")?.is_truthy()?; + let pipeline: Bound<'_, PyAny> = + with_verbose_inspection(py, verbose, || compile_pipeline(request, &invocation))?; + write_output(request, &invocation, &pipeline)?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_plan_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs new file mode 100644 index 0000000000..f9933699b4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb plan` and `sqb rules` commands. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::plan_command::{plan, rules}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan::register(module)?; + rules::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs new file mode 100644 index 0000000000..bfb27544ff --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -0,0 +1,341 @@ +//! `sqb rules`: list, show, install skills for, or run a selection of compiler rules. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; +const LIST_ACTION: &str = "list"; +const SHOW_ACTION: &str = "show"; +const SKILLS_ACTION: &str = "skills"; +const GENERIC_DIALECT: &str = "generic"; +const USAGE_EXIT_CODE: i32 = 2; +const RESULT_FIELDS: [&str; 5] = [ + "findings", + "evaluated_models", + "unevaluated_resources", + "cache_hits", + "cache_misses", +]; + +/// `print(text)`, or `print(text, file=sys.stderr)`. +fn print_line(py: Python<'_>, text: &str, stderr: bool) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + if stderr { + options.set_item("file", py.import("sys")?.getattr("stderr")?)?; + } + imported(py, "builtins", "print")?.call((text,), Some(&options))?; + Ok(()) +} + +fn rule_codes(catalogue: &Bound<'_, PyAny>) -> PyResult> { + let mut codes: Vec = Vec::new(); + for rule in catalogue.try_iter()? { + codes.push(rule?.getattr("code")?.extract()?); + } + Ok(codes) +} + +fn run_skills( + request: &Bound<'_, PyAny>, + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = request.py(); + let check: bool = request.getattr("skills_check")?.is_truthy()?; + let fresh: bool = call_with( + &imported(py, NATIVE_MODULE, "install_rules_skills")?, + &[ + ("config", config.clone()), + ("project_dir", project_dir.clone()), + ("check", request.getattr("skills_check")?), + ], + )? + .is_truthy()?; + if !check { + print_line(py, "Installed Rules skills", false)?; + return Ok(0); + } + print_line( + py, + if fresh { + "Rules skills are fresh" + } else { + "Rules skills are stale" + }, + false, + )?; + Ok(if fresh { 0 } else { 1 }) +} + +fn list_rules(catalogue: &Bound<'_, PyAny>, json_output: bool) -> PyResult { + let py: Python<'_> = catalogue.py(); + if !json_output { + let mut lines: Vec = Vec::new(); + for rule in catalogue.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + lines.push(format!( + "{} {}", + rule.getattr("code")?.str()?, + rule.getattr("message")?.str()? + )); + } + print_line(py, &lines.join("\n"), false)?; + return Ok(0); + } + let rules: Bound<'_, PyList> = PyList::empty(py); + for rule in catalogue.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + let item: Bound<'_, PyDict> = PyDict::new(py); + for field in ["code", "family", "message", "custom"] { + item.set_item(field, rule.getattr(field)?)?; + } + let subject: Bound<'_, PyAny> = rule.getattr("subject")?; + item.set_item( + "subject", + if subject.is_none() { + subject + } else { + subject.getattr("value")? + }, + )?; + rules.append(item)?; + } + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("rules", rules)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("sort_keys", true)?; + let text: String = imported(py, "json", "dumps")? + .call((payload,), Some(&options))? + .extract()?; + print_line(py, &text, false)?; + Ok(0) +} + +fn show_rule( + selector: &str, + catalogue: &Bound<'_, PyAny>, + config: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = catalogue.py(); + for rule in catalogue.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + if rule.getattr("code")?.extract::()? == selector { + let text: String = call_with( + &imported(py, NATIVE_MODULE, "format_rule")?, + &[("rule", rule), ("config", config.clone())], + )? + .extract()?; + print_line(py, &text, false)?; + return Ok(0); + } + } + print_line(py, &format!("Unknown rule: {selector}"), false)?; + Ok(USAGE_EXIT_CODE) +} + +/// The discovered project, its adapter, graph and selected keys a rule selection runs over. +struct RuleInputs<'py> { + discovered: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + graph: Bound<'py, PyAny>, + selected: Bound<'py, PyAny>, +} + +fn rule_inputs<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let select: Bound<'py, PyAny> = request.getattr("select")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", select.clone()), + ("exclude", exclude.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, project_dir)?; + let graph: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_project_graph")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ], + )?; + let selected: Bound<'py, PyAny> = call_with( + &imported(py, GRAPH_SELECTION_MODULE, "select_project_graph")?, + &[ + ("graph", graph.clone()), + ("select", select), + ("exclude", exclude), + ], + )?; + Ok(RuleInputs { + discovered, + adapter, + graph, + selected, + }) +} + +fn run_rule_selection( + request: &Bound<'_, PyAny>, + (selector, catalogue): (&str, &Bound<'_, PyAny>), + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = request.py(); + if !rule_codes(catalogue)? + .iter() + .any(|code| code.starts_with(selector)) + { + print_line(py, &format!("Unknown rule or family: {selector}"), false)?; + return Ok(USAGE_EXIT_CODE); + } + print_line( + py, + &format!("Evaluating rule selection {selector}..."), + true, + )?; + let RuleInputs { + discovered, + adapter, + graph, + selected, + } = rule_inputs(request, project_dir)?; + let narrowed: bool = + request.getattr("select")?.is_truthy()? || request.getattr("exclude")?.is_truthy()?; + let replaced: Bound<'_, PyDict> = PyDict::new(py); + replaced.set_item("select", PyTuple::new(py, [selector])?)?; + replaced.set_item("ignore", PyTuple::empty(py))?; + let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let result: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "run_rules")?, + &[ + ("graph", graph), + ("discovered_inputs", discovered), + ( + "config", + imported(py, "dataclasses", "replace")?.call((config,), Some(&replaced))?, + ), + ("project_dir", project_dir.clone()), + ( + "dialect", + if dialect.is_truthy()? { + dialect + } else { + pyo3::types::PyString::new(py, GENERIC_DIALECT).into_any() + }, + ), + ( + "selected_keys", + if narrowed { + selected + } else { + py.None().into_bound(py) + }, + ), + ], + )?; + let note: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_skipped_type_proof_rules")?, + &[("codes", result.getattr("skipped_type_proof_rules")?)], + )?; + if !note.is_none() { + print_line(py, &format!("note: {}", note.str()?), true)?; + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + for field in RESULT_FIELDS { + fields.set_item(field, result.getattr(field)?)?; + } + let text: String = call_with( + &imported(py, NATIVE_MODULE, "format_rules_result")?, + &[ + ( + "result", + imported(py, RULE_MODELS, "RulesResult")?.call((), Some(&fields))?, + ), + ("json_output", request.getattr("json_output")?), + ], + )? + .extract()?; + print_line(py, &text, false)?; + let findings: usize = result.getattr("findings")?.len()?; + if findings > 0 { + print_line( + py, + &format!("Rule selection {selector} failed with {findings} finding(s)."), + true, + )?; + return Ok(1); + } + print_line(py, &format!("Rule selection {selector} passed."), true)?; + Ok(0) +} + +/// List, inspect, execute, or generate guidance for compiler rules. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_rules_command(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let requested_dir: Bound<'_, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'_, PyAny> = if requested_dir.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested_dir + } + .call_method0("resolve")?; + let config: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "load_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + let action: String = request.getattr("action")?.extract()?; + if action == SKILLS_ACTION { + return run_skills(request, &config, &project_dir); + } + let catalogue: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_rule_catalogue")?, + &[ + ("config", config.clone()), + ("project_dir", project_dir.clone()), + ], + )?; + if action == LIST_ACTION { + return list_rules(&catalogue, request.getattr("json_output")?.is_truthy()?); + } + let selector: Option = request.getattr("rule_selector")?.extract()?; + let Some(selector) = selector else { + print_line(py, "A rule code or family prefix is required", false)?; + return Ok(USAGE_EXIT_CODE); + }; + if action == SHOW_ACTION { + return show_rule(&selector, &catalogue, &config); + } + run_rule_selection(request, (&selector, &catalogue), &config, &project_dir) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_rules_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 19c22aa036..3269ee20cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -27,15 +27,15 @@ use crate::bindings::_helpers::planner_changes::renames::origin_names; use crate::bindings::_helpers::planner_changes::results::{ MODELS_MODULE, backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, }; +use crate::bindings::_helpers::planner_identity::hashing::{ + function_definition_hash, model_query_hash, +}; +use crate::bindings::_helpers::planner_identity::metadata::{ + compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, + model_version_identity_metadata_json, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const MODEL_METADATA_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.model_metadata"; -const VERSION_METADATA_MODULE: &str = - "sqlbuild.compiler.planner.main.identity.version_identity_model_metadata"; -const FUNCTION_HASHES_MODULE: &str = - "sqlbuild.compiler.planner.main.identity.version_identity_function_hashes"; -const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; -const FUNCTIONS_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.functions"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; @@ -46,12 +46,12 @@ const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; /// Inputs shared by every model of one detection pass. -struct ModelPass<'py> { - snapshot: Bound<'py, PyAny>, - query_change_tracking: bool, - full_refresh: bool, - function_local_hashes: Bound<'py, PyAny>, - hook_version_hashes: Bound<'py, PyAny>, +pub(crate) struct ModelPass<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) query_change_tracking: bool, + pub(crate) full_refresh: bool, + pub(crate) function_local_hashes: Bound<'py, PyAny>, + pub(crate) hook_version_hashes: Bound<'py, PyAny>, } /// Selected model and function changes. @@ -87,47 +87,47 @@ fn detect_changes<'py>( )?; let functions: Bound<'py, PyDict> = PyDict::new(py); let result_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "FunctionChangeResult")?; - let fingerprint_sql_of: Bound<'py, PyAny> = imported( - py, - FUNCTIONS_MODULE, - "build_compiled_function_fingerprint_sql", - )?; - let detect_function: Bound<'py, PyAny> = - imported(py, FUNCTIONS_MODULE, "detect_function_change")?; let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let recorded: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("functions")?; + let query_change_tracking: bool = project + .getattr("settings")? + .getattr("query_change_tracking")? + .is_truthy()?; + let dialect: Option = snapshot.getattr("column_dialect")?.extract()?; for function in project.getattr("functions")?.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = function.getattr("name")?; - let fingerprint_sql: Bound<'py, PyAny> = fingerprint_sql_of.call1((&function,))?; - let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = - vec![("fingerprint_sql", fingerprint_sql.clone())]; + let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; + let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = vec![( + "fingerprint_sql", + PyString::new(py, &fingerprint_sql).into_any(), + )]; if selected.contains(function.getattr("key")?)? { - let reason: Bound<'py, PyAny> = call_with( - &detect_function, - &[ - ("function", function.clone()), - ("fingerprint_sql", fingerprint_sql), - ( - "fingerprint", - snapshot - .getattr("fingerprints")? - .getattr("functions")? - .call_method1("get", (&name,))?, - ), - ( - "query_change_tracking", - project - .getattr("settings")? - .getattr("query_change_tracking")?, - ), - ( - "full_refresh", - full_refresh.into_pyobject(py)?.to_owned().into_any(), - ), - ("dialect", snapshot.getattr("column_dialect")?), - ], - )?; - arguments.push(("reason", reason)); + let reason: &str = if full_refresh { + "full_refresh" + } else { + let fingerprint: Bound<'py, PyAny> = recorded.call_method1("get", (&name,))?; + if fingerprint.is_none() { + "first_run" + } else if query_change_tracking + && function_definition_hash( + py, + &name.extract::()?, + ( + &fingerprint_sql, + &function.getattr("language")?.str()?.extract::()?, + dialect.as_deref(), + ), + )? != fingerprint + .getattr("definition_hash")? + .extract::()? + { + "query_changed" + } else { + "no_change" + } + }; + arguments.push(("reason", planner_enum(py, "PlanReason", reason)?)); } functions.set_item(name, call_with(&result_class, &arguments)?)?; } @@ -231,13 +231,14 @@ fn scope_model_changes<'py>( .getattr("query_change_tracking")? .extract()?, }; - let function_local_hashes: Bound<'py, PyAny> = call_with( - &imported(py, FUNCTION_HASHES_MODULE, "build_function_local_hashes")?, - &[ - ("functions", project.getattr("functions")?), - ("dialect", project.getattr("sql_analysis_dialect")?), - ], - )?; + let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( + &project.getattr("functions")?, + project + .getattr("sql_analysis_dialect")? + .extract::>()? + .as_deref(), + )? + .into_any(); let hook_version_hashes: Bound<'py, PyDict> = PyDict::new(py); let identities: Bound<'py, PyAny> = imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")? @@ -288,7 +289,7 @@ fn lookup(mapping: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult( +pub(crate) fn model_change<'py>( pass: &ModelPass<'py>, model: &Bound<'py, PyAny>, (expected_version_hash, expected_metadata_json): (Option, Option), @@ -296,26 +297,14 @@ fn model_change<'py>( let py: Python<'py> = model.py(); let snapshot: &Bound<'py, PyAny> = &pass.snapshot; let model_name: Bound<'py, PyAny> = model.getattr("name")?; - let columns_hash: String = call_with( - &imported(py, MODEL_METADATA_MODULE, "declared_columns_hash")?, - &[("model", model.clone())], - )? - .extract()?; + let columns_hash: String = model_declared_columns_hash(model)?; let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { Some(text) => text, - None => call_with( - &imported( - py, - VERSION_METADATA_MODULE, - "build_model_version_identity_metadata_json", - )?, - &[ - ("model", model.clone()), - ("function_local_hashes", pass.function_local_hashes.clone()), - ("hook_version_hashes", pass.hook_version_hashes.clone()), - ], - )? - .extract()?, + None => model_version_identity_metadata_json( + model, + Some(&pass.function_local_hashes), + Some(&pass.hook_version_hashes), + )?, }; let metadata_json: String = with_declared_columns_hash(&base_metadata, &columns_hash); let mut result = ChangeResult::new( @@ -343,17 +332,14 @@ fn model_change<'py>( let recorded: Option = recorded_fingerprint(&fingerprint)?; let dialect: Bound<'py, PyAny> = snapshot.getattr("column_dialect")?; let compiled_query_hash: Option = if pass.query_change_tracking && recorded.is_some() { - Some( - call_with( - &imported(py, HASHING_MODULE, "model_definition_hash")?, - &[ - ("model_name", model_name.clone()), - ("query_sql", model.getattr("query_sql")?), - ("dialect", dialect.clone()), - ], - )? - .extract()?, - ) + Some(model_query_hash( + py, + &model_name.extract::()?, + ( + &model.getattr("query_sql")?.extract::()?, + dialect.extract::>()?.as_deref(), + ), + )?) } else { None }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index d19d59345b..32dfca7039 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; const NATIVE_MODULE: &str = "sqlbuild._native"; -const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const GENERIC_DIALECT: &str = "generic"; const FINAL_NEWLINE: &str = "\n"; @@ -238,7 +238,7 @@ fn changed_referencing_query( let recorded_models: Bound<'_, PyAny> = snapshot.getattr("fingerprints")?.getattr("models")?; let ref_kind: Bound<'_, PyAny> = imported(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?.getattr("REF")?; - let definition_hash: Bound<'_, PyAny> = imported(py, HASHING_MODULE, "model_definition_hash")?; + let dialect: Option = snapshot.getattr("column_dialect")?.extract()?; for model in selected { let recorded: Bound<'_, PyAny> = recorded_models.call_method1("get", (model.getattr("name")?,))?; @@ -253,15 +253,14 @@ fn changed_referencing_query( } } if references_models - && call_with( - &definition_hash, - &[ - ("model_name", model.getattr("name")?), - ("query_sql", model.getattr("query_sql")?), - ("dialect", snapshot.getattr("column_dialect")?), - ], - )? - .ne(recorded.getattr("definition_hash")?)? + && model_query_hash( + py, + &model.getattr("name")?.extract::()?, + ( + &model.getattr("query_sql")?.extract::()?, + dialect.as_deref(), + ), + )? != recorded.getattr("definition_hash")?.extract::()? { return Ok(true); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs new file mode 100644 index 0000000000..7257d905c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -0,0 +1,415 @@ +//! Planner execution edges, their injected origins, stable execution order and buildability over compiled keys. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, + PyTuple, +}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; +use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; +use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; +use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; + +use crate::bindings::_helpers::analysis::explicit_references::{ + GateRow, HookRow, native_execution_gate_edges, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_identity::hashing::graph_key; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const TABLE_FN_TEST_MODE: &str = "table_fn"; +const INLINE_SQL_HOOK: &str = "inline SQL"; + +/// Python keys by graph key, keeping the first object seen for each. +struct KeyObjects<'py> { + objects: HashMap>, +} + +impl<'py> KeyObjects<'py> { + fn new() -> Self { + Self { + objects: HashMap::new(), + } + } + + fn key(&mut self, object: Bound<'py, PyAny>) -> PyResult { + let key: GraphKey = graph_key(&object)?; + self.objects.entry(key.clone()).or_insert(object); + Ok(key) + } + + fn keys(&mut self, objects: &Bound<'py, PyAny>) -> PyResult> { + objects + .try_iter()? + .map(|object| self.key(object?)) + .collect() + } + + fn object(&self, key: &GraphKey) -> Bound<'py, PyAny> { + self.objects[key].clone() + } + + fn tuple(&self, keys: &[GraphKey], py: Python<'py>) -> PyResult> { + PyTuple::new(py, keys.iter().map(|key| self.object(key))) + } +} + +fn function_resource(key: &Bound<'_, PyAny>) -> PyResult { + let kind: String = key.getattr("resource_type")?.str()?.extract()?; + Ok(kind == UDF_RESOURCE || kind == TABLE_FN_RESOURCE) +} + +fn hook_label(hook_name: &str, kind: &str) -> String { + if kind == INLINE_SQL_HOOK { + return format!("inline SQL hook {hook_name}"); + } + format!("{kind} hook '{hook_name}'") +} + +/// Execution-order edges from lineage and woven SQL test nodes. +#[pyfunction] +fn build_execution_upstream_deps<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let upstream: Bound<'py, PyDict> = PyDict::new(py); + for collection in ["models", "sources", "seeds", "functions"] { + for resource in project.getattr(collection)?.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + upstream.set_item( + resource.getattr("key")?, + PyList::new( + py, + resource + .getattr("deps")? + .try_iter()? + .collect::>>()?, + )?, + )?; + } + } + for test in project.getattr("sql_tests")?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + let test_key: Bound<'py, PyAny> = test.getattr("key")?; + let scope_deps: Bound<'py, PyAny> = test.getattr("scope_deps")?; + let deps: Bound<'py, PyList> = PyList::empty(py); + if test.getattr("mode")?.str()?.extract::()? == TABLE_FN_TEST_MODE { + for scope_dep in scope_deps.try_iter()? { + let scope_dep: Bound<'py, PyAny> = scope_dep?; + if function_resource(&scope_dep)? { + deps.append(scope_dep)?; + } + } + upstream.set_item(test_key, deps)?; + continue; + } + let seen: Bound<'py, PySet> = PySet::empty(py)?; + for target_key in scope_deps.try_iter()? { + let Some(target_deps) = upstream.get_item(target_key?)? else { + continue; + }; + for dep_key in target_deps.try_iter()? { + let dep_key: Bound<'py, PyAny> = dep_key?; + if !function_resource(&dep_key)? || seen.contains(&dep_key)? { + continue; + } + seen.add(&dep_key)?; + deps.append(dep_key)?; + } + } + upstream.set_item(&test_key, deps)?; + for target_key in scope_deps.try_iter()? { + if let Some(target_deps) = upstream.get_item(target_key?)? { + target_deps.downcast::()?.append(&test_key)?; + } + } + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let cache: KeyCache<'py> = KeyCache::new(py)?; + let mut extra: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for (_, _, gated, read) in &gates { + extra.push((cache.key(gated)?, cache.key(read)?)); + } + for (_, gated, read, _, _) in &hooks { + extra.push((cache.key(gated)?, cache.key(read)?)); + } + for (gated, read) in extra { + let gated_upstream: Bound<'py, PyAny> = + upstream.call_method1("setdefault", (gated, PyList::empty(py)))?; + if !gated_upstream.contains(&read)? { + gated_upstream.downcast::()?.append(read)?; + } + } + let result: Bound<'py, PyDict> = PyDict::new(py); + for (key, deps) in upstream.iter() { + result.set_item( + key, + PyTuple::new(py, deps.try_iter()?.collect::>>()?)?, + )?; + } + Ok(result) +} + +/// Human-readable origins for execution edges injected beyond pure lineage. +#[pyfunction] +fn build_execution_edge_origins<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let origins: Bound<'py, PyDict> = PyDict::new(py); + for test in project.getattr("sql_tests")?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + let test_name: String = test.getattr("name")?.extract()?; + for target_key in test.getattr("scope_deps")?.try_iter()? { + let target_key: Bound<'py, PyAny> = target_key?; + let target_name: String = target_key.getattr("name")?.extract()?; + origins.set_item( + (target_key, test.getattr("key")?), + format!("SQL test '{test_name}' runs before '{target_name}'"), + )?; + } + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let cache: KeyCache<'py> = KeyCache::new(py)?; + let lineage: Bound<'py, PyAny> = project.getattr("lineage_graph")?; + let is_lineage = |gated: &(String, String), read: &(String, String)| -> PyResult { + lineage + .call_method1("upstream", (gated.clone(),))? + .contains(read.clone()) + }; + for (audit_name, target, gated, read) in &gates { + if is_lineage(gated, read)? { + continue; + } + let _ = origins.call_method1( + "setdefault", + ( + (cache.key(gated)?, cache.key(read)?), + format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), + ), + )?; + } + for (hook_name, gated, read, kind, _) in &hooks { + if is_lineage(gated, read)? { + continue; + } + let _ = origins.call_method1( + "setdefault", + ( + (cache.key(gated)?, cache.key(read)?), + format!( + "{} on '{}' reads '{}'", + hook_label(hook_name, kind), + gated.1, + read.1 + ), + ), + )?; + } + Ok(origins) +} + +/// Downstream edges keyed by upstream key, each list sorted by `(resource_type, name)`. +#[pyfunction] +fn build_downstream_deps<'py>(upstream: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = upstream.py(); + let result: Bound<'py, PyDict> = PyDict::new(py); + for (key, sources) in inverted(upstream)?.iter() { + let mut keyed: Vec<(GraphKey, Bound<'py, PyAny>)> = sources + .try_iter()? + .map(|source| { + let source: Bound<'py, PyAny> = source?; + Ok((graph_key(&source)?, source)) + }) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + result.set_item( + key, + PyTuple::new(py, keyed.into_iter().map(|(_, source)| source))?, + )?; + } + Ok(result) +} + +fn planner_input_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn edge_origins<'py>( + origins: &Bound<'py, PyAny>, + objects: &mut KeyObjects<'py>, +) -> PyResult> { + let mut edges: HashMap<(GraphKey, GraphKey), String> = HashMap::new(); + if origins.is_none() { + return Ok(edges); + } + for item in origins.call_method0("items")?.try_iter()? { + let ((dependent, dependency), origin): ((Bound<'py, PyAny>, Bound<'py, PyAny>), String) = + item?.extract()?; + edges.insert((objects.key(dependent)?, objects.key(dependency)?), origin); + } + Ok(edges) +} + +/// All keys in stable dependency order using Kahn's algorithm. +#[pyfunction] +#[pyo3(signature = (*, upstream, injected_edge_origins = None))] +fn topologically_order_keys<'py>( + upstream: &Bound<'py, PyAny>, + injected_edge_origins: Option>, +) -> PyResult> { + let py: Python<'py> = upstream.py(); + let mut objects = KeyObjects::new(); + let mut edges: Vec<(GraphKey, Vec)> = Vec::new(); + for item in upstream.call_method0("items")?.try_iter()? { + let (key, deps): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + edges.push((objects.key(key)?, objects.keys(&deps)?)); + } + let origins: HashMap<(GraphKey, GraphKey), String> = match injected_edge_origins { + Some(origins) => edge_origins(&origins, &mut objects)?, + None => HashMap::new(), + }; + let ordered: Vec = + topological_order(&edges, &origins).map_err(|message| planner_input_error(py, message))?; + objects.tuple(&ordered, py) +} + +/// Model names reachable downstream from the given keys, including model roots. +#[pyfunction] +#[pyo3(signature = (*, start_keys, downstream_deps))] +fn build_downstream_model_name_closure<'py>( + start_keys: &Bound<'py, PyAny>, + downstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = downstream_deps.py(); + let starts: Vec> = start_keys.try_iter()?.collect::>()?; + let mut names: Vec = Vec::new(); + for key in closure(starts, downstream_deps, (true, None))?.iter() { + let key: GraphKey = graph_key(&key)?; + if key.kind == MODEL_RESOURCE { + names.push(key.name); + } + } + PyFrozenSet::new(py, names) +} + +fn name_set(names: &Bound<'_, PyAny>) -> PyResult> { + names.try_iter()?.map(|name| name?.extract()).collect() +} + +fn missing_rows<'py>( + missing: &Bound<'py, PyAny>, + objects: &mut KeyObjects<'py>, +) -> PyResult> { + missing + .try_iter()? + .map(|entry| { + let entry: Bound<'py, PyAny> = entry?; + Ok(MissingUpstream { + key: objects.key(entry.getattr("key")?)?, + required_by: objects.keys(&entry.getattr("required_by")?)?, + }) + }) + .collect() +} + +/// Upstream deps of selected keys that exist neither in scope nor in the warehouse. +#[pyfunction] +#[pyo3(signature = (**options))] +fn check_buildability<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "check_buildability", + options, + ( + &["selected_keys", "upstream_deps", "snapshot"], + &["deferred_relations", "satisfied_keys"], + ), + )?; + let selected_keys: Bound<'py, PyAny> = arguments.value("selected_keys")?; + let upstream_deps: Bound<'py, PyAny> = arguments.value("upstream_deps")?; + let snapshot: Bound<'py, PyAny> = arguments.value("snapshot")?; + let deferred_relations: Bound<'py, PyAny> = arguments.value("deferred_relations")?; + let satisfied_keys: Bound<'py, PyAny> = arguments.value("satisfied_keys")?; + let mut objects = KeyObjects::new(); + let selected: Vec = objects.keys(&selected_keys)?; + let mut deps: HashMap> = HashMap::new(); + for key in &selected { + let dep_keys: Bound<'py, PyAny> = + upstream_deps.call_method1("get", (objects.object(key), PyTuple::empty(py)))?; + deps.insert(key.clone(), objects.keys(&dep_keys)?); + } + let available = AvailableUpstreams { + satisfied_keys: if satisfied_keys.is_none() { + HashSet::new() + } else { + objects.keys(&satisfied_keys)?.into_iter().collect() + }, + existing_relations: name_set(&snapshot.getattr("existing_relations")?)?, + deferred_relations: if deferred_relations.is_none() { + None + } else { + Some(name_set(&deferred_relations)?) + }, + }; + let entry_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "MissingUpstream")?; + let mut entries: Vec> = Vec::new(); + for missing in missing_upstreams(&selected, &deps, &available) { + entries.push(call_with( + &entry_class, + &[ + ("key", objects.object(&missing.key)), + ( + "required_by", + objects.tuple(&missing.required_by, py)?.into_any(), + ), + ], + )?); + } + PyTuple::new(py, entries) +} + +/// Describe missing upstream keys, naming the audit when only an audit read requires one. +#[pyfunction] +#[pyo3(signature = (*, missing, edge_origins))] +fn missing_upstream_message<'py>( + missing: &Bound<'py, PyAny>, + edge_origins: &Bound<'py, PyAny>, +) -> PyResult> { + let mut objects = KeyObjects::new(); + let rows: Vec = missing_rows(missing, &mut objects)?; + let origins: HashMap<(GraphKey, GraphKey), String> = + self::edge_origins(edge_origins, &mut objects)?; + Ok(PyString::new(missing.py(), &message_of(&rows, &origins))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_execution_upstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(build_execution_edge_origins, module)?)?; + module.add_function(wrap_pyfunction!(build_downstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(topologically_order_keys, module)?)?; + module.add_function(wrap_pyfunction!( + build_downstream_model_name_closure, + module + )?)?; + module.add_function(wrap_pyfunction!(check_buildability, module)?)?; + module.add_function(wrap_pyfunction!(missing_upstream_message, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs new file mode 100644 index 0000000000..af4645c209 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs @@ -0,0 +1,89 @@ +//! Query and function definition fingerprints for change detection, failing as planner input errors. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{FINGERPRINT_HELP, QUERY_FINGERPRINT_FAILED}; +use sqlbuild_analysis::planner_identity::models::GraphKey; +use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; +use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; +use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::types::CompilerDetach; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; + +/// The query fingerprint of a model, or the planner input error naming the model. +pub(crate) fn model_query_hash( + py: Python<'_>, + model_name: &str, + (query_sql, dialect): (&str, Option<&str>), +) -> PyResult { + py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) + .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) +} + +/// The definition fingerprint of a function, or the planner input error naming the function. +pub(crate) fn function_definition_hash( + py: Python<'_>, + function_name: &str, + (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), +) -> PyResult { + py.compiler_detach(|| definition_fingerprint(fingerprint_sql, language, dialect)) + .map_err(|error| fingerprint_error(py, &format!("Function '{function_name}'"), error)) +} + +fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { + if is_compiler_panic(&message) { + return compiler_error(message); + } + let text = |value: &str| PyString::new(py, value).into_any(); + let raised: PyResult> = imported(py, EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| { + class.call( + (format!( + "{subject} cannot be fingerprinted for change detection: {message}" + ),), + Some(&keywords( + py, + &[ + ("code", text(QUERY_FINGERPRINT_FAILED)), + ("help", text(FINGERPRINT_HELP)), + ], + )?), + ) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. +pub(crate) fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(GraphKey { + kind: key.getattr("resource_type")?.str()?.extract()?, + name: key.getattr("name")?.extract()?, + }) +} + +/// Fingerprint a model query for change detection, naming the model when it cannot. +#[pyfunction] +#[pyo3(signature = (*, model_name, query_sql, dialect))] +fn model_definition_hash( + py: Python<'_>, + model_name: &str, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { + model_query_hash(py, model_name, (query_sql, dialect)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_definition_hash, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs new file mode 100644 index 0000000000..0f06e11456 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -0,0 +1,394 @@ +//! Direct model version identities and honest write hashes over the planner scope. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::expected_graph_identity_hashes::expected_graph_identity_hashes; +use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; +use sqlbuild_analysis::planner_identity::main::model_local_identity_hash::model_local_identity_hash; +use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_query_hash}; +use crate::bindings::_helpers::planner_identity::metadata::{ + function_local_hashes, model_version_identity_metadata_json, seed_identity, +}; + +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +fn key_of(kind: &str, name: &str) -> GraphKey { + GraphKey { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +fn graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| graph_key(&key?)).collect() +} + +fn string_pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { + mapping + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract::<(String, String)>()) + .collect() +} + +/// `{name: identity.version_hash}` of the discovered Python hooks. +fn hook_version_hashes<'py>(hook_functions: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = hook_functions.py(); + let hashes: Bound<'py, PyDict> = PyDict::new(py); + let identities: Bound<'py, PyAny> = + imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")?.call1((hook_functions,))?; + for item in identities.call_method0("items")?.try_iter()? { + let (name, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + hashes.set_item(name, identity.getattr("version_hash")?)?; + } + Ok(hashes) +} + +/// Current direct model identities from code and upstream identities. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_direct_model_version_identities<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_direct_model_version_identities", + options, + ( + &["functions", "scope", "dialect"], + &["seeds", "source_version_hashes", "hook_functions"], + ), + )?; + let functions: Bound<'py, PyAny> = arguments.value("functions")?; + let scope: Bound<'py, PyAny> = arguments.value("scope")?; + let dialect: Option = arguments.value("dialect")?.extract()?; + let function_hashes: Bound<'py, PyDict> = + function_local_hashes(&functions, dialect.as_deref())?; + let hooks: Bound<'py, PyAny> = arguments.value("hook_functions")?; + let hook_hashes: Bound<'py, PyDict> = if hooks.is_none() { + PyDict::new(py) + } else { + hook_version_hashes(&hooks)? + }; + let mut graph = IdentityGraph::default(); + for function in functions.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let name: String = function.getattr("name")?.extract()?; + let Some(function_hash) = function_hashes.get_item(&name)? else { + continue; + }; + graph.nodes.insert( + graph_key(&function.getattr("key")?)?, + IdentityNode { + upstream_keys: graph_keys(&function.getattr("deps")?)?, + local_hash: Some(function_hash.extract()?), + }, + ); + } + let seed_hashes: Bound<'py, PyDict> = PyDict::new(py); + let seed_metadata: Bound<'py, PyDict> = PyDict::new(py); + let seeds: Bound<'py, PyAny> = arguments.value("seeds")?; + if !seeds.is_none() { + for seed in seeds.try_iter()? { + let seed: Bound<'py, PyAny> = seed?; + let (seed_hash, metadata_json): (String, String) = seed_identity(&seed)?; + let name: Bound<'py, PyAny> = seed.getattr("name")?; + seed_hashes.set_item(&name, &seed_hash)?; + seed_metadata.set_item(&name, metadata_json)?; + } + } + for (name, seed_hash) in string_pairs(seed_hashes.as_any())? { + graph.nodes.insert( + key_of(SEED_RESOURCE, &name), + IdentityNode { + upstream_keys: Vec::new(), + local_hash: Some(seed_hash), + }, + ); + } + let source_hashes: Bound<'py, PyAny> = arguments.value("source_version_hashes")?; + if source_hashes.is_truthy()? { + for (name, source_hash) in string_pairs(&source_hashes)? { + graph.nodes.insert( + key_of(SOURCE_RESOURCE, &name), + IdentityNode { + upstream_keys: Vec::new(), + local_hash: Some(source_hash), + }, + ); + } + } + let model_metadata: Bound<'py, PyDict> = PyDict::new(py); + let model_local: Bound<'py, PyDict> = PyDict::new(py); + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + let order_key: GraphKey = graph_key(&key)?; + graph.execution_order.push(order_key.clone()); + if order_key.kind != MODEL_RESOURCE { + continue; + } + let model: Bound<'py, PyAny> = models_by_name.call_method1("get", (&order_key.name,))?; + if model.is_none() { + continue; + } + let name: String = model.getattr("name")?.extract()?; + let metadata_json: String = model_version_identity_metadata_json( + &model, + Some(function_hashes.as_any()), + Some(hook_hashes.as_any()), + )?; + let query_fingerprint: String = model_query_hash( + py, + &name, + ( + &model.getattr("query_sql")?.extract::()?, + dialect.as_deref(), + ), + )?; + let local_hash: String = model_local_identity_hash(&query_fingerprint, &metadata_json); + model_metadata.set_item(&name, metadata_json)?; + model_local.set_item(&name, &local_hash)?; + graph.nodes.insert( + key_of(MODEL_RESOURCE, &name), + IdentityNode { + upstream_keys: graph_keys(&model.getattr("deps")?)?, + local_hash: Some(local_hash), + }, + ); + } + let graph_hashes: HashMap> = + expected_graph_identity_hashes(&graph).into_iter().collect(); + let version_hashes: Bound<'py, PyDict> = function_hashes.copy()?; + version_hashes.update(seed_hashes.as_mapping())?; + for name in model_local.keys() { + let name: String = name.extract()?; + if let Some(Some(model_hash)) = graph_hashes.get(&key_of(MODEL_RESOURCE, &name)) { + version_hashes.set_item(&name, model_hash)?; + } + } + call_with( + &imported(py, MODELS_MODULE, "DirectModelVersionIdentities")?, + &[ + ("function_local_hashes", function_hashes.into_any()), + ("seed_version_hashes", seed_hashes.into_any()), + ("seed_metadata_jsons", seed_metadata.into_any()), + ("model_metadata_jsons", model_metadata.into_any()), + ("model_local_hashes", model_local.into_any()), + ("model_version_hashes", version_hashes.into_any()), + ], + ) +} + +/// Planned write hashes replaced by hashes based on upstreams available this run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn with_honest_model_write_hashes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "with_honest_model_write_hashes", + options, + ( + &["scope", "snapshot", "changes", "version_identities"], + &["available_model_hashes"], + ), + )?; + let scope: Bound<'py, PyAny> = arguments.value("scope")?; + let snapshot: Bound<'py, PyAny> = arguments.value("snapshot")?; + let changes: Bound<'py, PyAny> = arguments.value("changes")?; + let identities: Bound<'py, PyAny> = arguments.value("version_identities")?; + let selected_python: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let selected: HashSet = graph_keys(&selected_python)?.into_iter().collect(); + let function_hashes: Vec<(String, String)> = + string_pairs(&identities.getattr("function_local_hashes")?)?; + let mut graph = IdentityGraph::default(); + let mut base_hashes: Vec<(GraphKey, String)> = Vec::new(); + let mut function_deps: HashMap> = HashMap::new(); + for item in scope + .getattr("upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (key, deps): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let key: GraphKey = graph_key(&key)?; + if key.kind == UDF_RESOURCE || key.kind == TABLE_FN_RESOURCE { + function_deps.insert(key, graph_keys(&deps)?); + } + } + for (name, function_hash) in &function_hashes { + for kind in [UDF_RESOURCE, TABLE_FN_RESOURCE] { + let function_key: GraphKey = key_of(kind, name); + graph.nodes.insert( + function_key.clone(), + IdentityNode { + upstream_keys: function_deps + .get(&function_key) + .cloned() + .unwrap_or_default(), + local_hash: Some(function_hash.clone()), + }, + ); + base_hashes.push((function_key, function_hash.clone())); + } + } + let recorded_seeds: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("seeds")?; + for (name, seed_hash) in string_pairs(&identities.getattr("seed_version_hashes")?)? { + let seed_key: GraphKey = key_of(SEED_RESOURCE, &name); + graph.nodes.insert( + seed_key.clone(), + IdentityNode { + upstream_keys: Vec::new(), + local_hash: Some(seed_hash.clone()), + }, + ); + let fingerprint: Bound<'py, PyAny> = recorded_seeds.call_method1("get", (&name,))?; + let write_hash: String = if selected.contains(&seed_key) || fingerprint.is_none() { + seed_hash + } else { + fingerprint.getattr("version_hash")?.extract()? + }; + base_hashes.push((seed_key, write_hash)); + } + let local_hashes: Bound<'py, PyAny> = identities.getattr("model_local_hashes")?; + for model in scope + .getattr("models_by_name")? + .call_method0("values")? + .try_iter()? + { + let model: Bound<'py, PyAny> = model?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + graph.nodes.insert( + key_of(MODEL_RESOURCE, &name.extract::()?), + IdentityNode { + upstream_keys: graph_keys(&model.getattr("deps")?)?, + local_hash: local_hashes.call_method1("get", (&name,))?.extract()?, + }, + ); + } + let recorded_models: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("models")?; + for item in recorded_models.call_method0("items")?.try_iter()? { + let (name, fingerprint): (String, Bound<'py, PyAny>) = item?.extract()?; + let version_hash: Bound<'py, PyAny> = fingerprint.getattr("version_hash")?; + if version_hash.is_truthy()? { + base_hashes.push((key_of(MODEL_RESOURCE, &name), version_hash.extract()?)); + } + } + let available: Bound<'py, PyAny> = arguments.value("available_model_hashes")?; + if available.is_truthy()? { + for (name, model_hash) in string_pairs(&available)? { + base_hashes.push((key_of(MODEL_RESOURCE, &name), model_hash)); + } + } + graph.execution_order = graph_keys(&scope.getattr("execution_order")?)?; + let write_hashes: HashMap = + graph_write_identity_hashes(&graph, &selected, base_hashes) + .into_iter() + .collect(); + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let model_changes: Bound<'py, PyDict> = PyDict::new(py); + model_changes.update( + changes + .getattr("models")? + .downcast::()? + .as_mapping(), + )?; + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + let order_key: GraphKey = graph_key(&key)?; + if order_key.kind != MODEL_RESOURCE || !selected.contains(&order_key) { + continue; + } + let model: Bound<'py, PyAny> = models_by_name.call_method1("get", (&order_key.name,))?; + if model.is_none() { + continue; + } + let name: Bound<'py, PyAny> = model.getattr("name")?; + if local_hashes.call_method1("get", (&name,))?.is_none() { + continue; + } + let Some(write_hash) = + write_hashes.get(&key_of(MODEL_RESOURCE, &name.extract::()?)) + else { + continue; + }; + let Some(change) = model_changes.get_item(&name)? else { + continue; + }; + model_changes.set_item( + &name, + replace_with( + &replace, + &change, + ( + "fingerprint_version_hash", + PyString::new(py, write_hash).into_any(), + ), + )?, + )?; + } + replace_with(&replace, &changes, ("models", model_changes.into_any())) +} + +/// Resolved actions with recomputed model changes patched in where available. +#[pyfunction] +#[pyo3(signature = (*, resolved_actions, changes))] +fn merge_recomputed_model_changes<'py>( + resolved_actions: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = resolved_actions.py(); + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let recomputed: Bound<'py, PyAny> = changes.getattr("models")?; + let merged: Bound<'py, PyDict> = PyDict::new(py); + for item in resolved_actions + .getattr("models")? + .call_method0("items")? + .try_iter()? + { + let (name, resolved): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let mut change: Bound<'py, PyAny> = recomputed.call_method1("get", (&name,))?; + if change.is_none() { + change = resolved.getattr("change")?; + } + merged.set_item(name, replace_with(&replace, &resolved, ("change", change))?)?; + } + replace_with(&replace, resolved_actions, ("models", merged.into_any())) +} + +fn replace_with<'py>( + replace: &Bound<'py, PyAny>, + instance: &Bound<'py, PyAny>, + (field, value): (&str, Bound<'py, PyAny>), +) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(replace.py()); + keywords.set_item(field, value)?; + replace.call((instance,), Some(&keywords)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_direct_model_version_identities, + module + )?)?; + module.add_function(wrap_pyfunction!(with_honest_model_write_hashes, module)?)?; + module.add_function(wrap_pyfunction!(merge_recomputed_model_changes, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs new file mode 100644 index 0000000000..382d078218 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs @@ -0,0 +1,150 @@ +//! Python values as `json.dumps(value, sort_keys=True, ...)` writes them, with or without `default=str`. + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, + PyTypeMethods, +}; + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +const BUILTINS_MODULE: &str = "builtins"; + +/// Compact `(",", ":")` separators. +pub(crate) const COMPACT: (&str, &str) = (",", ":"); +/// Python's default `(", ", ": ")` separators. +pub(crate) const SPACED: (&str, &str) = (", ", ": "); + +/// `json.dumps(value, sort_keys=True, separators=separators)`, with `default=str` when `default_str`. +pub(crate) fn sorted_dumps( + value: &Bound<'_, PyAny>, + separators: (&str, &str), + default_str: bool, +) -> PyResult { + let encoded: JsonValue = Encoder::new(value.py(), default_str)?.encode(value)?; + let dialect = JsonDialect::Stdlib( + StdlibJsonOptions::new(None).with_separators(separators.0, separators.1), + ); + dumps(&encoded, &dialect).map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +/// The C encoder's type dispatch, with dictionaries ordered as `sorted(dict.items())`. +struct Encoder<'py> { + default_str: bool, + int_repr: Bound<'py, PyAny>, + sorted: Bound<'py, PyAny>, +} + +impl<'py> Encoder<'py> { + fn new(py: Python<'py>, default_str: bool) -> PyResult { + let builtins = py.import(BUILTINS_MODULE)?; + Ok(Self { + default_str, + int_repr: builtins.getattr("int")?.getattr("__repr__")?, + sorted: builtins.getattr("sorted")?, + }) + } + + fn encode(&self, value: &Bound<'py, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(JsonValue::String(value.extract()?)); + } + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(JsonValue::Integer(self.integer(value)?)); + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(JsonValue::Array( + value + .try_iter()? + .map(|item| self.encode(&item?)) + .collect::>()?, + )); + } + if let Ok(mapping) = value.downcast::() { + return self.object(mapping); + } + if self.default_str { + return Ok(JsonValue::String(value.str()?.extract()?)); + } + Err(PyTypeError::new_err(format!( + "Object of type {} is not JSON serializable", + value.get_type().name()? + ))) + } + + fn integer(&self, value: &Bound<'py, PyAny>) -> PyResult { + let decimal: String = self.int_repr.call1((value,))?.extract()?; + JsonInteger::parse(&decimal) + .ok_or_else(|| PyValueError::new_err(format!("invalid integer text {decimal}"))) + } + + fn object(&self, mapping: &Bound<'py, PyDict>) -> PyResult { + let mut keys: Vec> = mapping.keys().into_iter().collect(); + if keys.iter().all(|key| key.is_instance_of::()) { + let mut texts: Vec<(String, Bound<'py, PyAny>)> = keys + .into_iter() + .map(|key| Ok((key.extract::()?, key))) + .collect::>()?; + texts.sort_by(|left, right| left.0.cmp(&right.0)); + keys = texts.into_iter().map(|(_, key)| key).collect(); + } else { + keys = self + .sorted + .call1((mapping.keys(),))? + .try_iter()? + .collect::>()?; + } + let mut entries: Vec<(String, JsonValue)> = Vec::with_capacity(keys.len()); + for key in keys { + let item: Bound<'py, PyAny> = mapping.as_any().get_item(&key)?; + entries.push((self.key_text(&key)?, self.encode(&item)?)); + } + Ok(JsonValue::Object(entries)) + } + + fn key_text(&self, key: &Bound<'py, PyAny>) -> PyResult { + if key.is_instance_of::() { + return key.extract(); + } + if key.is_instance_of::() { + let number: f64 = key.extract()?; + return Ok(if number.is_nan() { + "NaN".to_owned() + } else if number.is_infinite() { + if number.is_sign_positive() { + "Infinity" + } else { + "-Infinity" + } + .to_owned() + } else { + python_float_repr(number) + }); + } + if let Ok(flag) = key.downcast::() { + return Ok(if flag.is_true() { "true" } else { "false" }.to_owned()); + } + if key.is_none() { + return Ok("null".to_owned()); + } + if key.is_instance_of::() { + return Ok(self.integer(key)?.as_str().to_owned()); + } + Err(PyTypeError::new_err(format!( + "keys must be str, int, float, bool or None, not {}", + key.get_type().name()? + ))) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs new file mode 100644 index 0000000000..15dcb30116 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -0,0 +1,512 @@ +//! Version identity metadata JSON and hashes of models, functions and seeds, byte-identical to the recorded fingerprints. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + CONTRACT_CONFIG_KEY, CURSOR_INPUTS_CONFIG_KEY, ENFORCED_CONTRACT_POLICY, + MODEL_CUSTOM_CONFIG_KEY, MODEL_PLACEHOLDERS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY, + MODEL_PRE_HOOKS_CONFIG_KEY, SQL_HOOK_IDENTITY_FIELDS, TABLE_FN_RESOURCE, UDF_RESOURCE, + VERSION_IDENTITY_CONFIG_KEYS, +}; +use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; +use sqlbuild_analysis::planner_identity::main::stable_identity_hash::stable_identity_hash; +use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; + +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords, within}; +use crate::bindings::_helpers::discovery::hook_entries::hook_entries; +use crate::bindings::_helpers::planner_identity::hashing::function_definition_hash; +use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, SPACED, sorted_dumps}; + +const LOCAL_FUNCTION_HASHES_KEY: &str = "local_function_hashes"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const CSV_MODULE: &str = "csv"; +const BUILTINS_MODULE: &str = "builtins"; +const DEFAULT_SEED_ENCODING: &str = "utf-8-sig"; +/// CSV settings passed to `csv.reader` when set. +const SEED_DIALECT_OPTIONS: [&str; 5] = [ + "delimiter", + "quotechar", + "doublequote", + "escapechar", + "skipinitialspace", +]; + +/// The config entries that affect a model's produced version identity. +fn version_identity_config<'py>(config_values: &Bound<'py, PyAny>) -> PyResult> { + let identity: Bound<'py, PyDict> = PyDict::new(config_values.py()); + for key in std::iter::once(CURSOR_INPUTS_CONFIG_KEY).chain(VERSION_IDENTITY_CONFIG_KEYS) { + if config_values.contains(key)? { + identity.set_item(key, config_values.get_item(key)?)?; + } + } + Ok(identity) +} + +/// `value or {}`. +fn or_empty<'py>(value: Option>, py: Python<'py>) -> PyResult> { + match value { + Some(value) if value.is_truthy()? => Ok(value), + _ => Ok(PyDict::new(py).into_any()), + } +} + +/// Deterministic metadata JSON for model version identity. +pub(crate) fn version_identity_metadata_json( + model_name: &Bound<'_, PyAny>, + config_values: &Bound<'_, PyAny>, + (local_function_hashes, execution_signature): ( + Option>, + Option>, + ), +) -> PyResult { + let py: Python<'_> = model_name.py(); + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("config", version_identity_config(config_values)?)?; + payload.set_item("execution_signature", or_empty(execution_signature, py)?)?; + payload.set_item( + LOCAL_FUNCTION_HASHES_KEY, + or_empty(local_function_hashes, py)?, + )?; + payload.set_item("model_name", model_name)?; + sorted_dumps(payload.as_any(), COMPACT, true) +} + +/// Deterministic non-query model fingerprint metadata JSON. +pub(crate) fn model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option<&Bound<'py, PyAny>>, + hook_version_hashes: Option<&Bound<'py, PyAny>>, +) -> PyResult { + let py: Python<'py> = model.py(); + let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); + if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { + for upstream_key in model.getattr("deps")?.try_iter()? { + let upstream_key: Bound<'py, PyAny> = upstream_key?; + let kind: String = upstream_key.getattr("resource_type")?.str()?.extract()?; + if kind != UDF_RESOURCE && kind != TABLE_FN_RESOURCE { + continue; + } + let name: Bound<'py, PyAny> = upstream_key.getattr("name")?; + let upstream_hash: Bound<'py, PyAny> = function_hashes.call_method1("get", (&name,))?; + if !upstream_hash.is_none() { + local_function_hashes.set_item(name, upstream_hash)?; + } + } + } + let config_values: Bound<'py, PyAny> = model.getattr("config")?.getattr("values")?; + let signature: Bound<'py, PyDict> = + execution_signature(model, &config_values, hook_version_hashes)?; + version_identity_metadata_json( + &model.getattr("name")?, + &config_values, + ( + Some(local_function_hashes.into_any()), + Some(signature.into_any()), + ), + ) +} + +fn execution_signature<'py>( + model: &Bound<'py, PyAny>, + config_values: &Bound<'py, PyAny>, + hook_version_hashes: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let signature: Bound<'py, PyDict> = PyDict::new(py); + if let Some(contract) = contract_output_signature(model, config_values)? { + signature.set_item("contract", contract)?; + } + for (config_key, signature_key) in [ + (MODEL_CUSTOM_CONFIG_KEY, "custom_config"), + (MODEL_PLACEHOLDERS_CONFIG_KEY, "custom_placeholders"), + ] { + if config_values.contains(config_key)? { + signature.set_item(signature_key, config_values.get_item(config_key)?)?; + } + } + let python_hook_fields: Bound<'py, PyDict> = PyDict::new(py); + if let Some(hashes) = hook_version_hashes.filter(|hashes| !hashes.is_none()) { + for item in hashes.call_method0("items")?.try_iter()? { + let (name, version_hash): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("version_hash", version_hash)?; + python_hook_fields.set_item(name, fields)?; + } + } + let sql_fields: Vec = SQL_HOOK_IDENTITY_FIELDS + .iter() + .map(|field| (*field).to_owned()) + .collect(); + for config_key in [MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY] { + if config_values.contains(config_key)? { + signature.set_item( + config_key, + hook_entries( + &config_values.get_item(config_key)?, + &sql_fields, + Some(&python_hook_fields), + )?, + )?; + } + } + Ok(signature) +} + +/// Required model output metadata that participates in execution identity. +fn contract_output_signature<'py>( + model: &Bound<'py, PyAny>, + config_values: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = model.py(); + let schema_entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if schema_entry.is_none() { + return Ok(None); + } + let columns: Bound<'py, PyAny> = schema_entry.getattr("columns")?; + let families: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + if !(columns.is_truthy()? || families.is_truthy()?) { + return Ok(None); + } + let enforced: bool = config_values + .call_method1("get", (CONTRACT_CONFIG_KEY,))? + .eq(ENFORCED_CONTRACT_POLICY)?; + if !enforced && schema_entry.getattr("model_schema")?.is_none() { + return Ok(None); + } + let enum_columns: Bound<'py, PyAny> = model.getattr("enum_columns")?; + let rendered: Bound<'py, PyList> = PyList::empty(py); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let entry: Bound<'py, PyDict> = column_signature(&column)?; + let declaration: Bound<'py, PyAny> = + enum_columns.call_method1("get", (column.getattr("name")?,))?; + if !declaration.is_none() { + let members: Bound<'py, PyList> = PyList::empty(py); + for member in declaration.getattr("members")?.try_iter()? { + let member: Bound<'py, PyAny> = member?; + let rendered_member: Bound<'py, PyDict> = PyDict::new(py); + rendered_member.set_item("name", member.getattr("name")?)?; + rendered_member.set_item("value", member.getattr("value")?)?; + members.append(rendered_member)?; + } + let enum_entry: Bound<'py, PyDict> = PyDict::new(py); + enum_entry.set_item("name", declaration.getattr("name")?)?; + enum_entry.set_item("members", members)?; + entry.set_item("enum", enum_entry)?; + } + rendered.append(entry)?; + } + let signature: Bound<'py, PyDict> = PyDict::new(py); + signature.set_item("enforced", enforced)?; + signature.set_item("columns", rendered)?; + if families.is_truthy()? { + signature.set_item("dynamic_columns", family_signatures(&families)?)?; + } + Ok(Some(signature)) +} + +fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult> { + let entry: Bound<'py, PyDict> = PyDict::new(column.py()); + for field in ["name", "type", "nullable"] { + entry.set_item(field, column.getattr(field)?)?; + } + Ok(entry) +} + +fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult> { + let rendered: Bound<'py, PyList> = PyList::empty(families.py()); + for family in families.try_iter()? { + let family: Bound<'py, PyAny> = family?; + let entry: Bound<'py, PyDict> = PyDict::new(families.py()); + for field in [ + "name", + "pivot_column", + "value_column", + "aggregate", + "type", + "name_pattern", + ] { + entry.set_item(field, family.getattr(field)?)?; + } + rendered.append(entry)?; + } + Ok(rendered) +} + +/// The hash of a model's own declared columns, column families and contract setting. +pub(crate) fn model_declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + let schema_entry: Bound<'_, PyAny> = model.getattr("schema_entry")?; + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item( + "contract", + model + .getattr("config")? + .getattr("values")? + .call_method1("get", (CONTRACT_CONFIG_KEY,))?, + )?; + let columns: Bound<'_, PyList> = PyList::empty(py); + let mut families: Bound<'_, PyList> = PyList::empty(py); + let mut type_enforcement: Bound<'_, PyAny> = py.None().into_bound(py); + if !schema_entry.is_none() { + for column in schema_entry.getattr("columns")?.try_iter()? { + columns.append(column_signature(&column?)?)?; + } + families = family_signatures(&schema_entry.getattr("dynamic_columns")?)?; + type_enforcement = schema_entry.getattr("type_enforcement")?; + } + payload.set_item("columns", columns)?; + payload.set_item("dynamic_columns", families)?; + payload.set_item("type_enforcement", type_enforcement)?; + Ok(stable_identity_hash(&sorted_dumps( + payload.as_any(), + COMPACT, + true, + )?)) +} + +/// Local-only semantic hashes of functions by name; SQL bodies ignore layout and comments. +pub(crate) fn function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, +) -> PyResult> { + let py: Python<'py> = functions.py(); + let hashes: Bound<'py, PyDict> = PyDict::new(py); + for function in functions.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let name: String = function.getattr("name")?.extract()?; + let language: Bound<'py, PyAny> = function.getattr("language")?.getattr("value")?; + let body_hash: String = function_definition_hash( + py, + &name, + ( + &function.getattr("body_sql")?.extract::()?, + &language.extract::()?, + dialect, + ), + )?; + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item( + "arguments", + name_type_pairs(&function.getattr("arguments")?)?, + )?; + payload.set_item("returns", function.getattr("returns")?)?; + payload.set_item( + "return_columns", + name_type_pairs(&function.getattr("return_columns")?)?, + )?; + payload.set_item("body_sql", body_hash)?; + payload.set_item("language", language)?; + for field in ["runtime_version", "entry_point", "packages"] { + payload.set_item(field, function.getattr(field)?)?; + } + hashes.set_item( + name, + stable_identity_hash(&sorted_dumps(payload.as_any(), SPACED, true)?), + )?; + } + Ok(hashes) +} + +fn name_type_pairs<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let pairs: Bound<'py, PyList> = PyList::empty(items.py()); + for item in items.try_iter()? { + let item: Bound<'py, PyAny> = item?; + pairs.append(PyTuple::new( + items.py(), + [item.getattr("name")?, item.getattr("type")?], + )?)?; + } + Ok(pairs) +} + +/// `f"{value or ''}"` of an optional attribute. +fn text_or_empty(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { + let value: Bound<'_, PyAny> = match owner.getattr(name) { + Ok(value) => value, + Err(_) => return Ok(String::new()), + }; + if value.is_truthy()? { + value.str()?.extract() + } else { + Ok(String::new()) + } +} + +fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { + items + .try_iter()? + .map(|item| { + let item: Bound<'_, PyAny> = item?; + Ok((text_or_empty(&item, "name")?, text_or_empty(&item, "type")?)) + }) + .collect() +} + +/// Stable function definition text used for change fingerprints. +pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + Ok(function_fingerprint_sql(&FunctionFingerprintFields { + language: function.getattr("language")?.str()?.extract()?, + arguments: rendered_pairs(&function.getattr("arguments")?)?, + returns: function.getattr("returns")?.str()?.extract()?, + return_columns: rendered_pairs(&function.getattr("return_columns")?)?, + runtime_version: text_or_empty(function, "runtime_version")?, + entry_point: text_or_empty(function, "entry_point")?, + packages: function + .getattr("packages")? + .try_iter()? + .map(|package| package?.extract::()) + .collect::>()?, + body_sql: function.getattr("body_sql")?.extract()?, + })) +} + +/// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. +pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + let py: Python<'_> = seed.py(); + let schema_entry: Bound<'_, PyAny> = seed.getattr("schema_entry")?; + let csv_settings: Bound<'_, PyAny> = schema_entry.getattr("csv_settings")?; + let columns: Bound<'_, PyList> = PyList::empty(py); + for column in schema_entry.getattr("columns")?.try_iter()? { + columns.append(column_signature(&column?)?)?; + } + let rows: Bound<'_, PyAny> = if seed.getattr("external")?.is_truthy()? { + PyList::empty(py).into_any() + } else { + seed_rows( + &seed.getattr("seed_file")?.getattr("file_path")?, + &csv_settings, + )? + }; + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("name", seed.getattr("name")?)?; + payload.set_item( + "csv_settings", + imported(py, DATACLASSES_MODULE, "asdict")?.call1((&csv_settings,))?, + )?; + payload.set_item("columns", columns)?; + payload.set_item("rows", rows)?; + let metadata_json: String = sorted_dumps(payload.as_any(), COMPACT, false)?; + Ok((stable_identity_hash(&metadata_json), metadata_json)) +} + +/// Every row of the seed file as the standard library CSV reader yields it. +fn seed_rows<'py>( + file_path: &Bound<'py, PyAny>, + csv_settings: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let encoding: Bound<'py, PyAny> = match csv_settings.getattr("encoding")? { + value if value.is_truthy()? => value, + _ => PyString::new(py, DEFAULT_SEED_ENCODING).into_any(), + }; + let options: Bound<'py, PyDict> = PyDict::new(py); + for option in SEED_DIALECT_OPTIONS { + let value: Bound<'py, PyAny> = csv_settings.getattr(option)?; + if !value.is_none() { + options.set_item(option, value)?; + } + } + let handle: Bound<'py, PyAny> = imported(py, BUILTINS_MODULE, "open")?.call( + (file_path, "r"), + Some(&keywords( + py, + &[ + ("encoding", encoding), + ("newline", PyString::new(py, "").into_any()), + ], + )?), + )?; + within(&handle, |entered| { + let rows: Bound<'py, PyList> = PyList::empty(py); + let reader: Bound<'py, PyAny> = + imported(py, CSV_MODULE, "reader")?.call((entered,), Some(&options))?; + for row in reader.try_iter()? { + rows.append(PyList::new( + py, + row?.try_iter()?.collect::>>()?, + )?)?; + } + Ok(rows.into_any()) + }) +} + +/// Deterministic metadata JSON for model version identity. +#[pyfunction] +#[pyo3(signature = (*, model_name, config_values, local_function_hashes = None, execution_signature = None))] +fn build_version_identity_metadata_json<'py>( + model_name: &Bound<'py, PyAny>, + config_values: &Bound<'py, PyAny>, + local_function_hashes: Option>, + execution_signature: Option>, +) -> PyResult { + version_identity_metadata_json( + model_name, + config_values, + (local_function_hashes, execution_signature), + ) +} + +/// Deterministic non-query model fingerprint metadata JSON. +#[pyfunction] +#[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] +fn build_model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option>, + hook_version_hashes: Option>, +) -> PyResult { + model_version_identity_metadata_json( + model, + function_local_hashes.as_ref(), + hook_version_hashes.as_ref(), + ) +} + +/// Hash the model's own declared columns, column families and contract setting. +#[pyfunction] +#[pyo3(signature = (*, model))] +fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { + model_declared_columns_hash(model) +} + +/// Local-only semantic hashes of functions by name. +#[pyfunction] +#[pyo3(signature = (*, functions, dialect))] +fn build_function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, +) -> PyResult> { + function_local_hashes(functions, dialect) +} + +/// Stable function definition text used for change fingerprints. +#[pyfunction] +fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + compiled_function_fingerprint_sql(function) +} + +/// `(identity_hash, metadata_json)` for one compiled seed. +#[pyfunction] +fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + seed_identity(seed) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_model_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!(declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(build_function_local_hashes, module)?)?; + module.add_function(wrap_pyfunction!( + build_compiled_function_fingerprint_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_seed_identity, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs new file mode 100644 index 0000000000..638074cc97 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod graph; +pub(crate) mod hashing; +pub(crate) mod identities; +pub(crate) mod json_values; +pub(crate) mod metadata; +pub(crate) mod registration; +pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs new file mode 100644 index 0000000000..f9df4e384b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs @@ -0,0 +1,15 @@ +//! Register planner identity, pruning and graph-order bindings. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_identity::{ + graph, hashing, identities, metadata, staleness, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + graph::register(module)?; + hashing::register(module)?; + identities::register(module)?; + metadata::register(module)?; + staleness::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs new file mode 100644 index 0000000000..f1a0332f64 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -0,0 +1,290 @@ +//! Warnings for selected models stale through changed upstreams outside the run set. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::classify_selection_staleness::classify_selection_staleness; +use sqlbuild_analysis::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; +use sqlbuild_analysis::planner_identity::models::{ + GraphKey, StaleWarning, StalenessGraph, StalenessNames, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; +use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_identity::hashing::graph_key; + +/// Change kinds that mean the model's own identity changed. +const OWN_CHANGE_KINDS: [&str; 5] = [ + "first_run", + "query_changed", + "function_changed", + "config_changed", + "schema_changed", +]; + +fn own_change(change: &Bound<'_, PyAny>) -> PyResult { + let kind: String = change.getattr("change_kind")?.str()?.extract()?; + Ok(OWN_CHANGE_KINDS.contains(&kind.as_str())) +} + +fn names_of_kind(keys: &Bound<'_, PyAny>, kind: &str) -> PyResult> { + let mut names: HashSet = HashSet::new(); + for key in keys.try_iter()? { + let key: GraphKey = graph_key(&key?)?; + if key.kind == kind { + names.insert(key.name); + } + } + Ok(names) +} + +fn upstream_graph(upstream_deps: &Bound<'_, PyAny>) -> PyResult>> { + let mut deps: HashMap> = HashMap::new(); + for item in upstream_deps.call_method0("items")?.try_iter()? { + let (key, upstream): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let upstream_keys: Vec = upstream + .try_iter()? + .map(|upstream_key| graph_key(&upstream_key?)) + .collect::>()?; + deps.insert(graph_key(&key)?, upstream_keys); + } + Ok(deps) +} + +/// Inputs of one stale-warning pass. +struct StalenessInputs<'py> { + original_scope: Bound<'py, PyAny>, + changes: Bound<'py, PyAny>, + snapshot: Bound<'py, PyAny>, + identities: Bound<'py, PyAny>, + model_changes_complete: bool, +} + +fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult> { + let py: Python<'_> = inputs.snapshot.py(); + let models: Bound<'_, PyAny> = inputs.changes.getattr("models")?; + let mut changed: HashSet = HashSet::new(); + for item in models.call_method0("items")?.try_iter()? { + let (name, change): (String, Bound<'_, PyAny>) = item?.extract()?; + if own_change(&change)? { + changed.insert(name); + } + } + let version_hashes: Bound<'_, PyAny> = inputs.identities.getattr("model_version_hashes")?; + let metadata_jsons: Bound<'_, PyAny> = inputs.identities.getattr("model_metadata_jsons")?; + let pass = ModelPass { + snapshot: inputs.snapshot.clone(), + query_change_tracking: true, + full_refresh: false, + function_local_hashes: py.None().into_bound(py), + hook_version_hashes: py.None().into_bound(py), + }; + let models_by_name: Bound<'_, PyAny> = inputs.original_scope.getattr("models_by_name")?; + for item in models_by_name.call_method0("items")?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let selected_change: Bound<'_, PyAny> = models.call_method1("get", (&name,))?; + if !selected_change.is_none() + && (inputs.model_changes_complete || own_change(&selected_change)?) + { + if own_change(&selected_change)? { + changed.insert(name); + } + continue; + } + if model.is_none() { + continue; + } + let change: Bound<'_, PyAny> = model_change( + &pass, + &model, + ( + version_hashes.call_method1("get", (&name,))?.extract()?, + metadata_jsons.call_method1("get", (&name,))?.extract()?, + ), + )?; + if own_change(&change)? { + changed.insert(name); + } + } + Ok(changed) +} + +fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> { + let recorded: Bound<'_, PyAny> = inputs.snapshot.getattr("fingerprints")?.getattr("seeds")?; + let mut changed: HashSet = HashSet::new(); + for item in inputs + .identities + .getattr("seed_version_hashes")? + .call_method0("items")? + .try_iter()? + { + let (name, expected): (String, Bound<'_, PyAny>) = item?.extract()?; + if expected.is_none() { + continue; + } + let fingerprint: Bound<'_, PyAny> = recorded.call_method1("get", (&name,))?; + if fingerprint.is_none() || !fingerprint.getattr("version_hash")?.eq(&expected)? { + changed.insert(name); + } + } + Ok(changed) +} + +fn plan_warnings<'py>( + py: Python<'py>, + warnings: Vec, +) -> PyResult> { + let warning_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "PlanWarning")?; + let severity: Bound<'py, PyAny> = planner_enum(py, "WarningSeverity", "warning")?; + let mut rendered: Vec> = Vec::with_capacity(warnings.len()); + for warning in warnings { + let message: String = stale_upstream_warning_message( + ("selected model", &warning.model_name), + "upstream(s)", + &warning.trigger_names, + ); + rendered.push(call_with( + &warning_class, + &[ + ( + "model_name", + PyString::new(py, &warning.model_name).into_any(), + ), + ("severity", severity.clone()), + ("message", PyString::new(py, &message).into_any()), + ], + )?); + } + PyTuple::new(py, rendered) +} + +/// Warn for selected models stale through changed upstreams outside the run set. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_stale_out_of_selection_warnings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_stale_out_of_selection_warnings", + options, + ( + &[ + "original_scope", + "execution_scope", + "changes", + "snapshot", + "version_identities", + "source_freshness", + ], + &["include_sources", "model_changes_complete"], + ), + )?; + let include_sources: bool = + !arguments.passed("include_sources")? || arguments.flag("include_sources")?; + let inputs = StalenessInputs { + original_scope: arguments.value("original_scope")?, + changes: arguments.value("changes")?, + snapshot: arguments.value("snapshot")?, + identities: arguments.value("version_identities")?, + model_changes_complete: arguments.flag("model_changes_complete")?, + }; + let run_keys: Bound<'py, PyAny> = arguments + .value("execution_scope")? + .getattr("selected_keys")?; + let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; + let mut changed_sources: HashSet = HashSet::new(); + if include_sources && !source_freshness.is_none() { + for identity in source_freshness.getattr("changed_identities")?.try_iter()? { + changed_sources.insert(identity?.getattr("source_name")?.extract()?); + } + } + let graph = StalenessGraph { + upstream_deps: upstream_graph(&inputs.original_scope.getattr("upstream_deps")?)?, + selected_model_names: names_of_kind( + &inputs.original_scope.getattr("selected_keys")?, + MODEL_RESOURCE, + )? + .into_iter() + .collect(), + run: StalenessNames { + models: names_of_kind(&run_keys, MODEL_RESOURCE)?, + seeds: names_of_kind(&run_keys, SEED_RESOURCE)?, + sources: names_of_kind(&run_keys, SOURCE_RESOURCE)?, + }, + changed: StalenessNames { + models: changed_model_names(&inputs)?, + seeds: changed_seed_names(&inputs)?, + sources: changed_sources, + }, + }; + plan_warnings(py, classify_selection_staleness(&graph)) +} + +fn name_set(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// Stale warnings for selected models with changed upstreams outside the run set. +#[pyfunction] +fn classify_selection_staleness_warnings<'py>( + graph: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let staleness = StalenessGraph { + upstream_deps: upstream_graph(&graph.getattr("upstream_deps")?)?, + selected_model_names: name_set(&graph.getattr("selected_model_names")?)? + .into_iter() + .collect(), + run: StalenessNames { + models: name_set(&graph.getattr("run_model_names")?)?, + seeds: name_set(&graph.getattr("run_seed_names")?)?, + sources: name_set(&graph.getattr("run_source_names")?)?, + }, + changed: StalenessNames { + models: name_set(&graph.getattr("changed_model_names")?)?, + seeds: name_set(&graph.getattr("changed_seed_names")?)?, + sources: name_set(&graph.getattr("changed_source_names")?)?, + }, + }; + let warning_class: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, "SelectionStalenessWarning")?; + let mut warnings: Vec> = Vec::new(); + for warning in classify_selection_staleness(&staleness) { + warnings.push(call_with( + &warning_class, + &[ + ( + "model_name", + PyString::new(py, &warning.model_name).into_any(), + ), + ( + "trigger_names", + PyTuple::new(py, warning.trigger_names)?.into_any(), + ), + ], + )?); + } + PyTuple::new(py, warnings) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_stale_out_of_selection_warnings, + module + )?)?; + module.add_function(wrap_pyfunction!( + classify_selection_staleness_warnings, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs new file mode 100644 index 0000000000..40cc222cce --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -0,0 +1,139 @@ +//! Loader dependency mapping and synthetic source boundary construction. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::views::edge_dict; +use crate::bindings::_helpers::planner_scope::values::{ + NATIVE, SOURCE_MODELS, dependency_names, edges, keys, loader_graph, loader_names, project_graph, +}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::planner_scope::main::{ + expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, +}; +use std::collections::HashSet; + +#[pyfunction] +#[pyo3(signature = (*, loader_function, loader_functions))] +pub(crate) fn upstream_loader_dependency_names<'py>( + loader_function: &Bound<'py, PyAny>, + loader_functions: &Bound<'py, PyAny>, +) -> PyResult> { + let graph = loader_graph(loader_functions)?; + let names = loader_names(loader_functions)?; + let deps = dependency_names(loader_function, &names)?; + PyTuple::new(loader_function.py(), graph.upstream(&deps)) +} + +#[pyfunction] +#[pyo3(signature = (*, project, loader))] +pub(crate) fn loader_to_source_entry<'py>( + project: &Bound<'py, PyAny>, + loader: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let name = loader.getattr("name")?; + let mut destination = loader.getattr("destination")?; + if destination.is_none() { + destination = format!("__loader__{}", name.extract::()?) + .into_pyobject(py)? + .into_any(); + } + let parts = call_with( + &imported(py, NATIVE, "loader_destination_parts")?, + &[ + ("destination", destination), + ( + "default_database", + project.getattr("effective_target_database")?, + ), + ( + "default_schema", + project.getattr("effective_target_schema")?, + ), + ], + )?; + let kwargs = PyDict::new(py); + kwargs.set_item("name", &name)?; + kwargs.set_item("loader", &name)?; + for field in ["database", "schema", "table"] { + kwargs.set_item(field, parts.getattr(field)?)?; + } + for field in [ + "write_strategy", + "cursor_column", + "unique_key", + "contract", + "columns", + ] { + kwargs.set_item(field, loader.getattr(field)?)?; + } + let meta = PyDict::new(py); + meta.set_item("sqlbuild_loader_node", true)?; + kwargs.set_item("meta", meta)?; + imported(py, SOURCE_MODELS, "SourceEntry")?.call((), Some(&kwargs)) +} + +pub(crate) fn intermediate_map<'py>( + project: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, + upstream: bool, +) -> PyResult> { + let graph = project_graph(project)?; + let names = intermediate_sources(&graph, &keys(selected)?, upstream); + let loaders = PyDict::new(project.py()); + for loader in project.getattr("loader_functions")?.try_iter()? { + let loader = loader?; + loaders.set_item(loader.getattr("name")?, &loader)?; + } + let result = PyDict::new(project.py()); + for name in names { + if let Some(loader) = loaders.get_item(&name)? { + result.set_item(&name, loader_to_source_entry(project, &loader)?)?; + } + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys))] +pub(crate) fn build_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + intermediate_map(project, selected_keys, false) +} + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys))] +pub(crate) fn build_upstream_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + intermediate_map(project, selected_keys, true) +} + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys, upstream_deps, executable_dependency_source_keys = None))] +pub(crate) fn expand_selected_loader_dependencies<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, + executable_dependency_source_keys: Option<&Bound<'py, PyAny>>, +) -> PyResult<(Bound<'py, PyFrozenSet>, Bound<'py, PyDict>)> { + let graph = project_graph(project)?; + let selected = keys(selected_keys)?; + let upstream = edges(upstream_deps)?; + let executable = executable_dependency_source_keys + .map(keys) + .transpose()? + .unwrap_or_default() + .into_iter() + .collect::>(); + let (selected, upstream) = expand_loaders(&graph, selected, upstream, &executable); + let cache = KeyCache::new(project.py())?; + Ok(( + PyFrozenSet::new(project.py(), cache.keys(&selected)?)?, + edge_dict(&cache, &upstream)?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs new file mode 100644 index 0000000000..e7194181c9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -0,0 +1,7 @@ +//! Planner scope Python boundary models. + +mod loaders; +pub(crate) mod registration; +mod scope; +mod sources; +mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs new file mode 100644 index 0000000000..517f9c5e66 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs @@ -0,0 +1,35 @@ +//! Register planner scope and loader entry points. + +use crate::bindings::_helpers::planner_scope::{loaders, scope, sources}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope::build_planner_scope, module)?)?; + module.add_function(wrap_pyfunction!( + scope::managed_source_upstream_keys, + module + )?)?; + module.add_function(wrap_pyfunction!( + loaders::upstream_loader_dependency_names, + module + )?)?; + module.add_function(wrap_pyfunction!(loaders::loader_to_source_entry, module)?)?; + module.add_function(wrap_pyfunction!( + loaders::build_intermediate_source_map, + module + )?)?; + module.add_function(wrap_pyfunction!( + loaders::build_upstream_intermediate_source_map, + module + )?)?; + module.add_function(wrap_pyfunction!( + loaders::expand_selected_loader_dependencies, + module + )?)?; + module.add_function(wrap_pyfunction!(sources::build_source_load_map, module)?)?; + module.add_function(wrap_pyfunction!( + sources::build_source_load_entries, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs new file mode 100644 index 0000000000..52d74f9dc4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -0,0 +1,160 @@ +//! Planner scope assembly using native graph and selector operations. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_dependencies; +use crate::bindings::_helpers::planner_scope::values::{MODELS, NATIVE, edges, keys}; +use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::PyDictMethods; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::planner_scope::constants::SOURCE; +use sqlbuild_analysis::planner_scope::main::{ + managed_sources::managed_sources, source_selectors::source_selectors, +}; +use std::collections::HashSet; + +#[pyfunction] +#[pyo3(signature = (*, selected_keys, upstream_deps, project))] +pub(crate) fn managed_source_upstream_keys<'py>( + selected_keys: &Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let mut managed: HashSet = HashSet::new(); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if !source.getattr("source_entry")?.getattr("loader")?.is_none() { + managed.insert(source.getattr("name")?.extract()?); + } + } + let selected = managed_sources(&keys(selected_keys)?, &edges(upstream_deps)?, &managed); + PyFrozenSet::new( + project.py(), + KeyCache::new(project.py())?.keys(&selected.into_iter().collect::>())?, + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_planner_scope<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_planner_scope", + options, + ( + &["project"], + &[ + "select", + "exclude", + "auto_load_sources", + "selected_keys", + "python_read_source_names", + "sql_test_selection", + ], + ), + )?; + let project = args.value("project")?; + let select = default_tuple(&args.value("select")?); + let exclude = default_tuple(&args.value("exclude")?); + let auto_load = args.value("auto_load_sources")?.is_truthy()?; + let mut upstream = imported(py, NATIVE, "build_execution_upstream_deps")?.call1((&project,))?; + let mut downstream = imported(py, NATIVE, "build_downstream_deps")?.call1((&upstream,))?; + let lineage = project.getattr("lineage_graph")?; + let all_keys = imported(py, NATIVE, "lineage_graph_names")?.call1((&lineage,))?; + let mut selected = args.value("selected_keys")?; + if selected.is_none() { + selected = call_with( + &imported(py, NATIVE, "resolve_graph_selection")?, + &[ + ("graph", lineage), + ("select", select.clone()), + ("exclude", exclude), + ], + )?; + } + let names: Vec<(String, (String, String))> = all_keys + .call_method0("items")? + .try_iter()? + .map(|item| { + let (name, key): (String, Bound<'py, PyAny>) = item?.extract()?; + Ok(( + name, + crate::bindings::_helpers::graph::keys::native_key(&key)?, + )) + }) + .collect::>()?; + let texts = py_strings(&select)?; + let carrier = Carrier::new(py, &texts, || { + Ok(names.iter().map(|(name, _)| name.clone()).collect()) + })?; + let source_keys = source_selectors(&carrier.send_all(&texts)?, &names, &keys(&selected)?) + .map_err(|error| carrier.error(error))?; + let cache = KeyCache::new(py)?; + let mut executable = PyFrozenSet::new( + py, + cache.keys(&source_keys.into_iter().collect::>())?, + )? + .into_any(); + if auto_load { + let loaded = managed_source_upstream_keys(&selected, &upstream, &project)?; + selected = selected.call_method1("union", (&loaded,))?; + executable = executable.call_method1("union", (&loaded,))?; + } + executable = executable.call_method1("intersection", (&selected,))?; + if auto_load || keys(&selected)?.iter().any(|key| key.0 == SOURCE) { + let (expanded, deps) = + expand_selected_loader_dependencies(&project, &selected, &upstream, Some(&executable))?; + selected = expanded.into_any(); + upstream = deps.into_any(); + downstream = imported(py, NATIVE, "build_downstream_deps")?.call1((&upstream,))?; + } + let origins = imported(py, NATIVE, "build_execution_edge_origins")?.call1((&project,))?; + let order = call_with( + &imported(py, NATIVE, "topologically_order_keys")?, + &[ + ("upstream", upstream.clone()), + ("injected_edge_origins", origins), + ], + )?; + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + let mut reads = args.value("python_read_source_names")?; + if reads.is_none() { + reads = PyFrozenSet::empty(py)?.into_any(); + } + let mut tests = args.value("sql_test_selection")?; + if !tests.is_truthy()? { + tests = imported(py, MODELS, "SqlTestSelection")?.call0()?; + } + call_with( + &imported(py, MODELS, "PlannerScope")?, + &[ + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ("all_keys", all_keys), + ("models_by_name", models.into_any()), + ("selected_keys", selected.clone()), + ("execution_order", order), + ("user_selected_keys", selected), + ("python_read_source_names", reads), + ("sql_test_selection", tests), + ], + ) +} + +fn default_tuple<'py>(value: &Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + PyTuple::empty(value.py()).into_any() + } else { + value.clone() + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs new file mode 100644 index 0000000000..ba79f23caa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs @@ -0,0 +1,115 @@ +//! Existing source and load plan models materialised at the Python boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; +use crate::bindings::_helpers::planner_scope::values::MODELS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::types::{PyDictMethods, PySetMethods}; +use sqlbuild_analysis::planner_scope::constants::SOURCE; + +const EXECUTION_TYPES: &str = "sqlbuild.runtime.contracts.types"; + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys, upstream_deps = None))] +pub(crate) fn build_source_load_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + upstream_deps: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let relevant = upstream_deps + .map(|edges| { + closure( + selected_keys.try_iter()?.collect::>>()?, + edges, + (true, None), + ) + }) + .transpose()?; + let result = PyDict::new(project.py()); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if let Some(keys) = &relevant + && !keys.contains(source.getattr("key")?)? + { + continue; + } + let entry = source.getattr("source_entry")?; + result.set_item(entry.getattr("name")?, entry)?; + } + for upstream in [false, true] { + for (key, entry) in intermediate_map(project, selected_keys, upstream)?.iter() { + result.set_item(key, entry)?; + } + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, execution_order, selected_keys, source_map, is_reload))] +pub(crate) fn build_source_load_entries<'py>( + execution_order: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + is_reload: bool, +) -> PyResult> { + let py = execution_order.py(); + let class = imported(py, MODELS, "SourceLoadPlanEntry")?; + let kinds = imported(py, EXECUTION_TYPES, "ExecutionResourceKind")?; + let mut entries: Vec> = Vec::new(); + for key in execution_order.try_iter()? { + let key = key?; + if !selected_keys.contains(&key)? + || key.getattr("resource_type")?.str()?.extract::()? != SOURCE + { + continue; + } + let Some(entry) = source_map.get_item(key.getattr("name")?)? else { + continue; + }; + if entry.getattr("loader")?.is_none() { + continue; + } + let kwargs = PyDict::new(py); + kwargs.set_item("key", key)?; + for field in [ + "name", + "loader", + "write_strategy", + "cursor_column", + "unique_key", + ] { + kwargs.set_item(field, entry.getattr(field)?)?; + } + let table = entry.getattr("table")?; + kwargs.set_item( + "destination", + if table.is_truthy()? { + table + } else { + entry.getattr("name")? + }, + )?; + let intermediate = entry + .getattr("meta")? + .call_method1("get", ("sqlbuild_loader_node",))? + .is(PyBool::new(py, true)); + kwargs.set_item( + "resource_kind", + kinds.getattr(if intermediate { "LOADER" } else { "SOURCE" })?, + )?; + kwargs.set_item("is_reload", is_reload)?; + let integration = entry.getattr("integration_loader")?; + kwargs.set_item( + "integration_kind", + if integration.is_none() { + py.None().into_bound(py) + } else { + integration.getattr("kind")? + }, + )?; + entries.push(class.call((), Some(&kwargs))?); + } + PyTuple::new(py, entries) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs new file mode 100644 index 0000000000..a9fd5203d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -0,0 +1,78 @@ +//! Python function identity and compiled graph conversion. + +use crate::bindings::_helpers::graph::keys::native_key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_scope::models::LoaderGraph; + +pub(crate) const NATIVE: &str = "sqlbuild._native"; +pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const SOURCE_MODELS: &str = "sqlbuild.spec.contracts.models"; + +pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { + values + .try_iter()? + .map(|value| native_key(&value?)) + .collect() +} + +pub(crate) fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { + values + .call_method0("items")? + .try_iter()? + .map(|item| { + let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + Ok((native_key(&key)?, keys(&deps)?)) + }) + .collect() +} + +/// Resolve dependencies using the same Python dictionary lookup as the authored function boundary. +pub(crate) fn loader_names<'py>(loaders: &Bound<'py, PyAny>) -> PyResult> { + let names = PyDict::new(loaders.py()); + for loader in loaders.try_iter()? { + let loader = loader?; + names.set_item(loader.getattr("function")?, loader.getattr("name")?)?; + } + Ok(names) +} + +pub(crate) fn dependency_names( + loader: &Bound<'_, PyAny>, + names: &Bound<'_, PyDict>, +) -> PyResult> { + let mut dependencies: Vec = Vec::new(); + for dependency in loader.getattr("depends_on")?.try_iter()? { + if let Some(name) = names.get_item(dependency?)? { + dependencies.push(name.extract()?); + } + } + Ok(dependencies) +} + +pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult { + let names = loader_names(loaders)?; + let mut graph = LoaderGraph::default(); + for loader in loaders.try_iter()? { + let loader = loader?; + graph.loaders.insert( + loader.getattr("name")?.extract()?, + dependency_names(&loader, &names)?, + ); + } + Ok(graph) +} + +pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { + let mut graph = loader_graph(&project.getattr("loader_functions")?)?; + for source in project.getattr("sources")?.try_iter()? { + let entry = source?.getattr("source_entry")?; + graph.sources.push(( + entry.getattr("name")?.extract()?, + entry.getattr("loader")?.extract()?, + )); + } + Ok(graph) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs new file mode 100644 index 0000000000..0bf31dc9c1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs @@ -0,0 +1,143 @@ +//! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. + +use std::collections::{BTreeSet, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use sqlbuild_analysis::graph::errors::SelectorError; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; + +/// `str.translate` tables to and from carrier characters; empty when no text holds a surrogate. +pub(crate) struct Carrier<'py> { + py: Python<'py>, + tables: Option<(Bound<'py, PyDict>, Bound<'py, PyDict>)>, +} + +impl<'py> Carrier<'py> { + /// A carrier for `texts`, choosing carriers that appear in neither `texts` nor `reserved`. + pub(crate) fn new( + py: Python<'py>, + texts: &[Bound<'py, PyString>], + reserved: impl FnOnce() -> PyResult>, + ) -> PyResult { + let mut surrogates: BTreeSet = BTreeSet::new(); + let mut taken: HashSet = HashSet::new(); + for text in texts { + if let Ok(plain) = text.to_str() { + taken.extend(plain.chars()); + continue; + } + for character in text.try_iter()? { + let character: Bound<'py, PyAny> = character?; + match character.extract::() { + Ok(plain) => { + taken.insert(plain); + } + Err(_) => { + surrogates.insert(ordinal(&character)?); + } + } + } + } + if surrogates.is_empty() { + return Ok(Self { py, tables: None }); + } + for name in reserved()? { + taken.extend(name.chars()); + } + let send: Bound<'py, PyDict> = PyDict::new(py); + let back: Bound<'py, PyDict> = PyDict::new(py); + let carriers = PRIVATE_USE + .filter_map(char::from_u32) + .filter(|candidate| !taken.contains(candidate)); + for (surrogate, carrier) in surrogates.into_iter().zip(carriers) { + send.set_item(surrogate, carrier)?; + back.set_item(u32::from(carrier), surrogate_text(py, surrogate)?)?; + } + Ok(Self { + py, + tables: Some((send, back)), + }) + } + + /// A carrier for one text with nothing reserved. + pub(crate) fn single(text: &Bound<'py, PyString>) -> PyResult { + Self::new(text.py(), std::slice::from_ref(text), || Ok(Vec::new())) + } + + /// The text with surrogates replaced by their carriers. + pub(crate) fn send(&self, text: &Bound<'py, PyString>) -> PyResult { + match &self.tables { + None => Ok(text.to_str()?.to_owned()), + Some((send, _)) => text.call_method1("translate", (send,))?.extract(), + } + } + + /// The texts with surrogates replaced by their carriers. + pub(crate) fn send_all(&self, texts: &[Bound<'py, PyString>]) -> PyResult> { + texts.iter().map(|text| self.send(text)).collect() + } + + /// The text with carriers restored to their surrogates. + pub(crate) fn back(&self, text: &str) -> PyResult> { + let restored: Bound<'py, PyString> = PyString::new(self.py, text); + match &self.tables { + None => Ok(restored), + Some((_, back)) => Ok(restored + .call_method1("translate", (back,))? + .downcast_into()?), + } + } + + pub(crate) fn py(&self) -> Python<'py> { + self.py + } + + /// The `PlannerInputError` instance for a selector failure, with carriers restored. + pub(crate) fn exception(&self, error: SelectorError) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(self.py); + keywords.set_item("code", error.code)?; + let help: Option> = error + .help + .as_deref() + .map(|help| self.back(help)) + .transpose()?; + keywords.set_item("help", help)?; + imported(self.py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")? + .call((self.back(&error.message)?,), Some(&keywords)) + } + + /// The `PlannerInputError` to raise for a selector failure, with carriers restored. + pub(crate) fn error(&self, error: SelectorError) -> PyErr { + match self.exception(error) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } + } +} + +/// Python strings for each item of an iterable of `str`. +pub(crate) fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { + values + .try_iter()? + .map(|value| Ok(value?.downcast_into::()?)) + .collect() +} + +fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { + let code: u32 = character + .py() + .import("builtins")? + .getattr("ord")? + .call1((character,))? + .extract()?; + Ok(code) +} + +fn surrogate_text(py: Python<'_>, code: u32) -> PyResult> { + py.import("builtins")?.getattr("chr")?.call1((code,)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs new file mode 100644 index 0000000000..0097ebb362 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod carrier; +pub(crate) mod model_selectors; +pub(crate) mod registration; +pub(crate) mod selectors; +pub(crate) mod unit_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs new file mode 100644 index 0000000000..def3598381 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs @@ -0,0 +1,121 @@ +//! `+` marker splitting and SQLBuild-owned model selectors for dbt interop. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyBool; +use pyo3::types::{PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_selection::main::split_selector_expansion::split_selector_expansion as split; +use sqlbuild_analysis::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; +use sqlbuild_analysis::planner_selection::models::{ + ModelSelectorNames, SelectableModel, SelectorExpansion, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_selection::carrier::Carrier; + +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; + +fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { + let raised = imported(carrier.py(), SHARED_EXCEPTIONS_MODULE, "SharedInputError") + .and_then(|class| class.call1((carrier.back(message)?,))); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// Split `+selector+` marker syntax from the selector core without resolving the core. +#[pyfunction] +fn split_selector_expansion<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + let py: Python<'py> = raw.py(); + let carrier: Carrier<'py> = Carrier::single(raw)?; + let expansion: SelectorExpansion = + split(&carrier.send(raw)?).map_err(|message| shared_input_error(&carrier, &message))?; + call_with( + &imported(py, PLANNER_MODELS_MODULE, "SelectorExpansion")?, + &[ + ("core", carrier.back(&expansion.core)?.into_any()), + ( + "upstream", + PyBool::new(py, expansion.upstream).to_owned().into_any(), + ), + ( + "downstream", + expansion + .downstream + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + ) +} + +fn model_tags(model: &Bound<'_, PyAny>) -> PyResult> { + let tags: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("tags",))?; + if !tags.is_instance_of::() && !tags.is_instance_of::() { + return Ok(Vec::new()); + } + tags.try_iter()?.map(|tag| tag?.str()?.extract()).collect() +} + +fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { + Ok(SelectableModel { + name: model.getattr("name")?.extract()?, + tags: model_tags(model)?, + folder: model + .getattr("relative_path")? + .getattr("parent")? + .call_method0("as_posix")? + .extract()?, + }) +} + +/// One SQLBuild-owned model selector core's model names, and its path selector when translated. +#[pyfunction] +#[pyo3(signature = (*, project, term))] +fn resolve_sqlbuild_model_selector_names<'py>( + project: &Bound<'py, PyAny>, + term: &Bound<'py, PyString>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let models: Vec = project + .getattr("models")? + .try_iter()? + .map(|model| selectable_model(&model?)) + .collect::>()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { + Ok(models + .iter() + .flat_map(|model| [model.name.clone(), model.folder.clone()]) + .chain(models.iter().flat_map(|model| model.tags.clone())) + .collect()) + })?; + let resolved: ModelSelectorNames = sqlbuild_model_selector_names(&models, &carrier.send(term)?); + let translated: Option> = resolved + .translated + .as_deref() + .map(|path| carrier.back(path)) + .transpose()?; + PyTuple::new( + py, + [ + PyTuple::new(py, resolved.names)?.into_any(), + translated.into_pyobject(py)?.into_any(), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(split_selector_expansion, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_sqlbuild_model_selector_names, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs new file mode 100644 index 0000000000..74289715bc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs @@ -0,0 +1,11 @@ +//! Register planner selection bindings. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_selection::{model_selectors, selectors, unit_tests}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_selectors::register(module)?; + selectors::register(module)?; + unit_tests::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs new file mode 100644 index 0000000000..2694a72991 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -0,0 +1,264 @@ +//! Planner selector parsing and resolution over native project graphs, raising `PlannerInputError`. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyFrozenSet, PySetMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; +use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; +use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; + +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const VIEW_OPTIONS: &[&str] = &[ + "all_keys", + "upstream_deps", + "downstream_deps", + "tag_index", + "path_index", +]; + +/// The parsed selector as Python's `ParsedSelector` or `PathSelector`. +pub(crate) fn parsed_selector<'py>( + carrier: &Carrier<'py>, + parsed: ParsedSelector, +) -> PyResult> { + let py: Python<'py> = carrier.py(); + match parsed { + ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } => call_with( + &imported(py, PLANNER_MODELS_MODULE, "ParsedSelector")?, + &[ + ( + "kind", + imported(py, PLANNER_TYPES_MODULE, "SelectorKind")?.call1((kind,))?, + ), + ("value", carrier.back(&value)?.into_any()), + ("upstream", PyBool::new(py, upstream).to_owned().into_any()), + ( + "downstream", + PyBool::new(py, downstream).to_owned().into_any(), + ), + ], + ), + ParsedSelector::Path { + start, + end, + upstream, + downstream, + } => call_with( + &imported(py, PLANNER_MODELS_MODULE, "PathSelector")?, + &[ + ("start_name", carrier.back(&start)?.into_any()), + ("end_name", carrier.back(&end)?.into_any()), + ("upstream", PyBool::new(py, upstream).to_owned().into_any()), + ( + "downstream", + PyBool::new(py, downstream).to_owned().into_any(), + ), + ], + ), + } +} + +fn graph_names(graph: &ProjectGraph) -> Vec { + graph.names.iter().map(|(name, _)| name.clone()).collect() +} + +fn selected<'py>( + carrier: &Carrier<'py>, + outcome: Result, SelectorError>, +) -> PyResult> { + let keys: Vec = outcome.map_err(|error| carrier.error(error))?; + compiled_keys(carrier.py(), &keys) +} + +fn compiled_keys<'py>(py: Python<'py>, keys: &[ObjectKey]) -> PyResult> { + PyFrozenSet::new(py, KeyCache::new(py)?.keys(keys)?) +} + +fn resolved<'py>( + graph: &ProjectGraph, + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = select.py(); + let select: Vec> = py_strings(select)?; + let exclude: Vec> = py_strings(exclude)?; + let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; + selected( + &carrier, + resolve_selectors( + graph, + &carrier.send_all(&select)?, + &carrier.send_all(&exclude)?, + ), + ) +} + +fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { + graph_from_view_mappings(mappings) +} + +/// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. +#[pyfunction] +fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + let carrier: Carrier<'py> = Carrier::single(raw)?; + let parsed: ParsedSelector = + parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; + parsed_selector(&carrier, parsed) +} + +/// `--select` minus `--exclude` on a native graph, adding the functions selected models need. +#[pyfunction] +#[pyo3(signature = (*, graph, select, exclude))] +fn resolve_graph_selection<'py>( + graph: PyRef<'py, NativeProjectGraph>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + resolved(graph.graph(), (select, exclude)) +} + +/// `--select` minus `--exclude` on a project graph, adding the functions selected models need. +#[pyfunction] +#[pyo3(signature = (*, graph, select, exclude))] +fn select_project_graph<'py>( + graph: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let native: PyRef<'py, NativeProjectGraph> = graph.getattr("native")?.extract()?; + resolved(native.graph(), (select, exclude)) +} + +/// Model, source, seed and function selectors resolved against compiled graph mappings. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_project_selectors<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selectors", + options, + (&[&["select", "exclude"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + resolved( + graph.graph(), + (&arguments.value("select")?, &arguments.value("exclude")?), + ) +} + +/// The keys whitespace-separated selector tokens match, without required build resources. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_project_selector_tokens<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selector_tokens", + options, + (&[&["selectors"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + let texts: Vec> = py_strings(&arguments.value("selectors")?)?; + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; + selected( + &carrier, + resolve_selector_tokens(graph.graph(), &carrier.send_all(&texts)?), + ) +} + +fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { + view_graph(GraphViewMappings { + all_keys: &arguments.value("all_keys")?, + upstream: &arguments.value("upstream_deps")?, + downstream: &arguments.value("downstream_deps")?, + tag_index: &arguments.value("tag_index")?, + path_index: &arguments.value("path_index")?, + }) +} + +/// The keys one parsed selector matches before `+` graph expansion. +#[pyfunction] +#[pyo3(signature = (*, parsed, all_keys, tag_index, path_index))] +fn match_project_selector<'py>( + parsed: &Bound<'py, PyAny>, + all_keys: &Bound<'py, PyAny>, + tag_index: &Bound<'py, PyAny>, + path_index: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = parsed.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let graph: NativeProjectGraph = view_graph(GraphViewMappings { + all_keys, + upstream: &empty, + downstream: &empty, + tag_index, + path_index, + })?; + let value: Bound<'py, PyString> = parsed.getattr("value")?.downcast_into()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { + Ok(graph_names(graph.graph())) + })?; + let kind: String = parsed.getattr("kind")?.str()?.extract()?; + selected( + &carrier, + match_selector(graph.graph(), &kind, &carrier.send(&value)?) + .map(|keys| keys.into_iter().collect()), + ) +} + +/// The upstream key closure of one compiled graph key. +#[pyfunction] +#[pyo3(signature = (*, key, upstream_deps))] +fn expand_project_upstream_keys<'py>( + key: Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let visited = closure(vec![key], upstream_deps, (false, None))?; + PyFrozenSet::new(upstream_deps.py(), visited.iter()) +} + +/// The error for a unit-test selector given where unit tests never run. +#[pyfunction] +#[pyo3(signature = (*, selector))] +fn unit_test_selector_error<'py>(selector: &Bound<'py, PyString>) -> PyResult> { + let carrier: Carrier<'py> = Carrier::single(selector)?; + carrier.exception(unit_test_selector_rejection(&carrier.send(selector)?)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_project_selector, module)?)?; + module.add_function(wrap_pyfunction!(resolve_graph_selection, module)?)?; + module.add_function(wrap_pyfunction!(select_project_graph, module)?)?; + module.add_function(wrap_pyfunction!(resolve_project_selectors, module)?)?; + module.add_function(wrap_pyfunction!(resolve_project_selector_tokens, module)?)?; + module.add_function(wrap_pyfunction!(match_project_selector, module)?)?; + module.add_function(wrap_pyfunction!(expand_project_upstream_keys, module)?)?; + module.add_function(wrap_pyfunction!(unit_test_selector_error, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs new file mode 100644 index 0000000000..c60a8eacd6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -0,0 +1,178 @@ +//! Unit-test selector splitting over discovered project names. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors as split; +use sqlbuild_analysis::planner_selection::models::{ + SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; + +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const RESOURCE_ENTRY_FILES: [(&str, &str); 3] = [ + ("schema_files", "model_entries"), + ("schema_files", "seed_entries"), + ("source_files", "source_entries"), +]; +const RESOURCE_PATH_FILES: [&str; 4] = [ + "model_files", + "seed_files", + "sql_function_files", + "python_function_files", +]; +const RESOURCE_NODES: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; + +fn stem(path: &Bound<'_, PyAny>) -> PyResult { + path.getattr("stem")?.extract() +} + +fn unit_test_names(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for test_file in discovered.getattr("test_files")?.try_iter()? { + let test_file: Bound<'_, PyAny> = test_file?; + for block in test_file.getattr("blocks")?.try_iter()? { + let name: Bound<'_, PyAny> = block?.getattr("name")?; + names.insert(if name.is_truthy()? { + name.extract()? + } else { + stem(&test_file.getattr("relative_path")?)? + }); + } + } + Ok(names) +} + +fn resource_names(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for field in RESOURCE_PATH_FILES { + for file in discovered.getattr(field)?.try_iter()? { + names.insert(stem(&file?.getattr("file_path")?)?); + } + } + for (field, entries) in RESOURCE_ENTRY_FILES { + for file in discovered.getattr(field)?.try_iter()? { + for entry in file?.getattr(entries)?.try_iter()? { + names.insert(entry?.getattr("name")?.extract()?); + } + } + } + for field in RESOURCE_NODES { + for node in discovered.getattr(field)?.try_iter()? { + names.insert(node?.getattr("name")?.extract()?); + } + } + Ok(names) +} + +fn restored<'py>(carrier: &Carrier<'py>, texts: &[String]) -> PyResult> { + let texts: Vec> = texts + .iter() + .map(|text| carrier.back(text)) + .collect::>()?; + PyTuple::new(carrier.py(), texts) +} + +fn split_python<'py>( + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + discovered: &Bound<'py, PyAny>, + accepts_unit_tests: bool, +) -> PyResult> { + let py: Python<'py> = select.py(); + let names = UnitTestSelectorNames { + tests: unit_test_names(discovered)?, + resources: resource_names(discovered)?, + }; + let select: Vec> = py_strings(select)?; + let exclude: Vec> = py_strings(exclude)?; + let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); + let carrier: Carrier<'py> = Carrier::new(py, &texts, || { + Ok(names.tests.union(&names.resources).cloned().collect()) + })?; + let (select, exclude): (Vec, Vec) = + (carrier.send_all(&select)?, carrier.send_all(&exclude)?); + let split: UnitTestSelectorSplit = split( + SelectorGroups { + select: &select, + exclude: &exclude, + }, + &names, + accepts_unit_tests, + ) + .map_err(|error| carrier.error(error))?; + let selection: Bound<'py, PyAny> = call_with( + &imported(py, PLANNER_MODELS_MODULE, "SqlTestSelection")?, + &[ + ("names", PyFrozenSet::new(py, &split.test_names)?.into_any()), + ( + "excluded_names", + PyFrozenSet::new(py, &split.excluded_test_names)?.into_any(), + ), + ( + "models_selected", + split + .models_selected + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + )?; + call_with( + &imported(py, PLANNER_MODELS_MODULE, "UnitTestSelectorSplit")?, + &[ + ("select", restored(&carrier, &split.select)?.into_any()), + ("exclude", restored(&carrier, &split.exclude)?.into_any()), + ("sql_test_selection", selection), + ], + ) +} + +/// Move unit-test selectors out of resource selectors, or reject them for this command. +#[pyfunction] +#[pyo3(signature = (*, select, exclude, discovered_inputs, accepts_unit_tests))] +fn split_project_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + accepts_unit_tests: bool, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, accepts_unit_tests) +} + +/// Split unit-test selectors from resource selectors for a command that runs unit tests. +#[pyfunction] +#[pyo3(signature = (*, select, exclude, discovered_inputs))] +fn split_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, true) +} + +/// Raise when a selector names a unit test; return the unchanged resource selectors. +#[pyfunction] +#[pyo3(signature = (*, select, exclude, discovered_inputs))] +fn reject_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, false) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(split_project_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(split_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(reject_unit_test_selectors, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs index 519415c3d5..42554e58ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod document; pub(crate) mod functions; pub(crate) mod registration; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs index 999bafd567..fd3b43e9ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::presentation_text::{document, functions}; +use crate::bindings::_helpers::presentation_text::{document, functions, terminal}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { document::register(module)?; - functions::register(module) + functions::register(module)?; + terminal::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs new file mode 100644 index 0000000000..369db30e07 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs @@ -0,0 +1,28 @@ +//! Terminal colour capability of the active output stream. + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const NO_COLOR_VARIABLE: &str = "NO_COLOR"; + +/// Whether `sys.stdout` is a terminal and `NO_COLOR` is unset. +pub(crate) fn stdout_supports_color(py: Python<'_>) -> PyResult { + if std::env::var_os(NO_COLOR_VARIABLE).is_some() { + return Ok(false); + } + let stdout = py.import("sys")?.getattr("stdout")?; + if !stdout.hasattr("isatty")? { + return Ok(false); + } + stdout.call_method0("isatty")?.is_truthy() +} + +/// Return whether the active output terminal supports color. +#[pyfunction(name = "supports_color")] +fn color_support(py: Python<'_>) -> PyResult { + stdout_supports_color(py) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(color_support, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 01c92d2a40..2368f1e8c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -2,11 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; /// The project directory, defaulting to the working directory, with its inputs and adapter. pub(crate) struct ProjectAdapter<'py> { @@ -34,13 +34,7 @@ pub(crate) fn project_adapter<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; Ok(ProjectAdapter { project_dir, discovered, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 09dc56013d..9a3490c70d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -35,13 +35,13 @@ use sqlbuild_refactor::refactoring::types::Originals; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, }; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const COMPILE_MODULE: &str = "sqlbuild.cli.commands._helpers.refactor.compile_facts"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; @@ -417,10 +417,8 @@ fn run_refactor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; let json_output: bool = request.getattr("json_output")?.extract()?; - let use_color: bool = !request.getattr("no_color")?.extract::()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let command: Command<'_> = Command { request: request.clone(), project_dir: project_dir.call_method0("resolve")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index edbec57e0a..abeb1410a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -115,9 +115,7 @@ fn rule<'py>( } /// Every built-in rule as a `Rule` whose `check` is the given placeholder. -#[pyfunction] -#[pyo3(signature = (*, check))] -fn builtin_rules<'py>( +pub(crate) fn builtin_rules<'py>( py: Python<'py>, check: Bound<'py, PyAny>, ) -> PyResult>> { @@ -137,7 +135,6 @@ fn selected_sql_rule_codes(select: Vec, ignore: Vec) -> PyResult } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(builtin_rules, module)?)?; module.add_function(wrap_pyfunction!(selected_sql_rule_codes, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs new file mode 100644 index 0000000000..7b74700c6e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -0,0 +1,413 @@ +//! The validated rule catalogue, the active rules it selects and resolved ignore selectors. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::models::RulesConfig; +use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSpec, OptionValue}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; +use crate::bindings::_helpers::rules::selection::{rules_error, selected_rule_codes}; + +const CUSTOM_CATALOGUE_MODULE: &str = "sqlbuild.rule_engine._helpers.engine._custom_catalogue"; +const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; +const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; +const PLANNER_SCOPE_MODULE: &str = "sqlbuild._native"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const JSON_MODULE: &str = "json"; +const BUILTINS_MODULE: &str = "builtins"; +const RULES_DIRECTORY: &str = "rules"; +const CUSTOM_RULE_PREFIX: &str = "XSQBR"; +const BUILTIN_CHECK_MESSAGE: &str = "built-in rules execute only through the native engine"; + +/// The Python rules configuration as the native configuration model. +pub(crate) fn native_rules_config(config: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = config.py(); + let fields: Bound<'_, PyAny> = imported(py, DATACLASSES_MODULE, "asdict")?.call1((config,))?; + let encoded: String = imported(py, JSON_MODULE, "dumps")? + .call( + (fields,), + Some(&keywords( + py, + &[("default", imported(py, BUILTINS_MODULE, "str")?)], + )?), + )? + .extract()?; + serde_json::from_str(&encoded) + .map_err(|error| rules_error(py, &format!("invalid rules configuration: {error}"), None)) +} + +/// Whether any selector, option, exception or ignore of `config` names a custom rule. +pub(crate) fn references_custom_rules(config: &Bound<'_, PyAny>) -> PyResult { + let mut selectors: Vec = Vec::new(); + for field in ["select", "ignore", "rule_options"] { + for selector in config.getattr(field)?.try_iter()? { + selectors.push(selector?.extract()?); + } + } + for entry in config.getattr("rule_exceptions")?.try_iter()? { + selectors.push(entry?.getattr("rule")?.extract()?); + } + for entry in config.getattr("rule_ignores")?.try_iter()? { + for rule in entry?.getattr("rules")?.try_iter()? { + selectors.push(rule?.extract()?); + } + } + Ok(selectors + .iter() + .any(|selector| selector.starts_with(CUSTOM_RULE_PREFIX))) +} + +fn option_kind(value_type: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value_type.py(); + let kinds: [(Bound<'_, PyAny>, OptionKind); 4] = [ + (py.get_type::().into_any(), OptionKind::Bool), + (py.get_type::().into_any(), OptionKind::Int), + (py.get_type::().into_any(), OptionKind::Str), + (py.get_type::().into_any(), OptionKind::Tuple), + ]; + for (class, kind) in kinds { + if value_type.is(&class) { + return Ok(kind); + } + } + let name: String = match value_type.getattr("__name__") { + Ok(name) => name.extract()?, + Err(_) => value_type.repr()?.extract()?, + }; + Ok(OptionKind::Other(name)) +} + +fn option_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(OptionValue::Bool(value.extract()?)); + } + if value.is_instance_of::() { + return Ok(value + .extract::() + .map_or_else(|_| OptionValue::Other("int".into()), OptionValue::Int)); + } + if value.is_instance_of::() { + return Ok(OptionValue::Str(value.extract()?)); + } + if let Ok(items) = value.cast::() { + return Ok(OptionValue::Tuple( + items + .iter() + .map(|item| option_value(&item)) + .collect::>()?, + )); + } + Ok(OptionValue::Other( + value.get_type().getattr("__name__")?.extract()?, + )) +} + +fn optional_integer(option: &Bound<'_, PyAny>, field: &str) -> PyResult> { + option.getattr(field)?.extract() +} + +fn option_spec(option: &Bound<'_, PyAny>) -> PyResult { + let item_type: Bound<'_, PyAny> = option.getattr("item_type")?; + Ok(OptionSpec { + name: option.getattr("name")?.extract()?, + description: option.getattr("description")?.extract()?, + value_type: option_kind(&option.getattr("value_type")?)?, + default: option_value(&option.getattr("default")?)?, + choices: option + .getattr("choices")? + .try_iter()? + .map(|choice| option_value(&choice?)) + .collect::>()?, + minimum: optional_integer(option, "minimum")?, + maximum: optional_integer(option, "maximum")?, + minimum_items: optional_integer(option, "minimum_items")?, + item_type: if item_type.is_none() { + None + } else { + Some(option_kind(&item_type)?) + }, + }) +} + +fn catalogue_rule(rule: &Bound<'_, PyAny>) -> PyResult { + Ok(CatalogueRule { + code: rule.getattr("code")?.extract()?, + options: rule + .getattr("options")? + .try_iter()? + .map(|option| option_spec(&option?)) + .collect::>()?, + }) +} + +fn configured_options( + config: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut configured: BTreeMap> = BTreeMap::new(); + let options: Bound<'_, PyDict> = config.getattr("rule_options")?.cast_into::()?; + for (code, values) in options.iter() { + let values: Bound<'_, PyDict> = values.cast_into::()?; + let mut entries: BTreeMap = BTreeMap::new(); + for (name, value) in values.iter() { + entries.insert(name.extract()?, option_value(&value)?); + } + configured.insert(code.extract()?, entries); + } + Ok(configured) +} + +fn custom_rules<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult>> { + let py: Python<'py> = project_dir.py(); + let root: PathBuf = project_dir.extract()?; + let directory: PathBuf = root.join(RULES_DIRECTORY); + if !directory.is_dir() { + return Ok(Vec::new()); + } + let escapes = |directory: &Path| -> bool { + match (directory.canonicalize(), root.canonicalize()) { + (Ok(resolved), Ok(root)) => !resolved.starts_with(root), + _ => true, + } + }; + if escapes(&directory) { + return Err(rules_error( + py, + &format!("rule path escapes the project: {RULES_DIRECTORY}"), + None, + )); + } + call_with( + &imported(py, CUSTOM_CATALOGUE_MODULE, "load_custom_rules")?, + &[("project_dir", project_dir.clone())], + )? + .try_iter()? + .collect() +} + +/// The validated built-in and, when requested, custom catalogue sorted by rule code. +pub(crate) fn rule_catalogue<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + include_custom: bool, +) -> PyResult>> { + let py: Python<'py> = config.py(); + let check: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + |arguments: &Bound<'_, PyTuple>, _| -> PyResult<()> { + Err(rules_error(arguments.py(), BUILTIN_CHECK_MESSAGE, None)) + }, + )?; + let mut rules: Vec> = builtin_rules(py, check.into_any())?; + if include_custom { + rules.extend(custom_rules(project_dir)?); + } + let views: Vec = rules.iter().map(catalogue_rule).collect::>()?; + let configured = configured_options(config)?; + let order: Vec = compiler_guard(|| Ok(sorted_catalogue_order(&views, &configured)))? + .map_err(|message| rules_error(py, &message, None))?; + Ok(order + .into_iter() + .map(|index| rules[index].clone()) + .collect()) +} + +/// The catalogue rules the configuration activates, in native selection order. +pub(crate) fn select_rules<'py>( + catalogue: &[Bound<'py, PyAny>], + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + custom_payloads: Option>, +) -> PyResult>> { + let py: Python<'py> = config.py(); + let payloads: Bound<'py, PyAny> = match custom_payloads { + Some(payloads) if !payloads.is_none() => payloads, + _ => call_with( + &imported(py, CUSTOM_PAYLOADS_MODULE, "custom_rule_payloads")?, + &[ + ("catalogue", PyTuple::new(py, catalogue)?.into_any()), + ("project_dir", project_dir.clone()), + ], + )?, + }; + let mut by_code: BTreeMap> = BTreeMap::new(); + for rule in catalogue { + by_code.insert(rule.getattr("code")?.extract()?, rule.clone()); + } + selected_rule_codes(config, &payloads)? + .into_iter() + .map(|code| { + by_code.get(&code).cloned().ok_or_else(|| { + rules_error( + py, + &format!("native compiler rules returned unknown rule {code}"), + None, + ) + }) + }) + .collect() +} + +/// Reject non-hermetic imports in the source closure of every selected custom rule. +pub(crate) fn verify_custom_rules<'py>( + rules: &[Bound<'py, PyAny>], + project_dir: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = project_dir.py(); + let _ = call_with( + &imported(py, HERMETICITY_MODULE, "verify_custom_rules")?, + &[ + ("rules", PyTuple::new(py, rules)?.into_any()), + ("project_dir", project_dir.clone()), + ], + )?; + Ok(()) +} + +/// The validated built-in and custom rule catalogue sorted by code. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, include_custom = true))] +fn build_rule_catalogue<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + include_custom: bool, +) -> PyResult> { + PyTuple::new( + config.py(), + rule_catalogue(config, project_dir, include_custom)?, + ) +} + +/// The catalogue rules the configuration activates; `custom_payloads` describes custom rules. +#[pyfunction] +#[pyo3(signature = (*, catalogue, config, project_dir, custom_payloads = None))] +fn select_catalogue_rules<'py>( + catalogue: &Bound<'py, PyAny>, + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + custom_payloads: Option>, +) -> PyResult> { + let rules: Vec> = catalogue.try_iter()?.collect::>()?; + PyTuple::new( + config.py(), + select_rules(&rules, config, project_dir, custom_payloads)?, + ) +} + +fn resource_paths<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let paths: Bound<'py, PyDict> = PyDict::new(project.py()); + for (collection, file) in [ + ("models", None), + ("functions", None), + ("seeds", Some("seed_file")), + ("sources", Some("source_file")), + ] { + for resource in project.getattr(collection)?.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + let owner: Bound<'py, PyAny> = match file { + Some(file) => resource.getattr(file)?, + None => resource.clone(), + }; + paths.set_item(resource.getattr("key")?, owner.getattr("relative_path")?)?; + } + } + Ok(paths) +} + +/// `config` with every selector-scoped rule ignore resolved into deterministic finding paths. +pub(crate) fn resolved_ignore_selectors<'py>( + config: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = config.py(); + let ignores: Vec> = config + .getattr("rule_ignores")? + .try_iter()? + .collect::>()?; + let mut scoped: bool = false; + for ignore in &ignores { + scoped |= ignore.getattr("selectors")?.is_truthy()?; + } + if !scoped { + return Ok(config.clone()); + } + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let scope: Bound<'py, PyAny> = imported(py, PLANNER_SCOPE_MODULE, "build_planner_scope")?; + let path_by_key: Bound<'py, PyDict> = resource_paths(project)?; + let mut resolved: Vec> = Vec::with_capacity(ignores.len()); + for ignore in ignores { + let selectors: Bound<'py, PyAny> = ignore.getattr("selectors")?; + if !selectors.is_truthy()? { + resolved.push(ignore); + continue; + } + let selected: Bound<'py, PyAny> = call_with( + &scope, + &[("project", project.clone()), ("select", selectors.clone())], + )?; + let mut selector_paths: BTreeSet = BTreeSet::new(); + for key in selected.getattr("selected_keys")?.try_iter()? { + if let Some(path) = path_by_key.get_item(key?)? { + selector_paths.insert(path.call_method0("as_posix")?.extract()?); + } + } + if selector_paths.is_empty() { + let joined: Vec = selectors.extract()?; + return Err(rules_error( + py, + &format!( + "rule ignore selectors resolve no resources with paths: {}", + joined.join(", ") + ), + None, + )); + } + for path in ignore.getattr("paths")?.try_iter()? { + selector_paths.insert(path?.extract()?); + } + resolved.push(replace.call( + (ignore,), + Some(&keywords( + py, + &[ + ("paths", PyTuple::new(py, selector_paths)?.into_any()), + ("selectors", PyTuple::empty(py).into_any()), + ], + )?), + )?); + } + replace.call( + (config,), + Some(&keywords( + py, + &[("rule_ignores", PyTuple::new(py, resolved)?.into_any())], + )?), + ) +} + +/// `config` with selector-scoped rule ignores resolved against the compiled project. +#[pyfunction] +#[pyo3(signature = (*, config, project))] +fn resolve_rule_ignore_selectors<'py>( + config: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + resolved_ignore_selectors(config, project) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_rule_catalogue, module)?)?; + module.add_function(wrap_pyfunction!(select_catalogue_rules, module)?)?; + module.add_function(wrap_pyfunction!(resolve_rule_ignore_selectors, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index 922c6cede2..fe1ab554a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -64,7 +64,7 @@ fn custom_rule_import_closure<'py>( /// One rule's source file and helper closure fingerprint, or its check's source digest. #[pyfunction] #[pyo3(signature = (*, rule, project_dir, import_closure = None))] -fn custom_rule_implementation_fingerprint( +pub(crate) fn custom_rule_implementation_fingerprint( rule: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>, import_closure: Option<&Bound<'_, PyAny>>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs new file mode 100644 index 0000000000..975e1a0262 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -0,0 +1,313 @@ +//! Speculative SQL checks started once compile inputs are ready, and the lint settings they use. + +use std::path::PathBuf; +use std::time::Instant; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; +const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_CONSTANTS_MODULE: &str = "sqlbuild.lint.constants"; +const FUTURES_MODULE: &str = "concurrent.futures"; +const THREADING_MODULE: &str = "threading"; +const LINT_THREAD_PREFIX: &str = "sqlbuild-lint"; +const SQL_RULES_CACHE_FILE: &str = "sql.json"; + +/// The SQL rule result cache file of `project_dir`. +pub(crate) fn sql_rule_cache_path<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let root: PathBuf = project_dir.extract()?; + Ok(rules_bulk_cache_path(&root, SQL_RULES_CACHE_FILE) + .into_pyobject(project_dir.py())? + .into_any()) +} + +/// The native SQL lint configuration that SQL rules run with. +pub(crate) fn sql_lint_config<'py>( + dialect: &Bound<'py, PyAny>, + codes: &[String], + thresholds: &Bound<'py, PyAny>, + (relation_keys, allow_suppressions): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = dialect.py(); + let threshold = |key: &str, default: &str| -> PyResult> { + thresholds.call_method1( + "get", + ( + imported(py, LINT_CONSTANTS_MODULE, key)?, + imported(py, LINT_CONSTANTS_MODULE, default)?, + ), + ) + }; + call_with( + &imported(py, LINT_MODELS_MODULE, "LintConfig")?, + &[ + ("dialect", dialect.clone()), + ("enabled_native_rules", PyTuple::new(py, codes)?.into_any()), + ("header_rules_enabled", false.into_bound_py_any(py)?), + ( + "max_literal_length", + threshold("MAX_LITERAL_LENGTH_THRESHOLD", "DEFAULT_MAX_LITERAL_LENGTH")?, + ), + ( + "max_ranking_order_by", + threshold( + "MAX_RANKING_ORDER_BY_THRESHOLD", + "DEFAULT_MAX_RANKING_ORDER_BY", + )?, + ), + ("relation_keys", relation_keys), + ("allow_suppressions", allow_suppressions), + ], + ) +} + +/// The project, SQL rule codes and lint settings early SQL lint runs with, if it should run. +#[pyfunction] +#[pyo3(signature = (*, inputs, dialect))] +fn sql_lint_preparation<'py>( + inputs: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = inputs.py(); + let native = |name: &str| imported(py, NATIVE_MODULE, name); + let project_dir: Bound<'py, PyAny> = inputs + .getattr("discovered_inputs")? + .getattr("project_dir")?; + if project_dir.is_none() { + return Ok(None); + } + for diagnostic in inputs.getattr("diagnostics")?.try_iter()? { + if diagnostic?.getattr("is_error")?.is_truthy()? { + return Ok(None); + } + } + let config: Bound<'py, PyAny> = call_with( + &native("valid_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + if config.is_none() || !config.getattr("select")?.is_truthy()? { + return Ok(None); + } + if config.getattr("cache")?.getattr("enabled")?.is_truthy()? + && call_with( + &native("load_sql_rule_cache")?, + &[("path", sql_rule_cache_path(&project_dir)?)], + )? + .is_truthy()? + { + return Ok(None); + } + let codes: Vec = call_with( + &native("selected_sql_rule_codes")?, + &[ + ( + "select", + PyList::new( + py, + config + .getattr("select")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ( + "ignore", + PyList::new( + py, + config + .getattr("ignore")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ], + )? + .extract()?; + if codes.is_empty() { + return Ok(None); + } + let mut model_rows: Vec> = Vec::new(); + for model in inputs.getattr("model_inputs")?.try_iter()? { + model_rows.push(model?.getattr("relation_key_row")?); + } + let mut seeds: Vec> = Vec::new(); + for seed in inputs.getattr("seed_inputs")?.try_iter()? { + seeds.push(seed?.getattr("schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in inputs.getattr("source_inputs")?.try_iter()? { + sources.push(source?.getattr("source_entry")?); + } + let relation_keys: Bound<'py, PyAny> = call_with( + &native("declared_relation_keys")?, + &[ + ("model_rows", PyList::new(py, model_rows)?.into_any()), + ("seeds", PyList::new(py, seeds)?.into_any()), + ("sources", PyList::new(py, sources)?.into_any()), + ], + )?; + let lint_config: Bound<'py, PyAny> = sql_lint_config( + dialect, + &codes, + &config.getattr("thresholds")?, + (relation_keys, config.getattr("allow_exceptions")?), + )?; + Ok(Some(PyTuple::new( + py, + [ + project_dir, + PyTuple::new(py, codes)?.into_any(), + lint_config, + ], + )?)) +} + +/// Each model's compile expansion for early lint, whole when the compile handed one over. +#[pyfunction] +fn early_lint_expansions<'py>(model_inputs: &Bound<'py, PyAny>) -> PyResult> { + let expansions: Bound<'py, PyDict> = PyDict::new(model_inputs.py()); + for model in model_inputs.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let mut expansion: Bound<'py, PyAny> = model.getattr("lint_expansion")?; + if !expansion.is_truthy()? { + expansion = model.getattr("sql_expansion")?; + } + if !expansion.is_none() { + expansions.set_item( + model.getattr("model_file")?.getattr("file_path")?, + expansion, + )?; + } + } + Ok(expansions) +} + +/// Own speculative SQL checks for one compilation invocation. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct EarlySqlLint { + #[pyo3(get)] + enabled: bool, + #[pyo3(get)] + preparation: Py, + #[pyo3(get)] + expansion_reuse: Py, + stop_requested: Py, + executor: Py, +} + +impl EarlySqlLint { + /// A manager that starts speculative checks only when `enabled`. + pub(crate) fn new(py: Python<'_>, enabled: bool) -> PyResult { + let executor: Bound<'_, PyAny> = call_with( + &imported(py, FUTURES_MODULE, "ThreadPoolExecutor")?, + &[ + ("max_workers", 1.into_bound_py_any(py)?), + ( + "thread_name_prefix", + LINT_THREAD_PREFIX.into_bound_py_any(py)?, + ), + ], + )?; + Ok(Self { + enabled, + preparation: py.None(), + expansion_reuse: py.None(), + stop_requested: imported(py, THREADING_MODULE, "Event")?.call0()?.unbind(), + executor: executor.unbind(), + }) + } +} + +#[pymethods] +impl EarlySqlLint { + #[new] + #[pyo3(signature = (*, enabled))] + fn py_new(py: Python<'_>, enabled: bool) -> PyResult { + Self::new(py, enabled) + } + + fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { + slf + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { + let _ = self.stop(py)?; + Ok(()) + } + + /// Start early SQL lint for `inputs` when enabled and the project selects SQL rules. + #[pyo3(signature = (*, inputs, dialect))] + fn start( + mut slf: PyRefMut<'_, Self>, + inputs: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if !slf.enabled { + return Ok(()); + } + let py: Python<'_> = inputs.py(); + let preparation: Bound<'_, PyAny> = call_with( + &imported(py, RULE_THREADS_MODULE, "prepare_sql_rules")?, + &[ + ("inputs", inputs.clone()), + ("executor", slf.executor.bind(py).clone()), + ("dialect", dialect.clone()), + ("stop", slf.stop_requested.bind(py).clone()), + ], + )?; + let declaration_scope: Bound<'_, PyAny> = inputs.getattr("declaration_scope")?; + let expansion_reuse: Bound<'_, PyAny> = if declaration_scope.is_none() { + declaration_scope + } else { + call_with( + &imported(py, RULES_MODELS_MODULE, "SqlExpansionReuse")?, + &[ + ("discovered_inputs", inputs.getattr("discovered_inputs")?), + ("declaration_scope", declaration_scope), + ], + )? + }; + slf.preparation = preparation.unbind(); + slf.expansion_reuse = expansion_reuse.unbind(); + Ok(()) + } + + /// Abandon unconsumed speculative work and return the milliseconds spent joining it. + fn stop(&self, py: Python<'_>) -> PyResult { + let started: Instant = Instant::now(); + let _ = self.stop_requested.bind(py).call_method0("set")?; + let _ = self.executor.bind(py).call_method( + "shutdown", + (), + Some(&keywords( + py, + &[ + ("wait", true.into_bound_py_any(py)?), + ("cancel_futures", true.into_bound_py_any(py)?), + ], + )?), + )?; + Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(sql_lint_preparation, module)?)?; + module.add_function(wrap_pyfunction!(early_lint_expansions, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs new file mode 100644 index 0000000000..b9a2663c78 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs @@ -0,0 +1,227 @@ +//! Rule inspection text and the generated rules skill, rendered and installed natively. + +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; +use sqlbuild_rules::errors::{SkillInstallError, SkillInstallErrorKind}; +use sqlbuild_rules::models::RulesConfig; +use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; +use sqlbuild_rules::rule_catalogue::main::install_rendered_skills::install_rendered_skills; +use sqlbuild_rules::rule_catalogue::main::render_skills_body::render_skills_body; +use sqlbuild_rules::rule_catalogue::main::ruleset_fingerprint::ruleset_fingerprint; +use sqlbuild_rules::rule_catalogue::main::skill_lock_name::skill_lock_name; +use sqlbuild_rules::rule_catalogue::models::{FingerprintRule, OptionText, RuleText, SkillTarget}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::rules::catalogue::{ + native_rules_config, references_custom_rules, rule_catalogue, select_rules, verify_custom_rules, +}; +use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_implementation_fingerprint; +use crate::bindings::_helpers::rules::selection::rules_error; +use crate::bindings::_helpers::rules::skills::{owned_skill, skill_state}; + +const FILELOCK_MODULE: &str = "filelock"; +const SKILL_PATHS: [&str; 3] = [ + ".agents/skills/sqlbuild-rules/SKILL.md", + ".claude/skills/sqlbuild-rules/SKILL.md", + ".opencode/skills/sqlbuild-rules/SKILL.md", +]; +const SKILL_LOCK_DIRECTORY: &str = "sqlbuild-rules-skill-locks"; +const FRESH_SKILL: &str = "fresh"; + +fn text_field(object: &Bound<'_, PyAny>, field: &str) -> PyResult { + object.getattr(field)?.extract() +} + +fn repr_text(value: &Bound<'_, PyAny>) -> PyResult { + value.repr()?.extract() +} + +/// The text view of one Python `Rule`, with effective options under `config` when given. +fn rule_text(rule: &Bound<'_, PyAny>, config: Option<&Bound<'_, PyAny>>) -> PyResult { + let code: String = text_field(rule, "code")?; + let configured: Option> = match config { + Some(config) => config + .getattr("rule_options")? + .cast_into::()? + .get_item(&code)?, + None => None, + }; + let mut options: Vec = Vec::new(); + for option in rule.getattr("options")?.try_iter()? { + let option: Bound<'_, PyAny> = option?; + let name: String = text_field(&option, "name")?; + let default: Bound<'_, PyAny> = option.getattr("default")?; + let effective: Bound<'_, PyAny> = match &configured { + Some(values) => values.call_method1("get", (&name, &default))?, + None => default.clone(), + }; + options.push(OptionText { + name, + description: text_field(&option, "description")?, + default_repr: repr_text(&default)?, + effective_repr: repr_text(&effective)?, + }); + } + let guidance: Bound<'_, PyAny> = rule.getattr("guidance")?; + Ok(RuleText { + code, + family: text_field(rule, "family")?, + slug: text_field(rule, "slug")?, + message: text_field(rule, "message")?, + remediation: text_field(rule, "remediation")?, + enabled_by_default: rule.getattr("enabled_by_default")?.is_truthy()?, + custom: rule.getattr("custom")?.is_truthy()?, + source: rule.getattr("source")?.extract()?, + options, + good_example: if guidance.is_none() { + None + } else { + Some(text_field(&guidance, "good_example")?) + }, + }) +} + +/// `sqb rules show` text for one rule, with effective thresholds when a config is given. +#[pyfunction] +#[pyo3(signature = (*, rule, config = None))] +fn format_rule(rule: &Bound<'_, PyAny>, config: Option>) -> PyResult { + let config: Option> = config.filter(|config| !config.is_none()); + let native: Option = config.as_ref().map(native_rules_config).transpose()?; + let text: RuleText = rule_text(rule, config.as_ref())?; + compiler_guard(|| Ok(format_rule_text(&text, native.as_ref()))) +} + +fn fingerprint_rule( + rule: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let custom: bool = rule.getattr("custom")?.is_truthy()?; + Ok(FingerprintRule { + code: text_field(rule, "code")?, + family: text_field(rule, "family")?, + slug: text_field(rule, "slug")?, + message: text_field(rule, "message")?, + remediation: text_field(rule, "remediation")?, + source: rule.getattr("source")?.extract()?, + implementation_hash: if custom { + custom_rule_implementation_fingerprint(rule, project_dir, None)? + } else { + NATIVE_BUILD_IDENTITY.to_owned() + }, + }) +} + +/// The generated skill content and the ruleset fingerprint it records. +fn render_skills( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult<(String, String)> { + let py: Python<'_> = config.py(); + let native: RulesConfig = native_rules_config(config)?; + let catalogue = rule_catalogue(config, project_dir, references_custom_rules(config)?)?; + let rules = select_rules(&catalogue, config, project_dir, None)?; + let mut custom_selected: bool = false; + for rule in &rules { + custom_selected |= rule.getattr("custom")?.is_truthy()?; + } + if custom_selected { + verify_custom_rules(&rules, project_dir)?; + } + let fingerprints: Vec = rules + .iter() + .map(|rule| fingerprint_rule(rule, project_dir)) + .collect::>()?; + let texts: Vec = rules + .iter() + .map(|rule| rule_text(rule, Some(config))) + .collect::>()?; + let (body, fingerprint): (String, Result) = compiler_guard(|| { + Ok(( + render_skills_body(&texts, &native), + ruleset_fingerprint(&fingerprints, &native), + )) + })?; + let fingerprint: String = fingerprint.map_err(|message| rules_error(py, &message, None))?; + Ok((owned_skill(py, &body, &fingerprint)?, fingerprint)) +} + +fn current_content(path: &Path) -> PyResult> { + if !path.is_file() { + return Ok(None); + } + std::fs::read_to_string(path).map(Some).map_err(PyErr::from) +} + +fn install_error(py: Python<'_>, error: SkillInstallError) -> PyErr { + let raised: PyErr = match error.kind { + SkillInstallErrorKind::Refused => rules_error(py, &error.message, None), + SkillInstallErrorKind::Io => PyOSError::new_err(error.message), + }; + for note in error.notes { + let _ = raised.value(py).call_method1("add_note", (note,)); + } + raised +} + +fn lock_path(project_dir: &Path) -> PyResult { + let canonical: String = project_dir.canonicalize()?.display().to_string(); + let canonical: String = if cfg!(windows) { + canonical.to_lowercase().replace('/', "\\") + } else { + canonical + }; + let directory: PathBuf = std::env::temp_dir().join(SKILL_LOCK_DIRECTORY); + std::fs::create_dir_all(&directory)?; + Ok(directory.join(skill_lock_name(&canonical))) +} + +/// Install every owned rules skill target, or with `check` report whether all are fresh. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, check))] +fn install_rules_skills( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + check: bool, +) -> PyResult { + let py: Python<'_> = config.py(); + let (content, fingerprint): (String, String) = render_skills(config, project_dir)?; + let root: PathBuf = project_dir.extract()?; + let targets = || -> PyResult> { + let mut targets: Vec = Vec::with_capacity(SKILL_PATHS.len()); + for relative in SKILL_PATHS { + let path: PathBuf = root.join(relative); + let state: String = skill_state(current_content(&path)?.as_deref(), &fingerprint); + targets.push(SkillTarget { path, state }); + } + Ok(targets) + }; + if check { + return Ok(targets()?.iter().all(|target| target.state == FRESH_SKILL)); + } + let lock: Bound<'_, PyAny> = + imported(py, FILELOCK_MODULE, "FileLock")?.call1((lock_path(&root)?,))?; + let _ = lock.call_method0("acquire")?; + let installed = (|| -> PyResult<()> { + let targets: Vec = targets()?; + compiler_guard(|| Ok(install_rendered_skills(&content, &targets)))? + .map_err(|error| install_error(py, error)) + })(); + let released = lock.call_method0("release"); + installed?; + let _ = released?; + Ok(true) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_rule, module)?)?; + module.add_function(wrap_pyfunction!(install_rules_skills, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 5bad746aca..b5ef88405a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,9 +1,12 @@ pub(crate) mod builtin_catalogue; +pub(crate) mod catalogue; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; +pub(crate) mod early_sql_lint; pub(crate) mod evaluation; pub(crate) mod findings; +pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; pub(crate) mod lint_sql_bodies; @@ -13,8 +16,11 @@ pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; pub(crate) mod rule_cache; +pub(crate) mod rule_plan; +pub(crate) mod rule_run; pub(crate) mod rules_config; pub(crate) mod run_lint; pub(crate) mod selection; pub(crate) mod skills; +pub(crate) mod sql_rules; pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 1451384b04..7d9f87a4b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,9 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, - lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, - selection, skills, + builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, + early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_text, + results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -17,11 +17,16 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { dsl_headers::register(module)?; custom_rule_evidence::register(module)?; rule_cache::register(module)?; + rule_run::register(module)?; + rule_plan::register(module)?; + sql_rules::register(module)?; + early_sql_lint::register(module)?; custom_rule_cache::register(module)?; builtin_catalogue::register(module)?; + catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; selection::register(module)?; results::register(module)?; - skills::register(module) + guidance::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs index 5a0bcbd5ec..9e5c883e7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs @@ -6,6 +6,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; use sqlbuild_rules::rule_results::main::format_result::format_result; +use sqlbuild_rules::rule_results::main::format_rules_progress::format_rules_progress; +use sqlbuild_rules::rule_results::main::format_skipped_rules::format_skipped_rules; use sqlbuild_rules::rule_results::models::{ResultCounts, RuleFinding}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; @@ -13,7 +15,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const DATACLASSES_MODULE: &str = "dataclasses"; const UTF8: &str = "utf-8"; -fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { Ok(RuleFinding { code: finding.getattr("code")?.extract()?, path: finding @@ -40,7 +42,10 @@ fn rule_findings(findings: &Bound<'_, PyAny>) -> PyResult> { /// One rules result for the requested CLI surface. #[pyfunction] #[pyo3(signature = (*, result, json_output))] -fn format_rules_result(result: &Bound<'_, PyAny>, json_output: bool) -> PyResult { +pub(crate) fn format_rules_result( + result: &Bound<'_, PyAny>, + json_output: bool, +) -> PyResult { let counts = ResultCounts { evaluated_models: result.getattr("evaluated_models")?.extract()?, unevaluated_resources: result.getattr("unevaluated_resources")?.extract()?, @@ -54,6 +59,34 @@ fn format_rules_result(result: &Bound<'_, PyAny>, json_output: bool) -> PyResult )) } +/// The rules phase completion line for one `RulesRunResult`, optionally noting skipped rules. +pub(crate) fn rules_progress( + elapsed_seconds: f64, + result: &Bound<'_, PyAny>, + note_skipped_rules: bool, +) -> PyResult { + let skipped: Option> = if note_skipped_rules { + Some(result.getattr("skipped_type_proof_rules")?.extract()?) + } else { + None + }; + Ok(format_rules_progress( + elapsed_seconds, + ( + result.getattr("built_in_ms")?.extract()?, + result.getattr("custom_ms")?.extract()?, + ), + skipped.as_deref(), + )) +} + +/// Describe type-proof rules skipped because SQL analysis is disabled, if any. +#[pyfunction] +#[pyo3(signature = (*, codes))] +pub(crate) fn format_skipped_type_proof_rules(codes: Vec) -> Option { + format_skipped_rules(&codes) +} + /// The authored lines of one finding's file; an unreadable file has none. fn file_lines(project_dir: &Bound<'_, PyAny>, path: &str) -> PyResult> { let py = project_dir.py(); @@ -101,6 +134,7 @@ fn with_duplicate_literal_hints<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(format_rules_result, module)?)?; + module.add_function(wrap_pyfunction!(format_skipped_type_proof_rules, module)?)?; module.add_function(wrap_pyfunction!(with_duplicate_literal_hints, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 54a905591f..8163395c66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -4,12 +4,16 @@ use std::collections::HashSet; use std::path::PathBuf; use std::sync::Mutex; -use pyo3::exceptions::PyOSError; +use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods}; -use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use pyo3::{ + FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction, +}; +use serde_json::{Value, json}; +use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; @@ -19,26 +23,29 @@ use sqlbuild_rules::models::{ Declaration, DeclarationMember, EvaluateRequest, Fault, Model, ModelRow, ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, }; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; +use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::records::declaration_rows; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; +use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; use crate::bindings::types::CompilerDetach; const LEFT_ARGUMENT: &str = "left"; const RIGHT_ARGUMENT: &str = "right"; - -type EvaluationRow<'py> = ( - Vec>, - Vec, - u64, - u64, - u64, - u64, - bool, -); +const DIALECT_ARGUMENT: &str = "dialect"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; +const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const NATIVE_RULES_CACHE_FILE: &str = "native.json"; +const NATIVE_RULES_MEMO_FILE: &str = "native-response.json"; /// Name, relative path, members, value, value type and rendering of one declaration. #[derive(FromPyObject)] @@ -52,10 +59,7 @@ struct DeclarationRowPy<'py>( ); /// The request's project facts, which models, tests and scenarios are built beside. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct ProjectRowsPy<'py> { - header_json: Bound<'py, PyBytes>, +struct ProjectRows<'py> { memo: Option<(String, String)>, sql_analysis_enabled: bool, include_type_proof: bool, @@ -64,6 +68,7 @@ struct ProjectRowsPy<'py> { public_constants: Vec>, initial_findings: Vec>, types_equal: Bound<'py, PyAny>, + dialect: Bound<'py, PyAny>, } /// A natively built rules request held until it is evaluated once. @@ -91,16 +96,97 @@ impl NativeRulesRequest { } } +/// Build the built-in rules request natively from the compiled project's rows. #[pyfunction] -fn build_rules_request<'py>( - mut project: ProjectRowsPy<'py>, - models: (PyRef<'py, NativeCompiledProject>, Vec), - sql_tests: Vec>, - scenarios: Vec>, -) -> PyResult { +#[pyo3(signature = (**options))] +fn build_rules_request(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("build_rules_request", options)?; + let py: Python<'_> = request.py(); + as_rules_error(py, built_request(&request)) +} + +fn built_request(options: &Request<'_, '_>) -> PyResult { + let py: Python<'_> = options.py(); + let project: Bound<'_, PyAny> = options.get("project")?; + let config: Bound<'_, PyAny> = options.get("config")?; + let project_dir: Bound<'_, PyAny> = options.get("project_dir")?; + let dialect: Bound<'_, PyAny> = options.get("dialect")?; + let resolved: Bound<'_, PyAny> = project_dir.call_method0("resolve")?; + let resolved_path: PathBuf = resolved.extract()?; + let header: Value = json!({ + "version": API_VERSION, + "project_dir": resolved.str()?.to_str()?, + "dialect": dialect.extract::()?, + "config": config_value(&config)?, + "scope_index": plain_value(&call_with( + &imported(py, NATIVE_MODULE, "scope_metadata_projection")?, + &[("index", project.getattr("scope_index")?)], + )?)?, + "defer_suppressions": true, + "custom_rules": plain_value(&options.get("custom_payloads")?)?, + "rules_cache_path": path_text(py, rules_bulk_cache_path(&resolved_path, NATIVE_RULES_CACHE_FILE))?, + }); + let (public_enums, public_constants) = declaration_rows( + PyTuple::new(py, mapping_values(&project.getattr("public_enums")?)?)?.as_any(), + PyTuple::new(py, mapping_values(&project.getattr("public_constants")?)?)?.as_any(), + )?; + let enabled: bool = config.getattr("cache")?.getattr("enabled")?.is_truthy()?; + let memo: Option<(String, String)> = if enabled { + let unresolved: PathBuf = project_dir.extract()?; + Some(( + imported(py, CODE_IDENTITY_MODULE, "compiled_code_identity")? + .call0()? + .extract()?, + path_text( + py, + rules_bulk_cache_path(&unresolved, NATIVE_RULES_MEMO_FILE), + )?, + )) + } else { + None + }; + let mut attached_audit_targets: Vec = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + let target: Bound<'_, PyAny> = audit?.getattr("attached_target_name")?; + if !target.is_none() { + attached_audit_targets.push(target.extract()?); + } + } + let mut rows = ProjectRows { + memo, + sql_analysis_enabled: project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()?, + include_type_proof: options.get("include_type_proof")?.is_truthy()?, + attached_audit_targets, + public_enums: public_enums.extract()?, + public_constants: public_constants.extract()?, + initial_findings: options + .get("initial_findings")? + .try_iter()? + .collect::>()?, + types_equal: imported(py, TYPES_EQUAL_MODULE, "types_equal")?, + dialect, + }; + let facts: Bound<'_, NativeCompiledProject> = + imported(py, NATIVE_MODULE, "compiled_project_facts")? + .call1((&project,))? + .downcast_into()?; + let mut names: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + names.push(model?.getattr("name")?.extract()?); + } + let sql_tests: Vec> = project + .getattr("sql_tests")? + .try_iter()? + .collect::>()?; + let scenarios: Vec> = project + .getattr("sql_scenarios")? + .try_iter()? + .collect::>()?; compiler_guard(|| { - let mut request: EvaluateRequest = - serde_json::from_slice(project.header_json.as_bytes()).map_err(value_error)?; + let mut request: EvaluateRequest = serde_json::from_value(header).map_err(value_error)?; request.sql_tests = sql_tests .iter() .map(sql_test_fact) @@ -109,17 +195,17 @@ fn build_rules_request<'py>( .iter() .map(scenario_fact) .collect::>()?; - request.public_enums = declarations(std::mem::take(&mut project.public_enums))?; - request.public_constants = declarations(std::mem::take(&mut project.public_constants))?; - request.initial_findings = project + request.public_enums = declarations(std::mem::take(&mut rows.public_enums))?; + request.public_constants = declarations(std::mem::take(&mut rows.public_constants))?; + request.initial_findings = rows .initial_findings .iter() .map(finding_fault) .collect::>()?; - let rows: Vec = - project_model_rows(models.0.facts(), &models.1).map_err(value_error)?; - request.models = built_models(&project, &request, rows)?; - let memo: Option<(String, PathBuf)> = project + let model_rows: Vec = + project_model_rows(facts.borrow().facts(), &names).map_err(value_error)?; + request.models = built_models(&rows, &request, model_rows)?; + let memo: Option<(String, PathBuf)> = rows .memo .take() .map(|(identity, path)| (identity, PathBuf::from(path))); @@ -129,11 +215,19 @@ fn build_rules_request<'py>( }) } +/// Answer the request from the response memo or evaluate it, decoding findings natively. #[pyfunction] fn evaluate_rules_request<'py>( py: Python<'py>, request: &Bound<'py, NativeRulesRequest>, -) -> PyResult> { +) -> PyResult> { + as_rules_error(py, evaluated(py, request)) +} + +fn evaluated<'py>( + py: Python<'py>, + request: &Bound<'py, NativeRulesRequest>, +) -> PyResult> { let request: RowsRequest = request .borrow() .request @@ -149,44 +243,63 @@ fn evaluate_rules_request<'py>( RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; let classes = FindingClasses::new(py)?; - Ok(( - evaluation - .faults - .into_iter() - .map(|fault| classes.finding_of_fault(fault, false)) - .collect::>()?, - evaluation.selected_codes, - evaluation.evaluated_models, - evaluation.cache_hits, - evaluation.cache_misses, - evaluation.built_in_ms, - evaluation.reused, - )) + let findings: Vec> = evaluation + .faults + .into_iter() + .map(|fault| classes.finding_of_fault(fault, false)) + .collect::>()?; + call_with( + &imported(py, MODELS_MODULE, "NativeRulesEvaluation")?, + &[ + ("findings", PyTuple::new(py, findings)?.into_any()), + ( + "selected_codes", + PyTuple::new(py, evaluation.selected_codes)?.into_any(), + ), + ( + "evaluated_models", + evaluation.evaluated_models.into_bound_py_any(py)?, + ), + ("cache_hits", evaluation.cache_hits.into_bound_py_any(py)?), + ( + "cache_misses", + evaluation.cache_misses.into_bound_py_any(py)?, + ), + ("built_in_ms", evaluation.built_in_ms.into_bound_py_any(py)?), + ("reused", evaluation.reused.into_bound_py_any(py)?), + ], + ) } -/// The findings one finalisation applies the exception policy to. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct FinalizeRowsPy<'py> { - project_dir: String, - config_json: Bound<'py, PyBytes>, +/// Apply the exception policy natively, keeping each original finding's fixable mark. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, evaluated_codes, findings))] +pub(crate) fn finalize_rule_findings_rows<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, evaluated_codes: Vec, findings: Vec>, +) -> PyResult> { + let py: Python<'py> = config.py(); + as_rules_error( + py, + finalized(config, project_dir, (evaluated_codes, findings)), + ) } -#[pyfunction] -fn finalize_rule_findings_rows<'py>( - py: Python<'py>, - request: FinalizeRowsPy<'py>, -) -> PyResult>> { - let config: RulesConfig = serde_json::from_slice(request.config_json.as_bytes()) +fn finalized<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + (evaluated_codes, findings): (Vec, Vec>), +) -> PyResult> { + let py: Python<'py> = config.py(); + let config: RulesConfig = serde_json::from_value(config_value(config)?) .map_err(|error| value_error(format!("invalid findings request: {error}")))?; - let FinalizeRowsPy { - project_dir, - evaluated_codes, - findings, - .. - } = request; + let project_dir: String = project_dir + .call_method0("resolve")? + .str()? + .to_str()? + .to_owned(); let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); let mut faults: Vec = Vec::with_capacity(findings.len()); for finding in &findings { @@ -200,31 +313,63 @@ fn finalize_rule_findings_rows<'py>( .compiler_detach(|| finalize_rows(project_dir, config, &evaluated_codes, faults)) .map_err(value_error)?; let classes = FindingClasses::new(py)?; - finalized + let rows: Vec> = finalized .into_iter() .map(|fault| { let is_fixable: bool = fixable.contains(&fault_identity(&fault)); classes.finding_of_fault(fault, is_fixable) }) - .collect() + .collect::>()?; + PyTuple::new(py, rows) +} + +/// `asdict(config)` as orjson with `default=str` encodes it. +fn config_value(config: &Bound<'_, PyAny>) -> PyResult { + plain_value(&imported(config.py(), DATACLASSES_MODULE, "asdict")?.call1((config,))?) +} + +/// `str(path)` of the Python path built from `path`. +fn path_text(py: Python<'_>, path: PathBuf) -> PyResult { + Ok(path.into_pyobject(py)?.str()?.to_str()?.to_owned()) +} + +fn mapping_values<'py>(mapping: &Bound<'py, PyAny>) -> PyResult>> { + mapping.call_method0("values")?.try_iter()?.collect() +} + +/// Value and type errors of the request as `RulesError`, as the rules API reports them. +fn as_rules_error(py: Python<'_>, result: PyResult) -> PyResult { + result.map_err(|error| { + if error.is_instance_of::(py) || error.is_instance_of::(py) { + let message: String = error.value(py).to_string(); + rules_error(py, &message, Some(error)) + } else { + error + } + }) } -/// `left=` and `right=`: the callback is `types_equal`, whose arguments are keyword-only. -fn type_arguments<'py>(py: Python<'py>, left: &str, right: &str) -> PyResult> { - let arguments: Bound<'py, PyDict> = PyDict::new(py); +/// `left=`, `right=` and `dialect=`: `types_equal` takes keyword-only arguments. +fn type_arguments<'py>( + dialect: &Bound<'py, PyAny>, + left: &str, + right: &str, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(dialect.py()); arguments.set_item(LEFT_ARGUMENT, left)?; arguments.set_item(RIGHT_ARGUMENT, right)?; + arguments.set_item(DIALECT_ARGUMENT, dialect)?; Ok(arguments) } fn built_models( - project: &ProjectRowsPy<'_>, + project: &ProjectRows<'_>, request: &EvaluateRequest, rows: Vec, ) -> PyResult> { let mut python_error: Option = None; let fallback = |left: &str, right: &str| -> Result { - match type_arguments(project.types_equal.py(), left, right) + match type_arguments(&project.dialect, left, right) .and_then(|arguments| project.types_equal.call((), Some(&arguments))) .and_then(|equal| equal.extract::()) { @@ -286,7 +431,6 @@ fn fault_identity(fault: &Fault) -> (String, String, u64, u64, String) { ) } -/// A plain Python value as the rules request's JSON value, as orjson with `default=str` encodes it. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(build_rules_request, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs new file mode 100644 index 0000000000..64d111f236 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -0,0 +1,228 @@ +//! Plan one rules run: the catalogue, selected rules, project slice and settings it evaluates. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::catalogue::{ + references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, + verify_custom_rules, +}; +use crate::bindings::_helpers::rules::rule_run::freed_memory_released; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; +const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const PATHLIB_MODULE: &str = "pathlib"; + +/// Build the catalogue and select the rules, project slice and settings one run evaluates. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> { + let options = Request::new("plan_rule_run", options)?; + let py: Python<'py> = options.py(); + let graph: Bound<'py, PyAny> = options.get("graph")?; + let project: Bound<'py, PyAny> = graph.getattr("project")?; + freed_memory_released(py); + let effective_config: Bound<'py, PyAny> = + resolved_ignore_selectors(&options.get("config")?, &project)?; + let project_dir: Bound<'py, PyAny> = options.get("project_dir")?.call_method0("resolve")?; + let include_custom: bool = references_custom_rules(&effective_config)?; + let catalogue: Vec> = + rule_catalogue(&effective_config, &project_dir, include_custom)?; + let catalogue_tuple: Bound<'py, PyTuple> = PyTuple::new(py, &catalogue)?; + let custom_payloads: Bound<'py, PyAny> = call_with( + &imported(py, CUSTOM_PAYLOADS_MODULE, "custom_rule_payloads")?, + &[ + ("catalogue", catalogue_tuple.clone().into_any()), + ("project_dir", project_dir.clone()), + ], + )?; + let mut selected: Vec> = select_rules( + &catalogue, + &effective_config, + &project_dir, + Some(custom_payloads.clone()), + )?; + let mut skipped_type_proof_rules: Vec> = Vec::new(); + if options.get("no_sql_analysis")?.is_truthy()? + || !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + let type_proof: Bound<'py, PyAny> = + imported(py, RULES_CONSTANTS_MODULE, "TYPE_PROOF_RULE_CODES")?; + let mut kept: Vec> = Vec::with_capacity(selected.len()); + for rule in selected { + let code: Bound<'py, PyAny> = rule.getattr("code")?; + if type_proof.contains(&code)? { + skipped_type_proof_rules.push(code); + } else { + kept.push(rule); + } + } + selected = kept; + } + let mut custom_rules: Vec> = Vec::new(); + for rule in &selected { + if rule.getattr("custom")?.is_truthy()? { + custom_rules.push(rule.clone()); + } + } + if !custom_rules.is_empty() { + verify_custom_rules(&custom_rules, &project_dir)?; + } + let model_paths: Option> = + selected_model_paths(&graph, &options.get("selected_keys")?)?; + let selected_project: Bound<'py, PyAny> = match &model_paths { + None => project.clone(), + Some(paths) => selected_models_project(&project, paths)?, + }; + let mut codes: Vec> = Vec::with_capacity(selected.len()); + for rule in &selected { + codes.push(rule.getattr("code")?); + } + let selection_config: Bound<'py, PyAny> = + selection_rules_config(&effective_config, model_paths.as_ref(), &selected_project)?; + let selected_config: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?.call( + (selection_config,), + Some(&keywords( + py, + &[ + ("select", PyTuple::new(py, codes)?.into_any()), + ("ignore", PyTuple::empty(py).into_any()), + ], + )?), + )?; + call_with( + &imported(py, RULES_MODELS_MODULE, "RuleRunPlan")?, + &[ + ("effective_config", effective_config), + ("project_dir", project_dir), + ("catalogue", catalogue_tuple.into_any()), + ("custom_payloads", custom_payloads), + ("selected", PyTuple::new(py, selected)?.into_any()), + ( + "skipped_type_proof_rules", + PyTuple::new(py, skipped_type_proof_rules)?.into_any(), + ), + ( + "model_paths", + model_paths.map_or_else(|| py.None().into_bound(py), |paths| paths.into_any()), + ), + ("project", selected_project), + ("config", selected_config), + ], + ) +} + +/// The relative paths of the selected models, or `None` when the run evaluates every model. +fn selected_model_paths<'py>( + graph: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = graph.py(); + if selected_keys.is_none() { + return Ok(None); + } + let model_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let names: Bound<'py, PySet> = PySet::empty(py)?; + for key in selected_keys.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if key.getattr("resource_type")?.is(&model_type) { + names.add(key.getattr("name")?)?; + } + } + let paths: Bound<'py, PySet> = PySet::empty(py)?; + for model in graph.getattr("project")?.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + if names.contains(model.getattr("name")?)? { + paths.add(model.getattr("relative_path")?.call_method0("as_posix")?)?; + } + } + Ok(Some(PyFrozenSet::new(py, paths.iter())?)) +} + +/// `project` keeping only the models whose relative path is in `model_paths`. +fn selected_models_project<'py>( + project: &Bound<'py, PyAny>, + model_paths: &Bound<'py, PyFrozenSet>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let mut models: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + if model_paths.contains(model.getattr("relative_path")?.call_method0("as_posix")?)? { + models.push(model); + } + } + call_with( + &imported(py, NATIVE_MODULE, "project_with")?, + &[ + ("project", project.clone()), + ("models", PyTuple::new(py, models)?.into_any()), + ], + ) +} + +/// `config` keeping only the exceptions that apply to the selected models. +fn selection_rules_config<'py>( + config: &Bound<'py, PyAny>, + model_paths: Option<&Bound<'py, PyFrozenSet>>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = config.py(); + let Some(model_paths) = model_paths else { + return Ok(config.clone()); + }; + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let model_names: Bound<'py, PySet> = PySet::empty(py)?; + for model in project.getattr("models")?.try_iter()? { + model_names.add(model?.getattr("name")?)?; + } + let mut rule_exceptions: Vec> = Vec::new(); + for entry in config.getattr("rule_exceptions")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let path: Bound<'py, PyAny> = path_class + .call1((entry.getattr("path")?,))? + .call_method0("as_posix")?; + if model_paths.contains(path)? { + rule_exceptions.push(entry); + } + } + let mut graph_edge_exceptions: Vec> = Vec::new(); + for entry in config.getattr("graph_edge_exceptions")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if model_names.contains(entry.getattr("consumer")?)? { + graph_edge_exceptions.push(entry); + } + } + imported(py, DATACLASSES_MODULE, "replace")?.call( + (config,), + Some(&keywords( + py, + &[ + ( + "rule_exceptions", + PyTuple::new(py, rule_exceptions)?.into_any(), + ), + ( + "graph_edge_exceptions", + PyTuple::new(py, graph_edge_exceptions)?.into_any(), + ), + ], + )?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_rule_run, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs new file mode 100644 index 0000000000..d9406540f6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -0,0 +1,163 @@ +//! The finished result of one rules run, the bulk cache locations and freed-memory release. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path as bulk_cache_path; +use sqlbuild_rules::rule_results::main::group_unevaluated::group_unevaluated; +use sqlbuild_rules::rule_results::models::RuleFinding; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::rules::results::rule_finding; +use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; + +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const PATHLIB_MODULE: &str = "pathlib"; +const CTYPES_MODULE: &str = "ctypes"; + +fn integer(object: &Bound<'_, PyAny>, field: &str) -> PyResult { + object.getattr(field)?.extract() +} + +fn python_finding<'py>(py: Python<'py>, finding: RuleFinding) -> PyResult> { + call_with( + &imported(py, MODELS_MODULE, "Finding")?, + &[ + ("code", finding.code.into_bound_py_any(py)?), + ( + "path", + imported(py, PATHLIB_MODULE, "Path")?.call1((finding.path,))?, + ), + ("line", finding.line.into_bound_py_any(py)?), + ("column", finding.column.into_bound_py_any(py)?), + ("message", finding.message.into_bound_py_any(py)?), + ("remediation", finding.remediation.into_bound_py_any(py)?), + ("unevaluated", finding.unevaluated.into_bound_py_any(py)?), + ( + "affected_rules", + PyTuple::new(py, finding.affected_rules)?.into_any(), + ), + ("fixable", finding.fixable.into_bound_py_any(py)?), + ], + ) +} + +/// Finalize the combined SQL, built-in and custom findings of one planned rules run. +#[pyfunction] +#[pyo3(signature = (*, plan, sql_result, result))] +fn finish_rule_run<'py>( + plan: &Bound<'py, PyAny>, + sql_result: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = plan.py(); + let mut evaluated_codes: Vec = Vec::new(); + for rule in plan.getattr("selected")?.try_iter()? { + evaluated_codes.push(rule?.getattr("code")?.extract()?); + } + let mut combined: Vec> = Vec::new(); + for findings in [sql_result.getattr("findings")?, result.getattr("findings")?] { + combined.extend(findings.try_iter()?.collect::>>()?); + } + let finalized: Vec = finalize_rule_findings_rows( + &plan.getattr("config")?, + &plan.getattr("project_dir")?, + evaluated_codes, + combined, + )? + .iter() + .map(|finding| rule_finding(&finding)) + .collect::>()?; + let grouped: Vec = compiler_guard(|| Ok(group_unevaluated(finalized)))?; + let unevaluated: BTreeSet = grouped + .iter() + .filter(|finding| finding.unevaluated) + .map(|finding| finding.path.clone()) + .collect(); + let mut unevaluated_models: i64 = 0; + for model in plan.getattr("project")?.getattr("models")?.try_iter()? { + let path: String = model? + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?; + unevaluated_models += i64::from(unevaluated.contains(&path)); + } + let findings: Vec> = grouped + .into_iter() + .map(|finding| python_finding(py, finding)) + .collect::>()?; + let total = |field: &str| -> PyResult> { + (integer(result, field)? + integer(sql_result, field)?).into_bound_py_any(py) + }; + call_with( + &imported(py, MODELS_MODULE, "RulesRunResult")?, + &[ + ("findings", PyTuple::new(py, findings)?.into_any()), + ( + "evaluated_models", + (integer(result, "evaluated_models")? - unevaluated_models) + .max(0) + .into_bound_py_any(py)?, + ), + ( + "unevaluated_resources", + unevaluated.len().into_bound_py_any(py)?, + ), + ( + "built_in_ms", + (integer(sql_result, "sql_ms")? + integer(result, "built_in_ms")?) + .into_bound_py_any(py)?, + ), + ("custom_ms", result.getattr("custom_ms")?), + ("custom_cpu_ms", result.getattr("custom_cpu_ms")?), + ("cache_hits", total("cache_hits")?), + ("cache_misses", total("cache_misses")?), + ( + "skipped_type_proof_rules", + plan.getattr("skipped_type_proof_rules")?, + ), + ], + ) +} + +/// Run the selected built-in, SQL and custom rules; custom rules run on their worker threads. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_rules<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + imported(py, RULE_THREADS_MODULE, "evaluate_rules")?.call((), options.as_ref()) +} + +/// One bulk Rules cache file owned by the native compiler engine. +#[pyfunction] +#[pyo3(signature = (*, project_dir, file_name))] +fn rules_bulk_cache_path(project_dir: PathBuf, file_name: &str) -> PathBuf { + bulk_cache_path(&project_dir, file_name) +} + +/// Hand freed glibc heap pages back to the OS; a no-op elsewhere or without `malloc_trim`. +pub(crate) fn freed_memory_released(py: Python<'_>) { + if !cfg!(target_os = "linux") { + return; + } + let trim = || -> PyResult<()> { + let process: Bound<'_, PyAny> = imported(py, CTYPES_MODULE, "CDLL")?.call1((py.None(),))?; + let _ = process.getattr("malloc_trim")?.call1((0,))?; + Ok(()) + }; + let _ = trim(); +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finish_rule_run, module)?)?; + module.add_function(wrap_pyfunction!(run_rules, module)?)?; + module.add_function(wrap_pyfunction!(rules_bulk_cache_path, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 436ffbffa2..f70369c857 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -243,7 +243,10 @@ fn rules_error(py: Python<'_>, class: &str, message: String, help: Option, project_dir: PathBuf) -> PyResult> { +pub(crate) fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + let project_dir: PathBuf = project_dir + .canonicalize() + .or_else(|_| std::path::absolute(&project_dir))?; let config: RulesConfig = compiler_guard(|| Ok(load_config(&project_dir)))? .map_err(|message| rules_error(py, "RulesError", message, None))?; if let Some(error) = forbidden_exceptions(&config) { @@ -257,7 +260,21 @@ fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult Ok(config_object(py, &config)?.unbind()) } +/// The project's rules configuration, or `None` when it is invalid and the rules phase reports it. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn valid_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult>> { + match load_rules_config(py, project_dir) { + Ok(config) => Ok(Some(config)), + Err(error) if error.is_instance(py, &py.import(ERRORS_MODULE)?.getattr("RulesError")?) => { + Ok(None) + } + Err(error) => Err(error), + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(load_rules_config, module)?)?; + module.add_function(wrap_pyfunction!(valid_rules_config, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index f0c25105c0..fc819aaec2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -19,7 +19,7 @@ const BUILTINS_MODULE: &str = "builtins"; /// The selected built-in and custom rule codes; invalid configuration raises `RulesError`. #[pyfunction] #[pyo3(signature = (*, config, custom_payloads))] -fn selected_rule_codes( +pub(crate) fn selected_rule_codes( config: &Bound<'_, PyAny>, custom_payloads: &Bound<'_, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs index f6a103f885..3a6d23956f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs @@ -1,7 +1,6 @@ -//! Render and check the generated rules skill. +//! The ownership header and freshness of the generated rules skill. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::rules::selection::rules_error; @@ -10,9 +9,11 @@ const SKILL_OWNER: &str = "sqlbuild"; const SKILL_IDENTITY: &str = "sqlbuild-rules"; /// Generated guidance with its ownership header; a policy failure raises `RulesError`. -#[pyfunction] -#[pyo3(signature = (*, content, input_fingerprint))] -fn render_owned_skill(py: Python<'_>, content: &str, input_fingerprint: &str) -> PyResult { +pub(crate) fn owned_skill( + py: Python<'_>, + content: &str, + input_fingerprint: &str, +) -> PyResult { let rendered: Result = compiler_guard(|| { Ok(fensu_policy::render_owned_skill( SKILL_OWNER, @@ -26,10 +27,8 @@ fn render_owned_skill(py: Python<'_>, content: &str, input_fingerprint: &str) -> rendered.map_err(|message| rules_error(py, &message, None)) } -/// Whether generated guidance is current, stale or foreign under the ownership contract. -#[pyfunction] -#[pyo3(signature = (*, content, input_fingerprint))] -fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { +/// Whether generated guidance is current, stale, missing or foreign under the ownership contract. +pub(crate) fn skill_state(content: Option<&str>, input_fingerprint: &str) -> String { let freshness = fensu_policy::skill_freshness( content.map(str::as_bytes), SKILL_OWNER, @@ -38,9 +37,3 @@ fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { ); format!("{freshness:?}").to_lowercase() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; - module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs new file mode 100644 index 0000000000..c8172a16bf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -0,0 +1,661 @@ +//! Run the planned SQL rules through native lint, reusing cached and early-lint results. + +use std::collections::{BTreeSet, HashMap}; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, + PyTuple, +}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule_cache_path}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const LINT_CONSTANTS_MODULE: &str = "sqlbuild.lint.constants"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const JSON_MODULE: &str = "json"; +const BUILTINS_MODULE: &str = "builtins"; +const SQL_RULE_PREFIX: &str = "SQBRSQL"; +const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSQL000"; +const DEFAULT_REMEDIATION: &str = "Update the authored SQL to satisfy this rule."; + +/// `native.name`, looked up at call time. +fn native<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, NATIVE_MODULE, name) +} + +/// The SQL rule codes among `rules`, in order. +fn sql_rule_codes(rules: &[Bound<'_, PyAny>]) -> PyResult> { + let mut codes: Vec = Vec::new(); + for rule in rules { + let code: String = rule.getattr("code")?.extract()?; + if code.starts_with(SQL_RULE_PREFIX) { + codes.push(code); + } + } + Ok(codes) +} + +/// The settings, project and inputs one SQL rules evaluation reads. +struct SqlRun<'a, 'py> { + codes: &'a [String], + project_dir: Bound<'py, PyAny>, + discovered_inputs: Bound<'py, PyAny>, + project: Bound<'py, PyAny>, + lint_config: Bound<'py, PyAny>, + selected_model_paths: Bound<'py, PyAny>, + cache_enabled: bool, + prepared_sql: Bound<'py, PyAny>, + expansion_reuse: Bound<'py, PyAny>, +} + +/// Run the planned SQL rules on the caller while built-in rules evaluate natively. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> { + let options = Request::new("run_planned_sql_rules", options)?; + let py: Python<'py> = options.py(); + let plan: Bound<'py, PyAny> = options.get("plan")?; + let graph: Bound<'py, PyAny> = options.get("graph")?; + let effective_config: Bound<'py, PyAny> = plan.getattr("effective_config")?; + let mut native_rules: Vec> = Vec::new(); + for rule in plan.getattr("selected")?.try_iter()? { + let rule: Bound<'py, PyAny> = rule?; + if !rule.getattr("custom")?.is_truthy()? { + native_rules.push(rule); + } + } + let codes: Vec = sql_rule_codes(&native_rules)?; + let lint_config: Bound<'py, PyAny> = sql_lint_config( + &options.get("dialect")?, + &codes, + &effective_config.getattr("thresholds")?, + ( + native(py, "compiled_relation_keys")?.call1((graph.getattr("project")?,))?, + effective_config.getattr("allow_exceptions")?, + ), + )?; + let run = SqlRun { + codes: &codes, + project_dir: plan.getattr("project_dir")?, + discovered_inputs: options.get("discovered_inputs")?, + project: plan.getattr("project")?, + lint_config, + selected_model_paths: plan.getattr("model_paths")?, + cache_enabled: effective_config + .getattr("cache")? + .getattr("enabled")? + .is_truthy()?, + prepared_sql: options.get("prepared_sql")?, + expansion_reuse: options.get("expansion_reuse")?, + }; + let lint_key: String = lint_key(&run.lint_config)?; + let identities: Vec<(String, String)> = model_rule_identities(&run, &lint_key)?; + let project_files: Option> = if !identities.is_empty() + && run.prepared_sql.is_none() + && run.selected_model_paths.is_none() + { + Some(collected_project_files(&run.project_dir)?) + } else { + None + }; + let started: Instant = Instant::now(); + let (findings, hits, misses) = + evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; + let sql_ms: i64 = (started.elapsed().as_secs_f64() * 1000.0).round_ties_even() as i64; + call_with( + &imported(py, RULES_MODELS_MODULE, "SqlRulesEvaluation")?, + &[ + ("findings", findings), + ("cache_hits", hits.into_bound_py_any(py)?), + ("cache_misses", misses.into_bound_py_any(py)?), + ("sql_ms", sql_ms.into_bound_py_any(py)?), + ], + ) +} + +fn lint_key(lint_config: &Bound<'_, PyAny>) -> PyResult { + call_with( + &native(lint_config.py(), "sql_rule_lint_key")?, + &[("config", lint_config.clone())], + )? + .extract() +} + +fn collected_project_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + call_with( + &native(py, "collect_lint_project_files")?, + &[ + ("project_dir", project_dir.clone()), + ("selected_paths", py.None().into_bound(py)), + ], + ) +} + +/// The project's models SQL rules analyse, keyed by relative path in project order. +fn sql_rule_models<'py>(project: &Bound<'py, PyAny>) -> PyResult)>> { + let mut models: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(models); + } + let excluded: Bound<'py, PyAny> = false.into_bound_py_any(project.py())?; + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let analysis: Bound<'py, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("sql_analysis",))?; + if analysis.is(&excluded) { + continue; + } + let path: String = model + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?; + if let Some(index) = models.iter().position(|(known, _)| *known == path) { + models[index].1 = model; + } else { + models.push((path, model)); + } + } + Ok(models) +} + +/// Each analysed model's SQL rule cache key, in project order. +fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult> { + let py: Python<'_> = run.project.py(); + if run.codes.is_empty() { + return Ok(Vec::new()); + } + let mut models: Vec> = Vec::new(); + let mut paths: Vec = Vec::new(); + for (path, model) in sql_rule_models(&run.project)? { + models.push(PyTuple::new( + py, + [ + PyString::new(py, &path).into_any(), + model.getattr("authored_sql")?, + model.getattr("query_sql")?, + contract_text(&model)?, + ], + )?); + paths.push(path); + } + let keys: Bound<'_, PyAny> = call_with( + &native(py, "sql_model_rule_keys")?, + &[ + ("lint_key", PyString::new(py, lint_key).into_any()), + ("codes", PyList::new(py, run.codes)?.into_any()), + ("models", PyList::new(py, models)?.into_any()), + ], + )?; + paths + .into_iter() + .map(|path| { + let key: String = keys.get_item(&path)?.extract()?; + Ok((path, key)) + }) + .collect() +} + +/// Canonical text of the dynamic column contract that SQL rule findings depend on. +fn contract_text<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let contract: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; + if contract.is_none() { + return Ok(contract); + } + imported(py, JSON_MODULE, "dumps")?.call( + (imported(py, DATACLASSES_MODULE, "asdict")?.call1((contract,))?,), + Some(&keywords( + py, + &[ + ("sort_keys", true.into_bound_py_any(py)?), + ("separators", (",", ":").into_bound_py_any(py)?), + ("default", imported(py, BUILTINS_MODULE, "str")?), + ], + )?), + ) +} + +/// The stored findings of every path whose key still matches; undecodable ones miss. +fn cached_findings<'py>( + bucket: &Bound<'py, PyAny>, + keys: Vec<(String, String)>, +) -> PyResult> { + let py: Python<'py> = bucket.py(); + Ok(call_with( + &native(py, "lookup_sql_rule_cache")?, + &[ + ("entries", bucket.clone()), + ("keys", keys.into_bound_py_any(py)?), + ], + )? + .downcast_into()?) +} + +/// The early SQL lint's full-project file read, when it ran for this project and these codes. +fn prepared_project_files<'py>(run: &SqlRun<'_, 'py>) -> PyResult>> { + let py: Python<'py> = run.project_dir.py(); + let prepared: &Bound<'py, PyAny> = &run.prepared_sql; + if prepared.is_none() + || prepared.getattr("project_dir")?.ne(&run.project_dir)? + || !same_codes(&prepared.getattr("codes")?, run.codes)? + || !prepared + .getattr("future")? + .call_method0("exception")? + .is_none() + { + return Ok(None); + } + let texts: Bound<'py, PyAny> = prepared + .getattr("future")? + .call_method0("result")? + .getattr("result")? + .getattr("source_texts")?; + Ok(Some( + imported(py, BUILTINS_MODULE, "dict")?.call1((texts,))?, + )) +} + +fn same_codes(codes: &Bound<'_, PyAny>, expected: &[String]) -> PyResult { + let mut known: BTreeSet = BTreeSet::new(); + if !codes.is_none() { + for code in codes.try_iter()? { + known.insert(code?.extract()?); + } + } + Ok(known == expected.iter().cloned().collect()) +} + +/// Each analysed model's findings, cached or linted, then cache accounting. +fn evaluated_sql_rules<'py>( + run: &SqlRun<'_, 'py>, + (lint_key, identities): (&str, Vec<(String, String)>), + prepared_files: Option>, +) -> PyResult<(Bound<'py, PyAny>, usize, usize)> { + let py: Python<'py> = run.project_dir.py(); + if run.codes.is_empty() { + return Ok((PyTuple::empty(py).into_any(), 0, 0)); + } + let project_dir: &Bound<'py, PyAny> = &run.project_dir; + let models: Vec<(String, Bound<'py, PyAny>)> = sql_rule_models(&run.project)?; + let cache_path: Bound<'py, PyAny> = sql_rule_cache_path(project_dir)?; + let bucket: Bound<'py, PyAny> = if run.cache_enabled { + call_with( + &native(py, "load_sql_rule_cache")?, + &[("path", cache_path.clone())], + )? + } else { + PyDict::new(py).into_any() + }; + let findings: Bound<'py, PyList> = PyList::empty(py); + let model_hits: Bound<'py, PyDict> = cached_findings(&bucket, identities.clone())?; + let mut misses: Vec<(String, String)> = Vec::new(); + for (path, identity) in &identities { + match model_hits.get_item(path)? { + Some(hit) => { + for finding in hit.try_iter()? { + findings.append(finding?)?; + } + } + None => misses.push((path.clone(), identity.clone())), + } + } + let selected_paths: Bound<'py, PySet> = PySet::empty(py)?; + for (path, _) in &misses { + selected_paths.add(project_dir.div(path)?)?; + } + let mut file_identities: Vec<(String, String)> = Vec::new(); + let mut project_files: Bound<'py, PyAny> = py.None().into_bound(py); + if run.selected_model_paths.is_none() { + let model_paths: Bound<'py, PySet> = PySet::empty(py)?; + for model in run.project.getattr("models")?.try_iter()? { + model_paths.add(project_dir.div(model?.getattr("relative_path")?)?)?; + } + project_files = match prepared_project_files(run)? { + Some(files) => files, + None => match prepared_files { + Some(files) => files, + None => collected_project_files(project_dir)?, + }, + }; + let expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; + let mut file_keys: Vec<(Bound<'py, PyAny>, String, Option)> = Vec::new(); + for item in project_files.downcast::()?.iter() { + let (file_path, contents) = item; + if model_paths.contains(&file_path)? { + continue; + } + let relative_path: String = file_path + .call_method1("relative_to", (project_dir,))? + .call_method0("as_posix")? + .extract()?; + let key: Option = call_with( + &native(py, "sql_file_rule_key")?, + &[ + ("lint_key", PyString::new(py, lint_key).into_any()), + ("codes", PyList::new(py, run.codes)?.into_any()), + ( + "relative_path", + PyString::new(py, &relative_path).into_any(), + ), + ("contents", contents), + ( + "expanded", + expansions.contains(&file_path)?.into_bound_py_any(py)?, + ), + ], + )? + .extract()?; + file_keys.push((file_path, relative_path, key)); + } + let mut keyed: Vec<(String, String)> = Vec::new(); + for (_, path, key) in &file_keys { + if let Some(key) = key { + keyed.push((path.clone(), key.clone())); + } + } + let file_hits: Bound<'py, PyDict> = cached_findings(&bucket, keyed)?; + for (file_path, relative_path, identity) in file_keys { + match file_hits.get_item(&relative_path)? { + Some(hit) => { + for finding in hit.try_iter()? { + findings.append(finding?)?; + } + } + None => { + selected_paths.add(file_path)?; + if let Some(identity) = identity { + file_identities.push((relative_path, identity)); + } + } + } + } + } + let violations: Vec> = if selected_paths.is_empty() { + Vec::new() + } else { + let mut dynamic: Vec> = Vec::new(); + for (_, model) in &models { + let contract: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; + if !contract.is_none() && contract.getattr("output_proven")?.is_truthy()? { + dynamic.push( + project_dir + .div(model.getattr("relative_path")?)? + .call_method0("resolve")?, + ); + } + } + prepared_lint_violations( + run, + &PyFrozenSet::new(py, selected_paths.iter())?, + &PyFrozenSet::new(py, dynamic)?, + &project_files, + )? + }; + let model_paths: BTreeSet = models.into_iter().map(|(path, _)| path).collect(); + let evaluated: Vec> = violation_findings(run, &violations, &model_paths)?; + for finding in &evaluated { + findings.append(finding)?; + } + let mut written: Vec<(String, String)> = misses.clone(); + written.extend(file_identities); + if run.cache_enabled && !written.is_empty() { + write_cache(&bucket, (&cache_path, written), &evaluated)?; + } + let hinted: Bound<'py, PyAny> = call_with( + &native(py, "with_duplicate_literal_hints")?, + &[ + ("findings", PyTuple::new(py, findings.iter())?.into_any()), + ("project_dir", project_dir.clone()), + ], + )?; + Ok((hinted, identities.len() - misses.len(), misses.len())) +} + +/// Store each rewritten path's evaluated findings in `bucket` and replace the cache file. +fn write_cache<'py>( + bucket: &Bound<'py, PyAny>, + (cache_path, written): (&Bound<'py, PyAny>, Vec<(String, String)>), + evaluated: &[Bound<'py, PyAny>], +) -> PyResult<()> { + let py: Python<'py> = bucket.py(); + let encode: Bound<'py, PyAny> = native(py, "encode_sql_rule_findings")?; + let mut by_path: HashMap>> = HashMap::new(); + for finding in evaluated { + let path: String = finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?; + by_path.entry(path).or_default().push(finding.clone()); + } + for (path, identity) in written { + let own: &[Bound<'py, PyAny>] = by_path.get(&path).map(Vec::as_slice).unwrap_or_default(); + let encoded: Bound<'py, PyAny> = + call_with(&encode, &[("findings", PyList::new(py, own)?.into_any())])?; + bucket.set_item(path, (identity, encoded))?; + } + let _ = call_with( + &native(py, "write_sql_rule_cache")?, + &[("path", cache_path.clone()), ("entries", bucket.clone())], + )?; + Ok(()) +} + +/// Lint violations of `selected_paths`, reusing a matching early SQL lint. +fn prepared_lint_violations<'py>( + run: &SqlRun<'_, 'py>, + selected_paths: &Bound<'py, PyFrozenSet>, + dynamic_output_paths: &Bound<'py, PyFrozenSet>, + source_files: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = run.project_dir.py(); + let run_lint: Bound<'py, PyAny> = native(py, "run_lint")?; + let compiled_expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; + let enabled_codes: Bound<'py, PyAny> = run.lint_config.getattr("enabled_native_rules")?; + let enabled: Vec = if enabled_codes.is_none() { + Vec::new() + } else { + enabled_codes.extract()? + }; + if run.prepared_sql.is_none() || !same_codes(&run.prepared_sql.getattr("codes")?, &enabled)? { + let expansion_context: Bound<'py, PyAny> = + if run.lint_config.getattr("native_enabled")?.is_truthy()? { + lint_expansion_context(run)? + } else { + py.None().into_bound(py) + }; + return call_with( + &run_lint, + &[ + ("project_dir", run.project_dir.clone()), + ("config", run.lint_config.clone()), + ("selected_paths", selected_paths.clone().into_any()), + ("discovered_inputs", run.discovered_inputs.clone()), + ("compiled_expansions", compiled_expansions), + ( + "dynamic_output_paths", + dynamic_output_paths.clone().into_any(), + ), + ("expansion_context", expansion_context), + ("source_files", source_files.clone()), + ], + )? + .getattr("violations")? + .try_iter()? + .collect(); + } + let completed: Bound<'py, PyAny> = + run.prepared_sql.getattr("future")?.call_method0("result")?; + let prepared: Bound<'py, PyAny> = completed.getattr("result")?; + let proof_paths: Bound<'py, PyAny> = + selected_paths.call_method1("__and__", (dynamic_output_paths,))?; + let mut violations: Vec> = Vec::new(); + for violation in prepared.getattr("violations")?.try_iter()? { + let violation: Bound<'py, PyAny> = violation?; + let path: Bound<'py, PyAny> = violation.getattr("file_path")?; + if selected_paths.contains(&path)? && !proof_paths.contains(&path)? { + violations.push(violation); + } + } + if proof_paths.is_truthy()? { + let proven: Bound<'py, PyAny> = call_with( + &run_lint, + &[ + ("project_dir", run.project_dir.clone()), + ("config", run.lint_config.clone()), + ("selected_paths", proof_paths.clone()), + ("discovered_inputs", run.discovered_inputs.clone()), + ("compiled_expansions", compiled_expansions), + ("dynamic_output_paths", proof_paths), + ("expansion_context", completed.getattr("context")?), + ( + "source_files", + imported(py, BUILTINS_MODULE, "dict")? + .call1((prepared.getattr("source_texts")?,))?, + ), + ], + )?; + for violation in proven.getattr("violations")?.try_iter()? { + violations.push(violation?); + } + } + Ok(violations) +} + +/// Build lint expansion, reusing the compiler's scope only for the same discovery. +fn lint_expansion_context<'py>(run: &SqlRun<'_, 'py>) -> PyResult> { + let py: Python<'py> = run.project_dir.py(); + let reuse: &Bound<'py, PyAny> = &run.expansion_reuse; + let declaration_scope: Bound<'py, PyAny> = if !reuse.is_none() + && reuse + .getattr("discovered_inputs")? + .is(&run.discovered_inputs) + { + reuse.getattr("declaration_scope")? + } else { + py.None().into_bound(py) + }; + call_with( + &native(py, "build_lint_expansion_context")?, + &[ + ("project_dir", run.project_dir.clone()), + ("discovered_inputs", run.discovered_inputs.clone()), + ("declaration_scope", declaration_scope), + ], + ) +} + +/// The findings of the selected SQL rules' violations; an engine failure leaves each unevaluated. +fn violation_findings<'py>( + run: &SqlRun<'_, 'py>, + violations: &[Bound<'py, PyAny>], + model_paths: &BTreeSet, +) -> PyResult>> { + let py: Python<'py> = run.project_dir.py(); + let engine_error: String = imported(py, LINT_EXCEPTIONS_MODULE, "NativeLintError")? + .getattr("code")? + .extract()?; + let safe_fixes: Bound<'py, PyAny> = imported(py, LINT_CONSTANTS_MODULE, "SAFE_RULE_FIX_CODES")?; + let resolved: Bound<'py, PyAny> = run.project_dir.call_method0("resolve")?; + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let no_paths: BTreeSet = BTreeSet::new(); + let mut evaluated: Vec> = Vec::new(); + for violation in violations { + let code: String = violation.getattr("code")?.extract()?; + if code == SQL_RULE_SUPPRESSION_CODE || run.codes.contains(&code) { + evaluated.push(lint_finding( + violation, + (&resolved, &safe_fixes), + model_paths, + )?); + } else if code == engine_error { + let finding: Bound<'py, PyAny> = + lint_finding(violation, (&resolved, &safe_fixes), &no_paths)?; + let message: String = violation.getattr("message")?.extract()?; + for rule_code in run.codes { + evaluated.push( + replace.call( + (&finding,), + Some(&keywords( + py, + &[ + ("code", PyString::new(py, rule_code).into_any()), + ( + "message", + PyString::new( + py, + &format!( + "Rule {rule_code} could not be evaluated: {message}" + ), + ) + .into_any(), + ), + ("unevaluated", true.into_bound_py_any(py)?), + ], + )?), + )?, + ); + } + } + } + Ok(evaluated) +} + +/// One lint violation as a finding; fixable when `format --fix` would attempt its edit. +fn lint_finding<'py>( + violation: &Bound<'py, PyAny>, + (resolved_dir, safe_fixes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + fixable_paths: &BTreeSet, +) -> PyResult> { + let py: Python<'py> = violation.py(); + let mut path: Bound<'py, PyAny> = violation.getattr("file_path")?; + if path.call_method0("is_absolute")?.is_truthy()? + && path + .call_method1("is_relative_to", (resolved_dir,))? + .is_truthy()? + { + path = path.call_method1("relative_to", (resolved_dir,))?; + } + let code: Bound<'py, PyAny> = violation.getattr("code")?; + let remediation: Bound<'py, PyAny> = violation.getattr("remediation")?; + let fixable: bool = !violation.getattr("fix")?.is_none() + && safe_fixes.contains(&code)? + && fixable_paths.contains(&path.call_method0("as_posix")?.extract::()?); + call_with( + &imported(py, RULES_MODELS_MODULE, "Finding")?, + &[ + ("code", code), + ("path", path), + ("line", violation.getattr("line")?), + ("column", violation.getattr("column")?), + ("message", violation.getattr("message")?), + ( + "remediation", + if remediation.is_truthy()? { + remediation + } else { + PyString::new(py, DEFAULT_REMEDIATION).into_any() + }, + ), + ("fixable", fixable.into_bound_py_any(py)?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_planned_sql_rules, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index cd6fd74b01..5395052fe9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -20,19 +20,19 @@ use sqlbuild_core::scenario_command::models::{ SnapshotLimitInputs, SnapshotLimits, SqlAnalysisCommand, }; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; @@ -208,13 +208,7 @@ pub(crate) fn resolve_adapter<'py>( adapter_name: &str, ) -> PyResult> { let py: Python<'py> = project.discovered.py(); - call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", text(py, adapter_name)), - ("project_dir", project.project_dir.clone()), - ], - ) + resolve_cli_adapter(&text(py, adapter_name), &project.project_dir) } /// The adapter's SQL analysis dialect, which snapshots require. @@ -242,10 +236,8 @@ pub(crate) fn project_connection_config<'py>( /// `not no_color and supports_color()`. pub(crate) fn style(request: &Bound<'_, PyAny>) -> PyResult { - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && imported(request.py(), COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(request.py())?; Ok(CliStyle { use_color }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 2087649059..a9a4e1b0a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -13,6 +13,7 @@ use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; @@ -25,7 +26,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ enums, constants, tests and audits with their prefix, for example macro:"; @@ -329,10 +329,7 @@ fn use_color( if output_stream.is_some() || request.getattr("no_color")?.extract::()? { return Ok(false); } - py.import(COLOR_MODULE)? - .getattr("supports_color")? - .call0()? - .extract() + stdout_supports_color(py) } /// Load offline compiler facts, run one pure query, and present its result. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 02231a9701..375a2c3dad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -14,11 +14,11 @@ use sqlbuild_core::seed_command::main::render_seed_row::render_seed_row; use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; +use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -421,16 +421,8 @@ fn load_seeds<'py>( #[pyfunction] #[pyo3(signature = (*, request))] fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "SeedInvocation")?, - ), - ], - )?; + let invocation: Bound<'_, PyAny> = + resolve_command_invocation(request, &imported(py, COMMAND_MODELS, "SeedInvocation")?)?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; let concurrency: i64 = if requested.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index c28e0f40d9..9baf8c6a1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -24,24 +24,23 @@ use sqlbuild_core::test_command::models::{ }; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::result_output::integration_values::json_value; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; -const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; -const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_test_runtime_target"; const DECIMAL_KIND: &str = "decimal"; const PASS_OUTCOME: &str = "pass"; const EXPECT_LABEL: &str = "expect"; @@ -154,7 +153,7 @@ fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { /// Nested `expect` rows of a completed test, one per step. fn expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = result.py(); - let row_class: Bound<'py, PyAny> = imported(py, PROGRESS_MODELS, "NestedProgressChildRow")?; + let row_class: Bound<'py, PyAny> = imported(py, COMMAND_MODELS, "NestedProgressChildRow")?; let name: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_name")?; let detail: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_detail")?; let mut rows: Vec> = Vec::new(); @@ -635,19 +634,10 @@ fn complete( #[pyo3(signature = (*, request))] fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "TestInvocation")?, - ), - ( - "require_connection", - PyBool::new(py, !inspect_only).to_owned().into_any(), - ), - ], + let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( + request, + &imported(py, COMMAND_MODELS, "TestInvocation")?, + !inspect_only, )?; let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; @@ -673,19 +663,15 @@ fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult header.set_item("use_color", invocation.getattr("use_color")?)?; imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; let results: Vec> = run_tests(&invocation, &pipeline, concurrency)?; - call_with( - &imported(py, RUNTIME_TARGET_MODULE, "write_test_runtime_target")?, - &[ - ( - "target_dir", - invocation - .getattr("effective_project_dir")? - .call_method1("__truediv__", ("target",))?, - ), - ("adapter", invocation.getattr("adapter")?), - ("plan_output", pipeline.getattr("plan_output")?), - ("results", PyTuple::new(py, &results)?.into_any()), - ], + write_test_runtime_target( + ( + &invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + &invocation.getattr("adapter")?, + ), + &pipeline.getattr("plan_output")?, + &PyTuple::new(py, &results)?.into_any(), )?; complete(request, &invocation, &results) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index 28fd26df81..dfe56f585b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -20,12 +20,12 @@ use sqlbuild_config::workspace::models::{PlaygroundRequest, SkillInstallRequest} use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; const TEMPLATES_DIRECTORY: &str = "playground_templates"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const OUTPUT_MODELS: &str = "sqlbuild.cli.output.models"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SKILLS_OPTIONS: (&[&str], &[&str]) = (&["project_dir"], &["global_install", "targets", "force"]); const UPDATE_OPTIONS: (&[&str], &[&str]) = ( @@ -198,9 +198,7 @@ fn run_playground_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResul py, &skill_request(py, playground_project_dir(&target_dir, &template))?, )?; - let use_color: bool = imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = stdout_supports_color(py)?; print( py, &playground_summary(&display_path, &template, CliStyle { use_color }), diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 070cb9fca0..316a1427d3 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,10 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_strategy, - project_assembly, refactor, references, render, rules, scopes, semantic_checks, - semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_identity, + planner_scope, planner_selection, planner_strategy, project_assembly, refactor, references, + render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, + type_system, }; use crate::bindings::models; @@ -46,6 +47,9 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { refactor::registration::register(module)?; plan_output::registration::register(module)?; planner_changes::registration::register(module)?; + planner_identity::registration::register(module)?; + planner_selection::registration::register(module)?; + planner_scope::registration::register(module)?; planner_strategy::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; diff --git a/crates/sqlbuild-rules/src/errors.rs b/crates/sqlbuild-rules/src/errors.rs index 98cae0b073..83fd7ca87e 100644 --- a/crates/sqlbuild-rules/src/errors.rs +++ b/crates/sqlbuild-rules/src/errors.rs @@ -4,3 +4,18 @@ pub enum RowsError { Rules(String), Io(std::io::Error), } + +/// Why generated skills were not installed, with rollback notes for the raised error. +#[derive(Debug)] +pub struct SkillInstallError { + pub kind: SkillInstallErrorKind, + pub message: String, + pub notes: Vec, +} + +/// Whether a skill installation failure is a policy refusal or a filesystem error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkillInstallErrorKind { + Refused, + Io, +} diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index ae2bbcee68..3b9babee52 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -11,6 +11,7 @@ pub mod errors; pub mod lint_text; pub mod models; pub mod rule_cache; +pub mod rule_catalogue; pub mod rule_results; pub mod rules; pub mod scope_metadata; diff --git a/crates/sqlbuild-rules/src/rule_cache/constants.rs b/crates/sqlbuild-rules/src/rule_cache/constants.rs index 4e3c6cd34a..4b5361cf03 100644 --- a/crates/sqlbuild-rules/src/rule_cache/constants.rs +++ b/crates/sqlbuild-rules/src/rule_cache/constants.rs @@ -13,3 +13,6 @@ pub(crate) const BUILD_IDENTITY_KEY: &str = "build_identity"; pub(crate) const ENTRIES_KEY: &str = "entries"; pub(crate) const IDENTITY_KEY: &str = "identity"; pub(crate) const FINDINGS_KEY: &str = "findings"; +pub(crate) const TARGET_DIRECTORY: &str = "target"; +pub(crate) const RULES_CACHE_DIRECTORY: &str = "rules-cache-native-v1"; +pub(crate) const BULK_DIRECTORY: &str = "bulk"; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs index 528348321d..20a4012180 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -4,3 +4,4 @@ pub mod lookup_rule_cache; pub mod model_rule_key; pub mod parse_rule_cache; pub mod rule_cache_text; +pub mod rules_bulk_cache_path; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs b/crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs new file mode 100644 index 0000000000..070c25aeeb --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs @@ -0,0 +1,14 @@ +//! Where one bulk Rules cache file lives, namespaced by the native compiler engine. + +use std::path::{Path, PathBuf}; + +use crate::rule_cache::constants::{BULK_DIRECTORY, RULES_CACHE_DIRECTORY, TARGET_DIRECTORY}; + +/// `/target/rules-cache-native-v1/bulk/`. +pub fn rules_bulk_cache_path(project_dir: &Path, file_name: &str) -> PathBuf { + project_dir + .join(TARGET_DIRECTORY) + .join(RULES_CACHE_DIRECTORY) + .join(BULK_DIRECTORY) + .join(file_name) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs new file mode 100644 index 0000000000..3910a88da1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod options; +pub(crate) mod python_repr; +pub(crate) mod staging; +pub(crate) mod thresholds; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs new file mode 100644 index 0000000000..9f91b818b8 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs @@ -0,0 +1,46 @@ +//! Python `isinstance` and equality semantics over option values. + +use crate::rule_catalogue::constants::OBJECT_TYPE; +use crate::rule_catalogue::models::{OptionKind, OptionValue}; + +/// `isinstance(value, kind)`, where `bool` is a subclass of `int`. +pub(crate) fn is_instance(value: &OptionValue, kind: &OptionKind) -> bool { + match (kind, value) { + (OptionKind::Bool, OptionValue::Bool(_)) + | (OptionKind::Int, OptionValue::Int(_) | OptionValue::Bool(_)) + | (OptionKind::Str, OptionValue::Str(_)) + | (OptionKind::Tuple, OptionValue::Tuple(_)) => true, + (OptionKind::Other(name), _) if name == OBJECT_TYPE => true, + (OptionKind::Other(name), OptionValue::Other(type_name)) => name == type_name, + _ => false, + } +} + +/// Whether `value` has the declared type, where a bool is not an `int` value. +pub(crate) fn has_value_type(value: &OptionValue, kind: &OptionKind) -> bool { + is_instance(value, kind) + && !(matches!(kind, OptionKind::Int) && matches!(value, OptionValue::Bool(_))) +} + +/// Python `==` between two option values, where `True == 1`. +pub(crate) fn python_equal(left: &OptionValue, right: &OptionValue) -> bool { + match (left, right) { + (OptionValue::Bool(a), OptionValue::Bool(b)) => a == b, + (OptionValue::Int(a), OptionValue::Int(b)) => a == b, + (OptionValue::Bool(a), OptionValue::Int(b)) + | (OptionValue::Int(b), OptionValue::Bool(a)) => i64::from(*a) == *b, + (OptionValue::Str(a), OptionValue::Str(b)) => a == b, + (OptionValue::Tuple(a), OptionValue::Tuple(b)) => { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| python_equal(x, y)) + } + _ => false, + } +} + +/// The integer value Python's `isinstance(value, int) and not isinstance(value, bool)` admits. +pub(crate) fn plain_integer(value: &OptionValue) -> Option { + match value { + OptionValue::Int(integer) => Some(*integer), + _ => None, + } +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs new file mode 100644 index 0000000000..3739290e1f --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs @@ -0,0 +1,41 @@ +//! Python `repr` text of the strings and dictionaries guidance prints. + +use crate::rule_catalogue::constants::{DELETE_CHARACTER, FIRST_PRINTABLE}; + +/// `repr(text)` for a Python `str`. +pub(crate) fn string_repr(text: &str) -> String { + let quote: char = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + let mut rendered: String = String::with_capacity(text.len() + 2); + rendered.push(quote); + for character in text.chars() { + match character { + '\\' => rendered.push_str("\\\\"), + '\n' => rendered.push_str("\\n"), + '\r' => rendered.push_str("\\r"), + '\t' => rendered.push_str("\\t"), + _ if character == quote => { + rendered.push('\\'); + rendered.push(character); + } + _ if (character as u32) < FIRST_PRINTABLE || character as u32 == DELETE_CHARACTER => { + rendered.push_str(&format!("\\x{:02x}", character as u32)); + } + _ => rendered.push(character), + } + } + rendered.push(quote); + rendered +} + +/// `repr(dict)` of string keys and already-rendered value texts, in the given order. +pub(crate) fn dict_repr<'a>(entries: impl IntoIterator) -> String { + let parts: Vec = entries + .into_iter() + .map(|(key, value)| format!("{}: {value}", string_repr(key))) + .collect(); + format!("{{{}}}", parts.join(", ")) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs new file mode 100644 index 0000000000..2610ae6a27 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs @@ -0,0 +1,221 @@ +//! Staged temporary files and backups of generated skill targets, with rollback. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::errors::{SkillInstallError, SkillInstallErrorKind}; + +const DEFAULT_MODE: u32 = 0o644; + +/// One staged replacement: the temporary content, the backup of the old target, the target. +pub(crate) struct Staged { + pub(crate) temporary: PathBuf, + pub(crate) backup: Option, + pub(crate) target: PathBuf, +} + +pub(crate) fn io_error(error: &std::io::Error) -> SkillInstallError { + SkillInstallError { + kind: SkillInstallErrorKind::Io, + message: error.to_string(), + notes: Vec::new(), + } +} + +/// Create a new hidden sibling of `target` holding `content`, and return its path. +fn write_sibling( + target: &Path, + suffix: &str, + content: &[u8], + mode: u32, +) -> std::io::Result { + let name: String = target + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let nanos: u128 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_nanos()) + .unwrap_or_default(); + let mut attempt: u32 = 0; + loop { + let path: PathBuf = target.with_file_name(format!( + ".{name}.{}-{nanos}-{attempt}{suffix}", + std::process::id() + )); + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(mut file) => { + file.write_all(content)?; + set_mode(&path, mode)?; + return Ok(path); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => attempt += 1, + Err(error) => return Err(error), + } + } +} + +#[cfg(unix)] +fn file_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path) + .map(|metadata| metadata.permissions().mode() & 0o7777) + .unwrap_or(DEFAULT_MODE) +} + +#[cfg(not(unix))] +fn file_mode(_path: &Path) -> u32 { + DEFAULT_MODE +} + +#[cfg(unix)] +fn set_mode(path: &Path, mode: u32) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) +} + +#[cfg(not(unix))] +fn set_mode(_path: &Path, _mode: u32) -> std::io::Result<()> { + Ok(()) +} + +fn read_if_file(path: &Path) -> std::io::Result>> { + if path.is_file() { + return fs::read(path).map(Some); + } + Ok(None) +} + +/// A temporary copy of `content` beside each target and a backup of each existing target. +pub(crate) fn stage(targets: &[PathBuf], content: &[u8]) -> Result, SkillInstallError> { + let mut staged: Vec = Vec::with_capacity(targets.len()); + let staging = |staged: &mut Vec| -> std::io::Result<()> { + for target in targets { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + let mode: u32 = if target.is_file() { + file_mode(target) + } else { + DEFAULT_MODE + }; + let temporary: PathBuf = write_sibling(target, ".tmp", content, mode)?; + staged.push(Staged { + temporary, + backup: None, + target: target.clone(), + }); + if target.is_file() { + let backup: PathBuf = write_sibling(target, ".bak", &fs::read(target)?, mode)?; + if let Some(last) = staged.last_mut() { + last.backup = Some(backup); + } + } + } + Ok(()) + }; + if let Err(error) = staging(&mut staged) { + remove_staged(&staged, &[]); + return Err(io_error(&error)); + } + Ok(staged) +} + +/// Replace each target with its staged content unless it changed since staging. +pub(crate) fn apply(staged: &[Staged], installed: &[u8]) -> Result<(), SkillInstallError> { + let mut replaced: Vec<(Option<&Path>, &Path)> = Vec::new(); + let mut outcome: Result<(), SkillInstallError> = Ok(()); + for entry in staged { + let replacing = || -> Result<(), SkillInstallError> { + let current: Option> = + read_if_file(&entry.target).map_err(|error| io_error(&error))?; + let expected: Option> = match &entry.backup { + Some(backup) => Some(fs::read(backup).map_err(|error| io_error(&error))?), + None => None, + }; + if current != expected { + return Err(SkillInstallError { + kind: SkillInstallErrorKind::Refused, + message: format!( + "refusing to overwrite rules skill changed during installation: {}", + entry.target.display() + ), + notes: Vec::new(), + }); + } + fs::rename(&entry.temporary, &entry.target).map_err(|error| io_error(&error)) + }; + if let Err(error) = replacing() { + outcome = Err(error); + break; + } + replaced.push((entry.backup.as_deref(), &entry.target)); + } + let mut retained: Vec = Vec::new(); + if let Err(error) = &mut outcome { + let (kept, notes) = rollback(&replaced, installed); + retained = kept; + error.notes.extend(notes); + } + remove_staged(staged, &retained); + outcome +} + +/// Restore replaced targets; returns the backups kept for recovery and the rollback notes. +fn rollback(replaced: &[(Option<&Path>, &Path)], installed: &[u8]) -> (Vec, Vec) { + let mut retained: Vec = Vec::new(); + let mut notes: Vec = Vec::new(); + let backup_text = |backup: Option<&Path>| { + backup.map_or_else(|| "None".to_owned(), |path| path.display().to_string()) + }; + for (backup, target) in replaced.iter().rev() { + let restoring = || -> std::io::Result { + if read_if_file(target)?.as_deref() != Some(installed) { + return Ok(false); + } + match backup { + None => match fs::remove_file(target) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(true), + }, + Some(backup) => fs::rename(backup, target).map(|()| true), + } + }; + match restoring() { + Ok(true) => {} + Ok(false) => { + retained.extend(backup.map(Path::to_path_buf)); + notes.push(format!( + "rollback preserved concurrently changed target {}; recovery backup: {}", + target.display(), + backup_text(*backup) + )); + } + Err(error) => { + retained.extend(backup.map(Path::to_path_buf)); + notes.push(format!( + "rollback failed for {}: {error}; recovery backup: {}", + target.display(), + backup_text(*backup) + )); + } + } + } + (retained, notes) +} + +fn remove_staged(staged: &[Staged], retained: &[PathBuf]) { + for entry in staged { + let _ = fs::remove_file(&entry.temporary); + if let Some(backup) = &entry.backup + && !retained.contains(backup) + { + let _ = fs::remove_file(backup); + } + } +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs new file mode 100644 index 0000000000..16b4bdd7fb --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs @@ -0,0 +1,37 @@ +//! Effective rule test thresholds rendered as guidance lines. + +use std::collections::BTreeMap; + +use crate::models::RulesConfig; +use crate::rule_catalogue::_helpers::python_repr::dict_repr; +use crate::rule_catalogue::constants::THRESHOLD_DEFAULTS; + +/// Global and path-scoped thresholds in deterministic authored order. +pub(crate) fn threshold_lines(config: &RulesConfig) -> Vec { + let mut thresholds: BTreeMap<&str, u32> = THRESHOLD_DEFAULTS.into_iter().collect(); + for (name, value) in &config.thresholds { + thresholds.insert(name, *value); + } + let mut lines: Vec = thresholds + .iter() + .map(|(name, value)| format!("- `{name}` = `{value}`")) + .collect(); + for (index, entry) in config.threshold_overrides.iter().enumerate() { + let values: String = dict_repr( + entry + .thresholds + .iter() + .map(|(name, value)| (name.as_str(), value.to_string())), + ); + lines.push(format!( + "- Override {} at `{}`: `{values}` ({})", + index + 1, + entry.paths.join(","), + entry.reason + )); + } + if !config.threshold_overrides.is_empty() { + lines.push("- Matching overrides apply in authored order; the last match wins.".into()); + } + lines +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs new file mode 100644 index 0000000000..927e472f7c --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -0,0 +1,135 @@ +//! Threshold defaults, guidance wording and the rule codes guidance sections depend on. + +/// Effective threshold defaults, sorted by name as guidance renders them. +pub(crate) const THRESHOLD_DEFAULTS: [(&str, u32); 10] = [ + ("max_constant_container_files", 10), + ("max_enum_container_files", 10), + ("max_macro_container_files", 10), + ("max_role_container_depth", 1), + ("max_subdomain_depth", 1), + ("min_audits_per_model", 1), + ("min_custom_rule_test_cases", 1), + ("min_shared_container_prefix_files", 2), + ("min_shared_owner_prefix_directories", 2), + ("min_tests_per_model", 1), +]; +/// The Python type every option value is an instance of. +pub(crate) const OBJECT_TYPE: &str = "object"; +/// The first printable code point; Python `repr` escapes control characters below it. +pub(crate) const FIRST_PRINTABLE: u32 = 0x20; +/// The DEL control character, which Python `repr` escapes. +pub(crate) const DELETE_CHARACTER: u32 = 0x7f; +pub(crate) const MINIMUM_RULES_CODES: [&str; 2] = ["SQBRTEST201", "SQBRTEST202"]; +pub(crate) const LAYOUT_THRESHOLD_RULE_CODES: [&str; 5] = [ + "SQBRPROJECT203", + "SQBRPROJECT204", + "SQBRDECLARATION303", + "SQBRDECLARATION304", + "SQBRDECLARATION306", +]; +pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARATION3"]; +pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; +pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; +pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; +pub(crate) const RULE_EXAMPLES: [(&str, &str); 25] = [ + ( + "SQBRSQL040", + "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", + ), + ( + "SQBRSQL041", + "WITH orders AS (...), final AS (SELECT order_id FROM orders) SELECT order_id FROM final.", + ), + ( + "SQBRMODEL101", + "WITH upstream AS (SELECT * FROM __ref(\"domain__stg__entity\")), ...", + ), + ( + "SQBRMODEL102", + "SELECT id, status FROM final; use a lone SELECT * only when allowed.", + ), + ( + "SQBRMODEL103", + "A view uses stg_v, int_v, or mart_v; a table uses a non-v layer.", + ), + ( + "SQBRMODEL104", + "MODEL() omits schema; central path defaults provide it.", + ), + ( + "SQBRGRAPH101", + "Internal flow is stg -> int_clean -> int_enriched; marts are published interfaces.", + ), + ( + "SQBRPROJECT101", + "domain__int_clean__entity or domain__mart_v__entity.", + ), + ( + "SQBRPROJECT105", + "A domain__int_enriched__entity model lives under intermediate/enriched.", + ), + ( + "SQBRPROJECT106", + "A domain__stg__entity model resolves to the staging logical schema.", + ), + ( + "SQBRCONTRACT101", + "MODEL (contract enforced, columns (...)); declare authoritative columns.", + ), + ( + "SQBRPROJECT202", + "domain/level/owner contains models or child owners, never both.", + ), + ( + "SQBRPROJECT201", + "models///model.sql resolves one domain root.", + ), + ( + "SQBRPROJECT203", + "domain/level/subdomain/model.sql at the default depth of one.", + ), + ( + "SQBRPROJECT204", + "order_status and order_status_history consolidate under order_status.", + ), + ( + "SQBRDECLARATION302", + "_macros/ is flat, or every file uses one concern bucket.", + ), + ( + "SQBRDECLARATION303", + "_macros/normalisation/names.py uses one default bucket level.", + ), + ( + "SQBRDECLARATION304", + "Keep each flat role or concern bucket within its file cap.", + ), + ( + "SQBRDECLARATION305", + "Use temporal/ or scoring/, not utils/ or common/.", + ), + ( + "SQBRDECLARATION306", + "Group normalise_order.py and normalise_person.py under normalise/.", + ), + ( + "SQBRTEST101", + "Keep unit tests in tests/unit/ and scenarios in tests/scenarios/.", + ), + ( + "SQBRTEST102", + "test_stg_orders__excludes_cancelled.sql and daily_revenue__minimal.sql.", + ), + ( + "SQBRTEST103", + "models/staging/stg_orders.sql maps to tests/unit/staging/.", + ), + ( + "SQBRTEST104", + "TEST (name \"stg_orders__excludes_cancelled_orders\");", + ), + ( + "SQBRTEST201", + "Attach not_null/unique audits to the model key.", + ), +]; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs new file mode 100644 index 0000000000..96b6b05ca4 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs @@ -0,0 +1,54 @@ +//! The inspection text of one rule. + +use crate::models::RulesConfig; +use crate::rule_catalogue::_helpers::thresholds::threshold_lines; +use crate::rule_catalogue::constants::{MINIMUM_RULES_CODES, TEST_RULE_PREFIX}; +use crate::rule_catalogue::models::RuleText; + +/// `sqb rules show` text, with effective thresholds and test paths when a config is given. +pub fn format_rule_text(rule: &RuleText, config: Option<&RulesConfig>) -> String { + let mut lines: Vec = vec![ + format!("{}: {}", rule.code, rule.slug), + format!("Family: {}", rule.family), + format!( + "Enabled by default: {}", + if rule.enabled_by_default { "yes" } else { "no" } + ), + format!("Kind: {}", if rule.custom { "custom" } else { "built-in" }), + String::new(), + rule.message.clone(), + String::new(), + format!("Remediation: {}", rule.remediation), + ]; + if let Some(source) = &rule.source { + lines.extend([String::new(), format!("Source: {source}")]); + } + if !rule.options.is_empty() { + lines.extend([String::new(), "Options:".into()]); + for option in &rule.options { + lines.push(format!( + "- {}: default={} ({})", + option.name, option.default_repr, option.description + )); + } + } + if let Some(config) = config { + if MINIMUM_RULES_CODES.contains(&rule.code.as_str()) { + lines.extend([String::new(), "Effective thresholds:".into()]); + lines.extend(threshold_lines(config)); + } + if rule.code.starts_with(TEST_RULE_PREFIX) { + lines.extend([ + String::new(), + "Effective SQL test paths:".into(), + "- Unit tests: tests/unit/".into(), + "- Scenarios: tests/scenarios/".into(), + format!( + "- Cross-domain pipelines: tests/unit/{}/", + config.sql_tests.pipeline_directory + ), + ]); + } + } + lines.join("\n") +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs new file mode 100644 index 0000000000..7ffa96add2 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs @@ -0,0 +1,29 @@ +//! Validate and atomically replace every generated skill target. + +use crate::errors::{SkillInstallError, SkillInstallErrorKind}; +use crate::rule_catalogue::_helpers::staging::{apply, stage}; +use crate::rule_catalogue::constants::REPLACEABLE_SKILL_STATES; +use crate::rule_catalogue::models::SkillTarget; + +/// Install `content` at every target, refusing foreign content and rolling back on failure. +pub fn install_rendered_skills( + content: &str, + targets: &[SkillTarget], +) -> Result<(), SkillInstallError> { + for target in targets { + if !REPLACEABLE_SKILL_STATES.contains(&target.state.as_str()) { + return Err(SkillInstallError { + kind: SkillInstallErrorKind::Refused, + message: format!( + "refusing to overwrite {} rules skill: {}", + target.state, + target.path.display() + ), + notes: Vec::new(), + }); + } + } + let paths: Vec = targets.iter().map(|target| target.path.clone()).collect(); + let staged = stage(&paths, content.as_bytes())?; + apply(&staged, content.as_bytes()) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs new file mode 100644 index 0000000000..618095dd7e --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -0,0 +1,6 @@ +pub mod format_rule_text; +pub mod install_rendered_skills; +pub mod render_skills_body; +pub mod ruleset_fingerprint; +pub mod skill_lock_name; +pub mod sorted_catalogue_order; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs new file mode 100644 index 0000000000..2a44c33eb3 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs @@ -0,0 +1,155 @@ +//! The generated rules skill body for the selected rules. + +use crate::models::RulesConfig; +use crate::rule_catalogue::_helpers::python_repr::dict_repr; +use crate::rule_catalogue::_helpers::thresholds::threshold_lines; +use crate::rule_catalogue::constants::{ + GRAPH_EDGE_RULE_CODE, LAYOUT_RULE_PREFIXES, LAYOUT_THRESHOLD_RULE_CODES, MINIMUM_RULES_CODES, + RULE_EXAMPLES, TEST_RULE_PREFIX, +}; +use crate::rule_catalogue::models::RuleText; + +/// Stable guidance text for `rules` under `config`, ending in one newline. +pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { + let mut body: Vec = [ + "# SQLBuild Rules", + "", + "Rules enforce SQL model architecture and never modifies SQL files.", + "Run `sqb rules run SQBR` before handing off model changes.", + "", + "## Active Rules", + ] + .map(String::from) + .to_vec(); + for rule in rules { + body.extend([ + String::new(), + format!("### {}: {}", rule.code, rule.slug), + String::new(), + rule.message.clone(), + String::new(), + format!("Correct shape: {}", rule_example(rule)), + String::new(), + format!("Remediation: {}", rule.remediation), + ]); + if !rule.options.is_empty() { + let effective: String = dict_repr( + rule.options + .iter() + .map(|option| (option.name.as_str(), option.effective_repr.clone())), + ); + body.extend([String::new(), format!("Effective options: {effective}")]); + } + } + if !config.rule_exceptions.is_empty() + || !config.graph_edge_exceptions.is_empty() + || !config.rule_ignores.is_empty() + || !config.select_star_allow.is_empty() + { + body.extend([String::new(), "## Scoped Deviations".into(), String::new()]); + for entry in &config.rule_exceptions { + body.push(format!( + "- Exception `{}` at `{}`: {}", + entry.rule, entry.path, entry.reason + )); + } + for entry in &config.rule_ignores { + let scopes: Vec = entry + .paths + .iter() + .map(|path| format!("path:{path}")) + .chain( + entry + .selectors + .iter() + .map(|selector| format!("selector:{selector}")), + ) + .collect(); + body.push(format!( + "- Ignore `{}` at `{}`: {}", + entry.rules.join(","), + scopes.join(","), + entry.reason + )); + } + for entry in &config.graph_edge_exceptions { + body.push(format!( + "- `{GRAPH_EDGE_RULE_CODE}` exact edge `{}` -> `{}`: {}", + entry.consumer, entry.dependency, entry.reason + )); + } + for entry in &config.select_star_allow { + body.push(format!( + "- Lone-star allowance `{}`: {}", + entry.paths.join(","), + entry.reason + )); + } + } + let any_rule = |matches: &dyn Fn(&str) -> bool| rules.iter().any(|rule| matches(&rule.code)); + if any_rule(&|code| { + MINIMUM_RULES_CODES.contains(&code) || LAYOUT_THRESHOLD_RULE_CODES.contains(&code) + }) { + body.extend([ + String::new(), + "## Effective Thresholds".into(), + String::new(), + ]); + body.extend(threshold_lines(config)); + } + if any_rule(&|code| { + LAYOUT_RULE_PREFIXES + .iter() + .any(|prefix| code.starts_with(prefix)) + }) { + body.extend([String::new(), "## Owner Layout".into(), String::new()]); + body.push(format!( + "- Configured levels: `{}`", + config.layout.levels.join(", ") + )); + body.push("- Every model owner is a leaf or a branch, never both.".into()); + body.push("- Declaration-role buckets organize files without changing visibility.".into()); + } + if any_rule(&|code| code.starts_with(TEST_RULE_PREFIX)) { + body.extend([String::new(), "## SQL Test Paths".into(), String::new()]); + body.push("- Unit tests: `tests/unit/`".into()); + body.push("- Scenarios: `tests/scenarios/`".into()); + body.push(format!( + "- Cross-domain pipelines: `tests/unit/{}/`", + config.sql_tests.pipeline_directory + )); + } + if !config.domains.is_empty() + || !config.approved_source_tokens.is_empty() + || !config.retired_source_tokens.is_empty() + { + body.extend([String::new(), "## Naming Vocabulary".into(), String::new()]); + body.push(format!("- Domains: `{}`", config.domains.join(", "))); + body.push(format!( + "- Approved source tokens: `{}`", + config.approved_source_tokens.join(", ") + )); + for (retired, replacement) in &config.retired_source_tokens { + body.push(format!( + "- Replace retired `{retired}` with `{replacement}`" + )); + } + } + format!("{}\n", body.join("\n").trim_end()) +} + +fn rule_example(rule: &RuleText) -> String { + if let Some(example) = &rule.good_example { + return example.clone(); + } + RULE_EXAMPLES + .iter() + .find(|(code, _)| *code == rule.code) + .map(|(_, example)| (*example).to_owned()) + .unwrap_or_else(|| { + format!( + "Apply `{}` at the SQL node named by the finding.", + rule.slug + ) + }) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs new file mode 100644 index 0000000000..4a20a55fe4 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs @@ -0,0 +1,55 @@ +//! The fingerprint of one resolved ruleset that generated guidance records. + +use std::collections::BTreeMap; + +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::models::RulesConfig; +use crate::rule_catalogue::models::FingerprintRule; + +/// SHA-256 over the engine build, the selected rules, their sources and the guidance config. +pub fn ruleset_fingerprint( + rules: &[FingerprintRule], + config: &RulesConfig, +) -> Result { + let mut payload: Vec = Vec::with_capacity(rules.len()); + for rule in rules { + let source_hash: Option = match &rule.source { + Some(source) => Some(format!( + "{:x}", + Sha256::digest(std::fs::read(source).map_err(|error| error.to_string())?) + )), + None => None, + }; + payload.push(json!({ + "code": rule.code, + "family": rule.family, + "slug": rule.slug, + "message": rule.message, + "remediation": rule.remediation, + "options": config.rule_options.get(&rule.code).cloned().unwrap_or_else(BTreeMap::new), + "source_hash": source_hash, + "implementation_hash": rule.implementation_hash, + })); + } + let rules_payload: Value = json!({ + "engine": NATIVE_BUILD_IDENTITY, + "rules": payload, + "thresholds": config.thresholds, + "threshold_overrides": config.threshold_overrides, + "select_star_allow": config.select_star_allow, + "domains": config.domains, + "approved_source_tokens": config.approved_source_tokens, + "retired_source_tokens": config.retired_source_tokens, + "sql_tests": config.sql_tests, + "layout": config.layout, + "exceptions": config.rule_exceptions, + "graph_edge_exceptions": config.graph_edge_exceptions, + "ignores": config.rule_ignores, + }); + let encoded: String = + serde_json::to_string(&rules_payload).map_err(|error| error.to_string())?; + Ok(format!("{:x}", Sha256::digest(encoded.as_bytes()))) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs new file mode 100644 index 0000000000..faa2aff130 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs @@ -0,0 +1,8 @@ +//! The per-project lock file name that serializes generated skill installation. + +use sha2::{Digest, Sha256}; + +/// `.lock`. +pub fn skill_lock_name(canonical_project: &str) -> String { + format!("{:x}.lock", Sha256::digest(canonical_project.as_bytes())) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs new file mode 100644 index 0000000000..c3cdd874af --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs @@ -0,0 +1,157 @@ +//! The code order of a validated rule catalogue and its configured options. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rule_catalogue::_helpers::options::{ + has_value_type, is_instance, plain_integer, python_equal, +}; +use crate::rule_catalogue::models::{CatalogueRule, OptionSpec, OptionValue}; + +/// Indices of `rules` sorted by code, or the first `RulesError` message the catalogue raises. +pub fn sorted_catalogue_order( + rules: &[CatalogueRule], + configured: &BTreeMap>, +) -> Result, String> { + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for rule in rules { + *counts.entry(rule.code.as_str()).or_default() += 1; + } + let duplicates: Vec<&str> = counts + .iter() + .filter(|(_, count)| **count > 1) + .map(|(code, _)| *code) + .collect(); + if !duplicates.is_empty() { + return Err(format!("duplicate rule codes: {}", duplicates.join(", "))); + } + let empty: BTreeMap = BTreeMap::new(); + for rule in rules { + validate_options(rule, configured.get(&rule.code).unwrap_or(&empty))?; + } + let unknown: Vec<&str> = configured + .keys() + .map(String::as_str) + .filter(|code| !counts.contains_key(code)) + .collect(); + if !unknown.is_empty() { + return Err(format!( + "rule options target unknown codes: {}", + unknown.join(", ") + )); + } + let mut order: Vec = (0..rules.len()).collect(); + order.sort_by(|left, right| rules[*left].code.cmp(&rules[*right].code)); + Ok(order) +} + +fn validate_options( + rule: &CatalogueRule, + configured: &BTreeMap, +) -> Result<(), String> { + let code: &str = &rule.code; + for option in &rule.options { + validate_declaration(code, option)?; + } + let declared: BTreeMap<&str, &OptionSpec> = rule + .options + .iter() + .map(|option| (option.name.as_str(), option)) + .collect(); + let unknown: BTreeSet<&str> = configured + .keys() + .map(String::as_str) + .filter(|name| !declared.contains_key(name)) + .collect(); + if !unknown.is_empty() { + let names: Vec<&str> = unknown.into_iter().collect(); + return Err(format!( + "rule {code} has unknown options: {}", + names.join(", ") + )); + } + for (name, value) in configured { + let Some(option) = declared.get(name.as_str()) else { + continue; + }; + if !has_value_type(value, &option.value_type) { + return Err(format!("rule {code} option {name} has the wrong type")); + } + if !option.choices.is_empty() && !option.choices.iter().any(|c| python_equal(c, value)) { + return Err(format!( + "rule {code} option {name} is not an allowed choice" + )); + } + if let OptionValue::Tuple(items) = value { + if option + .minimum_items + .is_some_and(|minimum| (items.len() as i64) < minimum) + { + return Err(format!("rule {code} option {name} has too few items")); + } + if let Some(item_type) = &option.item_type + && items.iter().any(|item| !is_instance(item, item_type)) + { + return Err(format!( + "rule {code} option {name} has item values of the wrong type" + )); + } + } + if let Some(integer) = plain_integer(value) { + if option.minimum.is_some_and(|minimum| integer < minimum) { + return Err(format!("rule {code} option {name} is below its minimum")); + } + if option.maximum.is_some_and(|maximum| integer > maximum) { + return Err(format!("rule {code} option {name} exceeds its maximum")); + } + } + } + Ok(()) +} + +fn validate_declaration(code: &str, option: &OptionSpec) -> Result<(), String> { + let name: &str = &option.name; + if option.name.trim().is_empty() || option.description.trim().is_empty() { + return Err(format!("rule {code} declares an incomplete option")); + } + let default: &OptionValue = &option.default; + if !has_value_type(default, &option.value_type) { + return Err(format!( + "rule {code} option {name} has a default of the wrong type" + )); + } + if !option.choices.is_empty() && !option.choices.iter().any(|c| python_equal(c, default)) { + return Err(format!( + "rule {code} option {name} default is not an allowed choice" + )); + } + if let OptionValue::Tuple(items) = default { + if option + .minimum_items + .is_some_and(|minimum| (items.len() as i64) < minimum) + { + return Err(format!( + "rule {code} option {name} default has too few items" + )); + } + if let Some(item_type) = &option.item_type + && items.iter().any(|item| !is_instance(item, item_type)) + { + return Err(format!( + "rule {code} option {name} default has items of the wrong type" + )); + } + } + if let Some(integer) = plain_integer(default) { + if option.minimum.is_some_and(|minimum| integer < minimum) { + return Err(format!( + "rule {code} option {name} default is below its minimum" + )); + } + if option.maximum.is_some_and(|maximum| integer > maximum) { + return Err(format!( + "rule {code} option {name} default exceeds its maximum" + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/mod.rs new file mode 100644 index 0000000000..297276f558 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/mod.rs @@ -0,0 +1,9 @@ +//! Rule catalogue validation, rule inspection text, generated guidance and its installation. + +pub(crate) mod _helpers; +mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/models.rs b/crates/sqlbuild-rules/src/rule_catalogue/models.rs new file mode 100644 index 0000000000..2ebd66fc1b --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/models.rs @@ -0,0 +1,87 @@ +//! Typed views of catalogue rules, their declared options and the rule text guidance renders. + +/// The Python type an option declares for its value or its tuple items. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OptionKind { + Bool, + Int, + Str, + Tuple, + /// Any other Python type, by name; `object` accepts every value. + Other(String), +} + +/// A declared or configured option value with Python's scalar and tuple distinctions. +#[derive(Clone, Debug)] +pub enum OptionValue { + Bool(bool), + Int(i64), + Str(String), + Tuple(Vec), + /// A value of another Python type, by type name. + Other(String), +} + +/// One option a rule declares. +#[derive(Clone, Debug)] +pub struct OptionSpec { + pub name: String, + pub description: String, + pub value_type: OptionKind, + pub default: OptionValue, + pub choices: Vec, + pub minimum: Option, + pub maximum: Option, + pub minimum_items: Option, + pub item_type: Option, +} + +/// The code and declared options of one catalogue rule. +#[derive(Clone, Debug)] +pub struct CatalogueRule { + pub code: String, + pub options: Vec, +} + +/// One declared option as inspection and guidance show it, with Python `repr` texts. +#[derive(Clone, Debug, Default)] +pub struct OptionText { + pub name: String, + pub description: String, + pub default_repr: String, + pub effective_repr: String, +} + +/// The metadata of one rule that inspection text and guidance render. +#[derive(Clone, Debug, Default)] +pub struct RuleText { + pub code: String, + pub family: String, + pub slug: String, + pub message: String, + pub remediation: String, + pub enabled_by_default: bool, + pub custom: bool, + pub source: Option, + pub options: Vec, + pub good_example: Option, +} + +/// The inputs of one selected rule that the ruleset fingerprint covers. +#[derive(Clone, Debug, Default)] +pub struct FingerprintRule { + pub code: String, + pub family: String, + pub slug: String, + pub message: String, + pub remediation: String, + pub source: Option, + pub implementation_hash: String, +} + +/// One generated skill target and the ownership state of its current content. +#[derive(Clone, Debug)] +pub struct SkillTarget { + pub path: std::path::PathBuf, + pub state: String, +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests.rs new file mode 100644 index 0000000000..949a29ce14 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_rule_catalogue.rs"] +mod rule_catalogue; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs new file mode 100644 index 0000000000..7af1f8c6df --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs @@ -0,0 +1,101 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use crate::models::{RulesConfig, ThresholdOverride}; +use crate::rule_catalogue::models::{ + CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleText, SkillTarget, +}; +use crate::rule_catalogue::tests::test_types::InstallTestCase; + +pub(crate) fn integer_option(default: i64) -> OptionSpec { + OptionSpec { + name: "limit".into(), + description: "The limit.".into(), + value_type: OptionKind::Int, + default: OptionValue::Int(default), + choices: vec![], + minimum: Some(1), + maximum: Some(5), + minimum_items: None, + item_type: None, + } +} + +pub(crate) fn catalogue_rule(code: &str, options: Vec) -> CatalogueRule { + CatalogueRule { + code: code.into(), + options, + } +} + +pub(crate) fn configured( + code: &str, + value: OptionValue, +) -> BTreeMap> { + BTreeMap::from([(code.into(), BTreeMap::from([("limit".into(), value)]))]) +} + +pub(crate) fn custom_rule() -> RuleText { + RuleText { + code: "XSQBRTEST201".into(), + family: "XSQBRTEST".into(), + slug: "custom-check".into(), + message: "Check it.".into(), + remediation: "Fix it.".into(), + custom: true, + source: Some("rules/check.py".into()), + options: vec![OptionText { + name: "names".into(), + description: "Names.".into(), + default_repr: "('a',)".into(), + effective_repr: "('b', \"it's\")".into(), + }], + ..RuleText::default() + } +} + +pub(crate) fn overridden_config() -> RulesConfig { + RulesConfig { + thresholds: BTreeMap::from([("min_tests_per_model".into(), 3)]), + threshold_overrides: vec![ThresholdOverride { + paths: vec!["models/a".into()], + thresholds: BTreeMap::from([("min_audits_per_model".into(), 2)]), + reason: "Critical.".into(), + }], + ..RulesConfig::default() + } +} + +/// Two skill targets under `root`, seeded with the case's existing content. +pub(crate) fn skill_targets(root: &Path, test_case: &InstallTestCase) -> [SkillTarget; 2] { + let targets = [root.join("a/SKILL.md"), root.join("b/SKILL.md")].map(|path| SkillTarget { + path, + state: test_case.state.into(), + }); + std::fs::create_dir_all(root.join("a")).expect("a first skill folder"); + let _ = test_case + .existing + .map(|existing| std::fs::write(&targets[0].path, existing).expect("a first skill")); + let _ = test_case + .blocked_second + .then(|| std::fs::write(root.join("b"), "").expect("a blocking file")); + let _ = test_case + .existing + .filter(|_| !test_case.blocked_second) + .map(|existing| { + std::fs::create_dir_all(root.join("b")).expect("a second skill folder"); + std::fs::write(&targets[1].path, existing).expect("a second skill") + }); + targets +} + +/// The number of staged temporary or backup files left beside the targets. +pub(crate) fn staged_leftovers(root: &Path) -> usize { + ["a", "b"] + .iter() + .filter_map(|folder| std::fs::read_dir(root.join(folder)).ok()) + .flatten() + .flatten() + .filter(|entry| entry.file_name().to_string_lossy().starts_with('.')) + .count() +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs new file mode 100644 index 0000000000..69a6f4e799 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs @@ -0,0 +1,217 @@ +use std::collections::BTreeMap; + +use crate::models::RulesConfig; +use crate::rule_catalogue::main::format_rule_text::format_rule_text; +use crate::rule_catalogue::main::install_rendered_skills::install_rendered_skills; +use crate::rule_catalogue::main::render_skills_body::render_skills_body; +use crate::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use crate::rule_catalogue::models::{OptionValue, RuleText, SkillTarget}; +use crate::rule_catalogue::tests::helpers::{ + catalogue_rule, configured, custom_rule, integer_option, overridden_config, skill_targets, + staged_leftovers, +}; +use crate::rule_catalogue::tests::test_types::{ + CatalogueTestCase, InstallTestCase, RuleTextTestCase, SkillsBodyTestCase, +}; + +#[test] +fn given_catalogue_when_validating_then_orders_or_rejects_like_python() { + let test_cases = [ + CatalogueTestCase { + description: "valid rules sort by code", + rules: vec![ + catalogue_rule("XSQBRB", vec![]), + catalogue_rule("SQBRA", vec![]), + ], + configured: BTreeMap::new(), + expected_result: Ok(vec![1, 0]), + }, + CatalogueTestCase { + description: "duplicate codes are rejected", + rules: vec![ + catalogue_rule("SQBRA", vec![]), + catalogue_rule("SQBRA", vec![]), + ], + configured: BTreeMap::new(), + expected_result: Err("duplicate rule codes: SQBRA"), + }, + CatalogueTestCase { + description: "a bool is not an integer option value", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(2)])], + configured: configured("XSQBRA", OptionValue::Bool(true)), + expected_result: Err("rule XSQBRA option limit has the wrong type"), + }, + CatalogueTestCase { + description: "configured values respect the maximum", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(2)])], + configured: configured("XSQBRA", OptionValue::Int(9)), + expected_result: Err("rule XSQBRA option limit exceeds its maximum"), + }, + CatalogueTestCase { + description: "defaults respect the minimum", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(0)])], + configured: BTreeMap::new(), + expected_result: Err("rule XSQBRA option limit default is below its minimum"), + }, + CatalogueTestCase { + description: "options for unknown codes are rejected", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(2)])], + configured: configured("XSQBRZ", OptionValue::Int(2)), + expected_result: Err("rule options target unknown codes: XSQBRZ"), + }, + ]; + for test_case in test_cases { + let result = sorted_catalogue_order(&test_case.rules, &test_case.configured); + assert_eq!( + result, + test_case.expected_result.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rule_when_formatting_then_matches_python_inspection_text() { + let test_cases = [ + RuleTextTestCase { + description: "a custom rule with options and no config", + rule: custom_rule(), + with_config: false, + expected_text: "XSQBRTEST201: custom-check\nFamily: XSQBRTEST\nEnabled by default: no\nKind: custom\n\nCheck it.\n\nRemediation: Fix it.\n\nSource: rules/check.py\n\nOptions:\n- names: default=('a',) (Names.)", + }, + RuleTextTestCase { + description: "a test rule with config shows test paths only", + rule: RuleText { + code: "SQBRTEST101".into(), + enabled_by_default: true, + ..RuleText::default() + }, + with_config: true, + expected_text: "SQBRTEST101: \nFamily: \nEnabled by default: yes\nKind: built-in\n\n\n\nRemediation: \n\nEffective SQL test paths:\n- Unit tests: tests/unit/\n- Scenarios: tests/scenarios/\n- Cross-domain pipelines: tests/unit/pipelines/", + }, + RuleTextTestCase { + description: "a minimum rule shows effective thresholds and overrides", + rule: RuleText { + code: "SQBRTEST202".into(), + ..RuleText::default() + }, + with_config: true, + expected_text: "SQBRTEST202: \nFamily: \nEnabled by default: no\nKind: built-in\n\n\n\nRemediation: \n\nEffective thresholds:\n- `max_constant_container_files` = `10`\n- `max_enum_container_files` = `10`\n- `max_macro_container_files` = `10`\n- `max_role_container_depth` = `1`\n- `max_subdomain_depth` = `1`\n- `min_audits_per_model` = `1`\n- `min_custom_rule_test_cases` = `1`\n- `min_shared_container_prefix_files` = `2`\n- `min_shared_owner_prefix_directories` = `2`\n- `min_tests_per_model` = `3`\n- Override 1 at `models/a`: `{'min_audits_per_model': 2}` (Critical.)\n- Matching overrides apply in authored order; the last match wins.\n\nEffective SQL test paths:\n- Unit tests: tests/unit/\n- Scenarios: tests/scenarios/\n- Cross-domain pipelines: tests/unit/pipelines/", + }, + ]; + let config: RulesConfig = overridden_config(); + for test_case in test_cases { + let text: String = + format_rule_text(&test_case.rule, test_case.with_config.then_some(&config)); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} + +#[test] +fn given_rules_when_rendering_skills_then_lists_rules_and_sections() { + let test_cases = [ + SkillsBodyTestCase { + description: "custom options and test paths", + rules: vec![custom_rule()], + expected_lines: vec![ + "### XSQBRTEST201: custom-check", + "Correct shape: Apply `custom-check` at the SQL node named by the finding.", + "Effective options: {'names': ('b', \"it's\")}", + "## Active Rules", + ], + }, + SkillsBodyTestCase { + description: "a built-in example and the layout section", + rules: vec![RuleText { + code: "SQBRPROJECT203".into(), + slug: "depth".into(), + ..RuleText::default() + }], + expected_lines: vec![ + "Correct shape: domain/level/subdomain/model.sql at the default depth of one.", + "## Effective Thresholds", + "## Owner Layout", + "- Configured levels: `staging, intermediate/clean, intermediate/enriched, mart`", + ], + }, + ]; + for test_case in test_cases { + let body: String = render_skills_body(&test_case.rules, &RulesConfig::default()); + let lines: Vec<&str> = body.lines().collect(); + let found: Vec<&str> = test_case + .expected_lines + .iter() + .copied() + .filter(|line| lines.contains(line)) + .collect(); + assert_eq!(found, test_case.expected_lines, "{}", test_case.description); + } +} + +#[test] +fn given_skill_targets_when_installing_then_replaces_only_owned_content() { + let test_cases = [ + InstallTestCase { + description: "missing targets are created", + existing: None, + state: "missing", + blocked_second: false, + expected_error: None, + expected_contents: [Some("new"), Some("new")], + }, + InstallTestCase { + description: "stale targets are replaced", + existing: Some("old"), + state: "stale", + blocked_second: false, + expected_error: None, + expected_contents: [Some("new"), Some("new")], + }, + InstallTestCase { + description: "foreign content is refused and kept", + existing: Some("mine"), + state: "foreign", + blocked_second: false, + expected_error: Some("refusing to overwrite foreign rules skill: "), + expected_contents: [Some("mine"), Some("mine")], + }, + InstallTestCase { + description: "a later staging failure replaces no target", + existing: Some("old"), + state: "stale", + blocked_second: true, + expected_error: Some(""), + expected_contents: [Some("old"), None], + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("a temporary folder"); + let targets: [SkillTarget; 2] = skill_targets(directory.path(), &test_case); + let error: Option = install_rendered_skills("new", &targets) + .err() + .map(|error| error.message); + assert_eq!( + error.map(|message| message.starts_with(test_case.expected_error.unwrap_or("-"))), + test_case.expected_error.map(|_| true), + "{}", + test_case.description + ); + let contents: Vec> = targets + .iter() + .map(|target| std::fs::read_to_string(&target.path).ok()) + .collect(); + let expected: Vec> = test_case + .expected_contents + .iter() + .map(|content| content.map(str::to_owned)) + .collect(); + assert_eq!(contents, expected, "{}", test_case.description); + assert_eq!( + staged_leftovers(directory.path()), + 0, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs new file mode 100644 index 0000000000..68a40bc6cf --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs @@ -0,0 +1,32 @@ +use std::collections::BTreeMap; + +use crate::rule_catalogue::models::{CatalogueRule, OptionValue, RuleText}; + +pub(crate) struct CatalogueTestCase { + pub(crate) description: &'static str, + pub(crate) rules: Vec, + pub(crate) configured: BTreeMap>, + pub(crate) expected_result: Result, &'static str>, +} + +pub(crate) struct RuleTextTestCase { + pub(crate) description: &'static str, + pub(crate) rule: RuleText, + pub(crate) with_config: bool, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct SkillsBodyTestCase { + pub(crate) description: &'static str, + pub(crate) rules: Vec, + pub(crate) expected_lines: Vec<&'static str>, +} + +pub(crate) struct InstallTestCase { + pub(crate) description: &'static str, + pub(crate) existing: Option<&'static str>, + pub(crate) state: &'static str, + pub(crate) blocked_second: bool, + pub(crate) expected_error: Option<&'static str>, + pub(crate) expected_contents: [Option<&'static str>; 2], +} diff --git a/crates/sqlbuild-rules/src/rule_results/constants.rs b/crates/sqlbuild-rules/src/rule_results/constants.rs index e2bcd24e2a..55a1a8d737 100644 --- a/crates/sqlbuild-rules/src/rule_results/constants.rs +++ b/crates/sqlbuild-rules/src/rule_results/constants.rs @@ -13,3 +13,4 @@ pub(crate) const FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; pub(crate) const JSON_INDENT: usize = 2; pub(crate) const CURRENT_DIRECTORY: &str = "."; +pub(crate) const UNEVALUATED_CODE: &str = "rules-unevaluated"; diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs b/crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs new file mode 100644 index 0000000000..5eda51af7b --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs @@ -0,0 +1,20 @@ +//! The rules phase completion line. + +use crate::rule_results::main::format_skipped_rules::format_skipped_rules; + +/// The rules phase wall time first, then built-in and custom rule time, with the skipped note. +pub fn format_rules_progress( + elapsed_seconds: f64, + (built_in_ms, custom_ms): (i64, i64), + skipped_rules: Option<&[String]>, +) -> String { + let message: String = format!( + "Evaluated rules. ({elapsed_seconds:.2}s; built-in {:.2}s, custom {:.2}s)", + built_in_ms as f64 / 1000.0, + custom_ms as f64 / 1000.0 + ); + match skipped_rules.and_then(format_skipped_rules) { + Some(note) => format!("{message}; {note}"), + None => message, + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs new file mode 100644 index 0000000000..81490cd695 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs @@ -0,0 +1,12 @@ +//! The note naming type-proof rules skipped because SQL analysis is disabled. + +/// The skipped type-proof rules note, or `None` when no rule was skipped. +pub fn format_skipped_rules(codes: &[String]) -> Option { + if codes.is_empty() { + return None; + } + Some(format!( + "skipped type-proof rules ({}) because SQL analysis is disabled", + codes.join(", ") + )) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs b/crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs new file mode 100644 index 0000000000..129a372513 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs @@ -0,0 +1,60 @@ +//! Unevaluated findings grouped per resource and reason, in deterministic finding order. + +use crate::rule_results::constants::UNEVALUATED_CODE; +use crate::rule_results::models::RuleFinding; + +/// One `rules-unevaluated` finding per path and reason, then every finding sorted by location. +pub fn group_unevaluated(findings: Vec) -> Vec { + let mut result: Vec = Vec::with_capacity(findings.len()); + let mut groups: Vec<((String, String), Vec)> = Vec::new(); + for finding in findings { + if !finding.unevaluated { + result.push(finding); + continue; + } + let prefix: String = format!("Rule {} could not be evaluated", finding.code); + let reason: String = finding + .message + .strip_prefix(&prefix) + .unwrap_or(&finding.message) + .trim_start_matches([':', ' ']) + .to_owned(); + let key: (String, String) = (finding.path.clone(), reason); + match groups.iter_mut().find(|(existing, _)| *existing == key) { + Some((_, failures)) => failures.push(finding), + None => groups.push((key, vec![finding])), + } + } + for ((_, reason), failures) in groups { + let mut affected: Vec = failures.iter().map(|f| f.code.clone()).collect(); + affected.sort(); + affected.dedup(); + let Some(first) = failures.into_iter().reduce(|best, item| { + if (item.line, item.column) < (best.line, best.column) { + item + } else { + best + } + }) else { + continue; + }; + result.push(RuleFinding { + code: UNEVALUATED_CODE.into(), + message: format!( + "{} selected Rules could not evaluate this resource: {reason}", + affected.len() + ), + affected_rules: affected, + ..first + }); + } + result.sort_by(|left, right| { + (&left.path, left.line, left.column, &left.code).cmp(&( + &right.path, + right.line, + right.column, + &right.code, + )) + }); + result +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/mod.rs b/crates/sqlbuild-rules/src/rule_results/main/mod.rs index b7461bbc4e..a361ef467b 100644 --- a/crates/sqlbuild-rules/src/rule_results/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_results/main/mod.rs @@ -1,2 +1,5 @@ pub mod duplicate_literal_hints; pub mod format_result; +pub mod format_rules_progress; +pub mod format_skipped_rules; +pub mod group_unevaluated; diff --git a/crates/sqlbuild-rules/src/rule_results/tests.rs b/crates/sqlbuild-rules/src/rule_results/tests.rs index d63fc626d1..b007d17f4e 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_rule_results.rs"] mod rule_results; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules/src/rule_results/tests/helpers.rs b/crates/sqlbuild-rules/src/rule_results/tests/helpers.rs new file mode 100644 index 0000000000..e251d603fa --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/tests/helpers.rs @@ -0,0 +1,14 @@ +use crate::rule_results::models::RuleFinding; + +/// One unevaluated finding of `code` in `models/a.sql` at `line`. +pub(crate) fn unevaluated(code: &str, line: i64) -> RuleFinding { + RuleFinding { + code: code.into(), + path: "models/a.sql".into(), + line, + column: 1, + message: format!("Rule {code} could not be evaluated: too deep"), + unevaluated: true, + ..RuleFinding::default() + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs index cc70c1672c..97038120ff 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs @@ -2,8 +2,13 @@ use std::convert::Infallible; use crate::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; use crate::rule_results::main::format_result::format_result; +use crate::rule_results::main::format_rules_progress::format_rules_progress; +use crate::rule_results::main::group_unevaluated::group_unevaluated; use crate::rule_results::models::{ResultCounts, RuleFinding}; -use crate::rule_results::tests::test_types::{HintTestCase, ResultTestCase}; +use crate::rule_results::tests::helpers::unevaluated; +use crate::rule_results::tests::test_types::{ + GroupTestCase, HintTestCase, ProgressTestCase, ResultTestCase, +}; const SUFFIX: &str = " and reference it with `@const` in each file. R."; @@ -124,3 +129,81 @@ fn given_rules_result_when_formatting_then_matches_python_text_and_json() { ); } } + +#[test] +fn given_rules_run_when_formatting_progress_then_reports_phase_wall_time_first() { + let test_cases = [ + ProgressTestCase { + description: "wall time leads concurrent built-in and custom rule time", + elapsed_seconds: 14.5, + timings_ms: (5_600, 13_400), + skipped_rules: None, + expected_message: "Evaluated rules. (14.50s; built-in 5.60s, custom 13.40s)", + }, + ProgressTestCase { + description: "skipped type-proof rules follow the timings when requested", + elapsed_seconds: 0.25, + timings_ms: (100, 0), + skipped_rules: Some(vec!["SQBRCONTRACT105".into()]), + expected_message: "Evaluated rules. (0.25s; built-in 0.10s, custom 0.00s); skipped type-proof rules (SQBRCONTRACT105) because SQL analysis is disabled", + }, + ProgressTestCase { + description: "no skipped rules add no note", + elapsed_seconds: 1.0, + timings_ms: (0, 0), + skipped_rules: Some(vec![]), + expected_message: "Evaluated rules. (1.00s; built-in 0.00s, custom 0.00s)", + }, + ]; + for test_case in test_cases { + let message: String = format_rules_progress( + test_case.elapsed_seconds, + test_case.timings_ms, + test_case.skipped_rules.as_deref(), + ); + assert_eq!( + message, test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unevaluated_findings_when_grouping_then_one_finding_per_path_and_reason() { + let test_cases = [GroupTestCase { + description: "two rules failing for one reason group into one sorted finding", + findings: vec![ + unevaluated("SQBRB", 3), + RuleFinding { + code: "SQBRZ".into(), + path: "models/a.sql".into(), + line: 1, + ..RuleFinding::default() + }, + unevaluated("SQBRA", 2), + ], + expected_codes: vec!["SQBRZ", "rules-unevaluated"], + expected_messages: vec![ + "", + "2 selected Rules could not evaluate this resource: too deep", + ], + }]; + for test_case in test_cases { + let grouped: Vec = group_unevaluated(test_case.findings); + let codes: Vec<&str> = grouped + .iter() + .map(|finding| finding.code.as_str()) + .collect(); + let messages: Vec<&str> = grouped + .iter() + .map(|finding| finding.message.as_str()) + .collect(); + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs index 97f61de3cc..7d1a7ce23d 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs @@ -1,3 +1,5 @@ +use crate::rule_results::models::RuleFinding; + pub(crate) struct HintTestCase { pub(crate) description: &'static str, pub(crate) paths: Vec<&'static str>, @@ -9,3 +11,18 @@ pub(crate) struct ResultTestCase { pub(crate) json_output: bool, pub(crate) expected_text: &'static str, } + +pub(crate) struct ProgressTestCase { + pub(crate) description: &'static str, + pub(crate) elapsed_seconds: f64, + pub(crate) timings_ms: (i64, i64), + pub(crate) skipped_rules: Option>, + pub(crate) expected_message: &'static str, +} + +pub(crate) struct GroupTestCase { + pub(crate) description: &'static str, + pub(crate) findings: Vec, + pub(crate) expected_codes: Vec<&'static str>, + pub(crate) expected_messages: Vec<&'static str>, +} diff --git a/fensu.toml b/fensu.toml index 918a9a4fd8..f0f89d3c32 100644 --- a/fensu.toml +++ b/fensu.toml @@ -33,6 +33,11 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/constants.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cli/compile_reuse/__init__.py" +reason = "Native bindings own the compile reuse check, replay and launch records; this package declares the reuse models and constants and keeps the provider settings, runtime identity and startup inputs the native code reads from the interpreter." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cursor_algebra/__init__.py" diff --git a/python-boundary.toml b/python-boundary.toml index 503fd8bbd2..852366b9f6 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 51236 +remaining_lines = 42632 [[stays]] path = "executor" @@ -124,6 +124,11 @@ path = "rule_engine/_helpers/engine/custom_payloads.py" reason = "user-code" note = "Reads user rule functions (source lines, qualified names, import closures) to describe custom rules for native selection." +[[stays]] +path = "rule_engine/_helpers/engine/_custom_catalogue.py" +reason = "user-code" +note = "Imports the project's custom rule modules and collects their @rule functions for the native catalogue." + [[stays]] path = "rule_engine/_helpers/engine/custom_rules.py" reason = "user-code" @@ -299,6 +304,71 @@ path = "integrations/dbt/main/runtime/_execute_json_event_stream.py" reason = "user-code" note = "Entry point of the dbt subprocess runner." +[[stays]] +path = "integrations/dbt/classes/dbt_runner.py" +reason = "user-code" +note = "Runs dbt compile, deps, debug and ls as subprocesses over the user's dbt project." + +[[stays]] +path = "integrations/dbt/_helpers/profile/load.py" +reason = "user-code" +note = "Loads the user's dbt_project.yml and profiles.yml with PyYAML." + +[[stays]] +path = "integrations/dbt/_helpers/profile/render.py" +reason = "user-code" +note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does." + +[[stays]] +path = "cli/commands/main/entrypoint/_dispatch_with_compute_logs.py" +reason = "user-code" +note = "Captures the command's stdout, stderr and logging into the local compute log around user code." + +[[stays]] +path = "cli/commands/main/entrypoint/_dispatch_with_output_capture.py" +reason = "user-code" +note = "Tees the command's Python output streams to configured output capture sinks." + +[[stays]] +path = "cli/commands/_helpers/entry/compute_log_diagnostics.py" +reason = "user-code" +note = "Logs compute log capture failures through Python logging during capture." + +[[stays]] +path = "integrations/dbt/_helpers/profile/init.py" +reason = "user-code" +note = "Initializes a SQLBuild project from the user's dbt project and runs dbt debug as a subprocess." + +[[stays]] +path = "cli/compile_reuse/_helpers/provider_settings.py" +reason = "user-code" +note = "Inspects user provider settings classes for the environment and files they read." + +[[stays]] +path = "cli/compile_reuse/_helpers/runtime_identity.py" +reason = "user-code" +note = "Interpreter and invocation facts of the Python host that user code runs in." + +[[stays]] +path = "cli/compile_reuse/_helpers/startup_inputs.py" +reason = "user-code" +note = "Interpreter and site-path inputs user modules import from." + +[[stays]] +path = "cli/entry/main/entry.py" +reason = "public-api" +note = "The installed sqlbuild-python console entry point." + +[[stays]] +path = "cli/entry/main/launched.py" +reason = "public-api" +note = "The Python entry point the native sqb executable hands invocations to." + +[[stays]] +path = "cli/entry/_helpers/streams.py" +reason = "public-api" +note = "Reconfigures the Python process's standard streams for the console entry points." + [[stays]] path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" diff --git a/scripts/cli_preview/_helpers/workflow.py b/scripts/cli_preview/_helpers/workflow.py index b4a678c918..47c4d61aa0 100644 --- a/scripts/cli_preview/_helpers/workflow.py +++ b/scripts/cli_preview/_helpers/workflow.py @@ -13,7 +13,7 @@ from scripts.cli_preview.exceptions import PreviewSetupError from scripts.cli_preview.models import PreviewScene -from sqlbuild.presentation.main.supports_color import supports_color +from sqlbuild._native import supports_color _PAYMENTS_MODEL_PATH: Path = Path("models/staging/stg_payments.sql") _QUERY_MARKER: str = " amount_cents,\n" diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index a76454d163..7fe7dd708c 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -7,8 +7,7 @@ from scripts.cold_compile_performance._helpers.dense_project import dense_model_name from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine.main.load_config import load_rules_config +from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( @@ -51,8 +50,10 @@ def run_varied_cache_benchmark( def _assert_complete_rules(project_dir: Path) -> None: config: RulesConfig = load_rules_config(project_dir=project_dir) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) - selected: tuple[Rule, ...] = select_rules( + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=True + ) + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=config, project_dir=project_dir ) builtin_codes: set[str] = {rule.code for rule in filterfalse(attrgetter("custom"), catalogue)} diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 0c241bde84..e528e91df7 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -46,9 +46,6 @@ "src/sqlbuild/integrations/dbt/_helpers/planning/model_planning.py", } ) -ALLOWED_SELECTOR_PARSE_PATH: str = ( - "src/sqlbuild/compiler/planner/main/selection/selector_expansion.py" -) CLIENT_STYLE_PREFIXES: frozenset[tuple[str, ...]] = frozenset( {("src", "sqlbuild", "adapters"), ("src", "sqlbuild", "integrations")} ) @@ -192,7 +189,6 @@ PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) PUBLIC_COLOR_ENTRY_PARTS: frozenset[tuple[str, ...]] = frozenset( { - ("src", "sqlbuild", "presentation", "main", "supports_color.py"), ("src", "sqlbuild", "presentation", "main", "terminal_columns.py"), } ) diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index 69bc123b1f..8968f85d6d 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -11,7 +11,6 @@ from scripts.fensu_policy.constants import ( ALLOWED_DBT_REF_SCAN_PATHS, ALLOWED_MACRO_LOAD_PATHS, - ALLOWED_SELECTOR_PARSE_PATH, ALLOWED_SOURCE_FRESHNESS_INSERT_PREFIXES, DBT_INTEGRATION_PATH_PREFIX, GRAPH_KEY_CLASS_NAMES, @@ -31,8 +30,8 @@ code="XSB041", family=Family.CUSTOM, slug="color-capability-entry", - message="color capability imports must use the presentation main entry", - remediation="Import supports_color from sqlbuild.presentation.main.supports_color.", + message="terminal capability imports must use the presentation main entry", + remediation="Import terminal_columns from sqlbuild.presentation.main.terminal_columns.", ) def color_capability_entry(*, module: object, ctx: RuleContext) -> list[Fault]: del module @@ -97,15 +96,12 @@ def dbt_graph_projection(*, module: object, ctx: RuleContext) -> list[Fault]: family=Family.CUSTOM, slug="selector-marker-parsing", message="selector + markers must be parsed by split_selector_expansion", - remediation="Use compiler.planner.main.selection.selector_expansion.split_selector_expansion.", + remediation="Use sqlbuild._native.split_selector_expansion.", ) def selector_marker_parsing(*, module: object, ctx: RuleContext) -> list[Fault]: del module path: str = "/".join(ctx.repo_relative_parts()) - if ( - not (DBT_INTEGRATION_PATH_PREFIX in path or PLANNER_PATH_PREFIX in path) - or path == ALLOWED_SELECTOR_PARSE_PATH - ): + if not (DBT_INTEGRATION_PATH_PREFIX in path or PLANNER_PATH_PREFIX in path): return [] return [ ctx.fault_at(location=call.location) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b2116bd81e..9ef86c388e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -36,27 +36,31 @@ from sqlbuild.cli.commands.models import ( JanitorCommandRequest, LineageCommandRequest, LoadCommandRequest, + NestedProgressChildRow, + PlanCommandRequest, PlanSafetyGates, PlaygroundCommandRequest, QueryDiffPreparation, RefactorCommandRequest, + RulesCommandRequest, ScenarioCaptureCommandRequest, ScenarioTestCommandRequest, ScopeCommandRequest, SeedCommandRequest, TestCommandRequest, ) -from sqlbuild.cli.commands.types import CompileLineageMode, ScopeIndexLoader +from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, + CommandWarehouseScope, CompileCommandRequest, CompileProfileFlags, + ParsedCliInvocation, ResolvedWarehouse, ) from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget -from sqlbuild.cli.progress.models import NestedProgressChildRow from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, MeasurementContract, @@ -193,14 +197,19 @@ from sqlbuild.compiler.pipeline.models import ( from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChangeDetectionResult, CursorBounds, DeferralInputs, + DirectModelVersionIdentities, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, + MissingUpstream, ModelPlanEntry, + ParsedSelector, + PathSelector, PlannerChangeResults, PlannerResolvedActions, PlannerRuntime, @@ -210,7 +219,13 @@ from sqlbuild.compiler.planner.models import ( ReferenceRename, SchemaAction, SchemaFinding, + SelectionStalenessGraph, + SelectionStalenessWarning, + SelectorExpansion, + SourceLoadPlanEntry, SqlTestPlanEntry, + SqlTestSelection, + UnitTestSelectorSplit, WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( @@ -257,7 +272,11 @@ from sqlbuild.compiler.scopes.models import ( VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, JsonValue, ScopeQueryTarget -from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord +from sqlbuild.compiler.source_freshness.models import ( + DirectSourceFreshnessPlanningResult, + SourceFreshnessIdentity, + SourceFreshnessRecord, +) from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, SourceFreshnessComparableRecord, @@ -322,10 +341,21 @@ from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult -from sqlbuild.rule_engine.types import RuleCheck +from sqlbuild.rule_engine.models import ( + Finding, + NativeRulesEvaluation, + PreparedSqlLint, + Rule, + RuleRunPlan, + RulesConfig, + RulesResult, + RulesRunResult, + SqlExpansionReuse, + SqlRulesEvaluation, +) from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.contracts.types import ExecutionResourceKind +from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( DbtConfig, @@ -384,7 +414,6 @@ class LintPreparationRequest(TypedDict): dialect: str def analyze_column_references_json(request_json: str) -> str: ... -def builtin_rules(*, check: RuleCheck) -> list[Rule]: ... def selected_sql_rule_codes(*, select: list[str], ignore: list[str]) -> list[str]: ... class NativeRulesRequest: @@ -393,14 +422,16 @@ class NativeRulesRequest: def sql_facts_json(self) -> str: ... def build_rules_request( - project: dict[str, object], - models: tuple[NativeCompiledProject, list[str]], - sql_tests: list[CompiledSqlTest], - scenarios: list[CompiledSqlScenario], + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + dialect: str, + initial_findings: tuple[Finding, ...], + custom_payloads: list[dict[str, object]], + include_type_proof: bool, ) -> NativeRulesRequest: ... -def evaluate_rules_request( - request: NativeRulesRequest, -) -> tuple[list[Finding], list[str], int, int, int, int, bool]: ... +def evaluate_rules_request(request: NativeRulesRequest) -> NativeRulesEvaluation: ... def emit_orjson_report(report: object) -> str: ... def emit_json_report(report: object) -> str: ... @@ -1203,10 +1234,68 @@ class NativeProjectGraph: self, selected: list[_GraphKey], include: tuple[bool, bool, bool] ) -> list[_GraphKey]: ... -def parse_project_selector( - raw: str, -) -> tuple[tuple[str, str, str, bool, bool] | None, _SelectorFailure | None]: ... -def project_selector_name_help(value: str, candidates: list[str]) -> str | None: ... +def parse_project_selector(raw: str) -> ParsedSelector | PathSelector: ... +def resolve_graph_selection( + *, graph: NativeProjectGraph, select: Sequence[str], exclude: Sequence[str] +) -> frozenset[CompiledObjectKey]: ... +def select_project_graph( + *, graph: ProjectGraph, select: Sequence[str], exclude: Sequence[str] +) -> frozenset[CompiledObjectKey]: ... +def resolve_project_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + all_keys: Mapping[str, CompiledObjectKey], + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> frozenset[CompiledObjectKey]: ... +def resolve_project_selector_tokens( + *, + selectors: Sequence[str], + all_keys: Mapping[str, CompiledObjectKey], + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> frozenset[CompiledObjectKey]: ... +def match_project_selector( + *, + parsed: ParsedSelector, + all_keys: Mapping[str, CompiledObjectKey], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> frozenset[CompiledObjectKey]: ... +def expand_project_upstream_keys( + *, + key: CompiledObjectKey, + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> frozenset[CompiledObjectKey]: ... +def unit_test_selector_error(*, selector: str) -> PlannerInputError: ... +def split_project_unit_test_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + discovered_inputs: DiscoveredProjectInputs, + accepts_unit_tests: bool, +) -> UnitTestSelectorSplit: ... +def split_unit_test_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + discovered_inputs: DiscoveredProjectInputs, +) -> UnitTestSelectorSplit: ... +def reject_unit_test_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + discovered_inputs: DiscoveredProjectInputs, +) -> UnitTestSelectorSplit: ... +def split_selector_expansion(raw: str) -> SelectorExpansion: ... +def resolve_sqlbuild_model_selector_names( + *, project: CompiledProjectView, term: str +) -> tuple[tuple[str, ...], str | None]: ... class NativeReuseAttempt: def outcome(self) -> str: ... @@ -1220,6 +1309,7 @@ class NativeReuseAttempt: def record(self, record: dict[str, object]) -> tuple[str, int]: ... def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... +def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ... class CompileArtifactWrites: @property @@ -1724,6 +1814,12 @@ def execute_query_diff( *, request: DiffCommandRequest, preparation: QueryDiffPreparation ) -> DiffExecutionResult: ... def machine_output_mode(args: CliNamespace, /) -> bool: ... + +class CliObservabilityScope: + def __enter__(self) -> EventDispatcher: ... + def __exit__(self, *exc_info: object) -> bool: ... + +def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> CliObservabilityScope: ... def current_native_progress_projector() -> NativeProgressProjector | None: ... class ConnectionProgressReporter: @@ -1950,7 +2046,13 @@ class PreparedCompileArtifacts: ) -> WrittenTarget | None: ... def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: ... -def finalize_rule_findings_rows(request: dict[str, object]) -> list[Finding]: ... +def finalize_rule_findings_rows( + *, + config: RulesConfig, + project_dir: Path, + evaluated_codes: Sequence[str], + findings: Sequence[Finding], +) -> tuple[Finding, ...]: ... def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... @@ -1992,12 +2094,26 @@ def apply_lint_suppressions( allow_suppressions: bool, ) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... +def valid_rules_config(*, project_dir: Path) -> RulesConfig | None: ... def selected_rule_codes( *, config: RulesConfig, custom_payloads: list[dict[str, object]] ) -> list[str]: ... def rule_selected(*, config: RulesConfig, code: str) -> bool: ... -def render_owned_skill(*, content: str, input_fingerprint: str) -> str: ... -def skill_freshness(*, content: str | None, input_fingerprint: str) -> str: ... +def build_rule_catalogue( + *, config: RulesConfig, project_dir: Path, include_custom: bool | None = None +) -> tuple[Rule, ...]: ... +def select_catalogue_rules( + *, + catalogue: tuple[Rule, ...], + config: RulesConfig, + project_dir: Path, + custom_payloads: list[dict[str, object]] | None = None, +) -> tuple[Rule, ...]: ... +def resolve_rule_ignore_selectors( + *, config: RulesConfig, project: CompiledProjectView +) -> RulesConfig: ... +def format_rule(*, rule: Rule, config: RulesConfig | None = None) -> str: ... +def install_rules_skills(*, config: RulesConfig, project_dir: Path, check: bool) -> bool: ... def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ... def parse_model_headers( headers: list[str], @@ -2585,6 +2701,19 @@ def resolve_target_name( def resolve_target_config( *, project_config: ProjectConfig, local_config: LocalConfig, target_name: str ) -> TargetConfig: ... +def resolve_adapter(*, adapter_name: str, project_dir: Path | None = None) -> BaseAdapter: ... + +class CommandWarehouseScopeInstall: + def __enter__(self) -> CommandWarehouseScope: ... + def __exit__(self, *exc_info: object) -> bool: ... + +def command_warehouse_scope( + *, + command: CliCommand | str | None, + cli_warehouse: str | None, + cli_vars: Mapping[str, object] | None = None, +) -> CommandWarehouseScopeInstall: ... +def current_command_warehouse_scope() -> CommandWarehouseScope: ... def resolve_effective_adapter_name( *, project_config: ProjectConfig, local_config: LocalConfig ) -> str: ... @@ -2842,9 +2971,8 @@ class SqlReferenceScanner: def scope_expected_model_names( texts: list[tuple[str, str]], scenario: bool, syntax: dict[str, object] ) -> list[tuple[str | None, list[str]]]: ... -def parse_cli_arguments( - argv: list[str], defaults: list[tuple[str, object]], *, use_color: bool, columns: int -) -> tuple[list[tuple[str, object]] | None, int | None]: ... +def supports_color() -> bool: ... +def parse_cli_invocation(argv: Sequence[str], *, use_color: bool) -> ParsedCliInvocation: ... def run_cli(argv: Sequence[str] | None = None, handlers: object | None = None) -> int: ... def extract_sql_test_ctes( *, sql: str, file_label: str, syntax: SqlLexicalSyntax @@ -3366,6 +3494,12 @@ def resolve_loader_defaults( target_config: TargetConfig | None = None, cli_vars: dict[str, object] | None = None, ) -> tuple[str | None, str | None]: ... +def write_runtime_target( + *, target_dir: Path, plan_output: PlanOutput, result: BuildExecutionResult +) -> None: ... +def run_plan_command(*, request: PlanCommandRequest) -> int: ... +def run_rules_command(*, request: RulesCommandRequest) -> int: ... +def stop_progress_spinners() -> None: ... class TransientStatusReporter: def __init__( @@ -3788,6 +3922,62 @@ def custom_rule_test_evidence( *, rule: Rule, project_dir: Path ) -> list[tuple[str, int, int, str]]: ... def format_rules_result(*, result: RulesResult, json_output: bool) -> str: ... +def format_skipped_type_proof_rules(*, codes: tuple[str, ...]) -> str | None: ... +def finish_rule_run( + *, plan: RuleRunPlan, sql_result: SqlRulesEvaluation, result: RulesResult +) -> RulesRunResult: ... +def run_rules( + *, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + config: RulesConfig, + project_dir: Path, + dialect: str, + selected_keys: frozenset[CompiledObjectKey] | None = None, + prepared_sql: PreparedSqlLint | None = None, + expansion_reuse: SqlExpansionReuse | None = None, + no_sql_analysis: bool = False, +) -> RulesRunResult: ... +def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: ... +def plan_rule_run( + *, + graph: ProjectGraph, + config: RulesConfig, + project_dir: Path, + selected_keys: frozenset[CompiledObjectKey] | None, + no_sql_analysis: bool, +) -> RuleRunPlan: ... +def run_planned_sql_rules( + *, + plan: RuleRunPlan, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + dialect: str, + prepared_sql: PreparedSqlLint | None, + expansion_reuse: SqlExpansionReuse | None, +) -> SqlRulesEvaluation: ... +def sql_lint_preparation( + *, inputs: CompileInputsView, dialect: str +) -> tuple[Path, tuple[str, ...], LintConfig] | None: ... +def early_lint_expansions( + model_inputs: tuple[ModelInput, ...], +) -> dict[Path, CompiledSqlExpansion]: ... + +class EarlySqlLint: + """Own speculative SQL checks for one compilation invocation.""" + + def __init__(self, *, enabled: bool) -> None: ... + @property + def enabled(self) -> bool: ... + @property + def preparation(self) -> PreparedSqlLint | None: ... + @property + def expansion_reuse(self) -> SqlExpansionReuse | None: ... + def __enter__(self) -> EarlySqlLint: ... + def __exit__(self, *exception: object) -> None: ... + def start(self, *, inputs: CompileInputsView, dialect: str) -> None: ... + def stop(self) -> int: ... + def with_duplicate_literal_hints( *, findings: tuple[Finding, ...], project_dir: Path ) -> tuple[Finding, ...]: ... @@ -3989,6 +4179,92 @@ def origin_reference_names( ) -> dict[str, str] | None: ... def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: ... def without_declared_columns_hash(metadata_json: str) -> str: ... +def model_definition_hash(*, model_name: str, query_sql: str, dialect: str | None) -> str: ... +def build_version_identity_metadata_json( + *, + model_name: str, + config_values: Mapping[str, object], + local_function_hashes: Mapping[str, str] | None = None, + execution_signature: Mapping[str, object] | None = None, +) -> str: ... +def build_model_version_identity_metadata_json( + *, + model: CompiledModelView, + function_local_hashes: Mapping[str, str] | None = None, + hook_version_hashes: Mapping[str, str] | None = None, +) -> str: ... +def declared_columns_hash(*, model: CompiledModelView) -> str: ... +def build_function_local_hashes( + *, functions: Iterable[CompiledFunctionView], dialect: str | None +) -> dict[str, str]: ... +def build_compiled_function_fingerprint_sql(function: CompiledFunctionView) -> str: ... +def build_seed_identity(seed: CompiledSeedView) -> tuple[str, str]: ... +def build_direct_model_version_identities( + *, + functions: Iterable[CompiledFunctionView], + scope: PlannerScope, + dialect: str | None, + seeds: Iterable[CompiledSeedView] = (), + source_version_hashes: Mapping[str, str] | None = None, + hook_functions: Iterable[DiscoveredHookFunction] = (), +) -> DirectModelVersionIdentities: ... +def with_honest_model_write_hashes( + *, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + changes: PlannerChangeResults, + version_identities: DirectModelVersionIdentities, + available_model_hashes: Mapping[str, str] | None = None, +) -> PlannerChangeResults: ... +def merge_recomputed_model_changes( + *, resolved_actions: PlannerResolvedActions, changes: PlannerChangeResults +) -> PlannerResolvedActions: ... +def build_stale_out_of_selection_warnings( + *, + original_scope: PlannerScope, + execution_scope: PlannerScope, + changes: PlannerChangeResults, + snapshot: WarehouseSnapshot, + version_identities: DirectModelVersionIdentities, + source_freshness: DirectSourceFreshnessPlanningResult | None, + include_sources: bool = True, + model_changes_complete: bool = False, +) -> tuple[PlanWarning, ...]: ... +def classify_selection_staleness_warnings( + graph: SelectionStalenessGraph, +) -> tuple[SelectionStalenessWarning, ...]: ... +def build_execution_upstream_deps( + project: CompiledProjectView, +) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... +def build_execution_edge_origins( + project: CompiledProjectView, +) -> dict[tuple[CompiledObjectKey, CompiledObjectKey], str]: ... +def build_downstream_deps( + upstream: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... +def topologically_order_keys( + *, + upstream: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + injected_edge_origins: Mapping[tuple[CompiledObjectKey, CompiledObjectKey], str] | None = None, +) -> tuple[CompiledObjectKey, ...]: ... +def build_downstream_model_name_closure( + *, + start_keys: Iterable[CompiledObjectKey], + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> frozenset[str]: ... +def check_buildability( + *, + selected_keys: Set[CompiledObjectKey], + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + snapshot: WarehouseSnapshot, + deferred_relations: Mapping[str, RelationInfo] | None = None, + satisfied_keys: Set[CompiledObjectKey] | None = None, +) -> tuple[MissingUpstream, ...]: ... +def missing_upstream_message( + *, + missing: tuple[MissingUpstream, ...], + edge_origins: Mapping[tuple[CompiledObjectKey, CompiledObjectKey], str], +) -> str: ... def model_resource_type(entry: ModelPlanEntry | None) -> str: ... def model_execution_annotation(entry: ModelPlanEntry | None) -> str: ... def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: ... @@ -3998,3 +4274,61 @@ def resolve_effective_microbatch_batch_size(*, batch_size: str, effective_grain: def aligned_timestamp_bounds( *, start: DateValue | TimestampValue, end: DateValue | TimestampValue ) -> tuple[datetime, datetime]: ... +def build_planner_scope( + *, + project: CompiledProjectView, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + auto_load_sources: bool = False, + selected_keys: frozenset[CompiledObjectKey] | None = None, + python_read_source_names: frozenset[str] = frozenset(), + sql_test_selection: SqlTestSelection | None = None, +) -> PlannerScope: ... +def managed_source_upstream_keys( + *, + selected_keys: frozenset[CompiledObjectKey], + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + project: CompiledProjectView, +) -> frozenset[CompiledObjectKey]: ... +def expand_selected_loader_dependencies( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + executable_dependency_source_keys: frozenset[CompiledObjectKey] = frozenset(), +) -> tuple[ + frozenset[CompiledObjectKey], dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] +]: ... +def upstream_loader_dependency_names( + *, + loader_function: DiscoveredLoaderFunction, + loader_functions: tuple[DiscoveredLoaderFunction, ...], +) -> tuple[str, ...]: ... +def loader_to_source_entry( + *, + project: CompiledProjectView, + loader: DiscoveredLoaderFunction, +) -> SourceEntry: ... +def build_intermediate_source_map( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], +) -> dict[str, SourceEntry]: ... +def build_upstream_intermediate_source_map( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], +) -> dict[str, SourceEntry]: ... +def build_source_load_map( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, +) -> dict[str, SourceEntry]: ... +def build_source_load_entries( + *, + execution_order: tuple[CompiledObjectKey, ...], + selected_keys: frozenset[CompiledObjectKey], + source_map: dict[str, SourceEntry], + is_reload: bool, +) -> tuple[SourceLoadPlanEntry, ...]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index e5d80c7ce0..2600991963 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -8,9 +8,11 @@ from sqlbuild._native import ( build_compiled_project, + build_planner_scope, enforce_model_execution_limit, resolve_target_config, run_clone_pipeline_with_options, + split_unit_test_selectors, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( @@ -35,7 +37,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.unit_test_selection import split_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, ClonePipelineOptions, @@ -47,7 +48,6 @@ from sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain import ( resolve_skipped_view_chain, ) -from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope from sqlbuild.compiler.planner.models import PlannerScope, UnitTestSelectorSplit from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone.main.build_destination_transient_models import ( diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index 8132be96ca..caff1a6b4d 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -12,6 +12,7 @@ enforce_plan_safety_policies, format_build_footer, format_execution_header, + write_runtime_target, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands.constants import ( @@ -22,7 +23,6 @@ TABLE_TYPE_DOWNGRADE_DBT_HELP, ) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext, PlanSafetyGates -from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import ( PlanOutput, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py deleted file mode 100644 index ccfc2f01f9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ /dev/null @@ -1,305 +0,0 @@ -"""CLI lifecycle dispatcher, native progress, and exporter scope.""" - -from __future__ import annotations - -import logging -import sys -from collections.abc import Callable, Iterator -from contextlib import ExitStack, contextmanager -from contextvars import Token -from functools import partial -from pathlib import Path - -from sqlbuild._native import ( - NativeProgressProjector, - TerminalEventIndex, - machine_output_mode, - terminal_event_index_scope, -) -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace -from sqlbuild.cli.commands.constants import ( - DEBUG_COMMAND, - LINEAGE_COMMAND, -) -from sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture import ( - configured_output_capture_scope, -) -from sqlbuild.cli.entry.main._withhold_launch_record import withhold_launch_record -from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredProvider, - DiscoveredRuntimeExtensions, -) -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope -from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher -from sqlbuild.runtime.event_exporting.constants import ( - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, - INVOCATION_TERMINAL_EVENT_TYPES, -) -from sqlbuild.runtime.event_exporting.main.event_exporter_command_scope import ( - event_exporter_command_scope, -) -from sqlbuild.runtime.event_exporting.models import ( - EventExporterAccounting, - EventExporterCounts, - EventExporterFailure, - EventExportSummary, -) -from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.main.current_execution_identity import ( - current_execution_identity, -) -from sqlbuild.runtime.observability.main.dispatcher_scope import dispatcher_scope -from sqlbuild.runtime.observability.models import ( - DispatchFailure, - ExecutionIdentity, - LifecycleEvent, -) -from sqlbuild.runtime.observability.types import Unsubscribe - -_LOGGER: logging.Logger = logging.getLogger("sqlbuild.cli.observability") -_INVOCATION_COMPLETED_EVENT_TYPE: str = "invocation_completed" -_SHUTDOWN_TIMEOUT_SETTING: str = "sinks.lifecycle.shutdown_timeout" - - -@contextmanager -def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterator[EventDispatcher]: - """Install native progress, terminal indexing, and configured event exporters.""" - - dispatcher: EventDispatcher = EventDispatcher(health_callback=_log_dispatch_failure) - terminal_index: TerminalEventIndex = TerminalEventIndex() - unsubscribe_terminal_index: Unsubscribe = dispatcher.subscribe_lifecycle( - subscriber=terminal_index.consume, - accepts_opaque=False, - ) - machine_output: bool = machine_output_mode(args) - projector: NativeProgressProjector = NativeProgressProjector( - stream=sys.stderr - if machine_output or args.debug or args.command == DEBUG_COMMAND - else sys.stdout, - use_color=not machine_output and not args.no_color and supports_color(), - debug=args.debug, - ) - unsubscribe_progress: Unsubscribe = dispatcher.subscribe_lifecycle( - subscriber=projector.consume, - accepts_opaque=False, - ) - projector_token: Token[NativeProgressProjector | None] = projector.install() - exporter_scope: EventExporterCommandScope | None = None - unsubscribe_exporters: Unsubscribe | None = None - unsubscribe_invocation_outcome: Unsubscribe | None = None - invocation_outcome: _InvocationOutcome = _InvocationOutcome() - final_export_summary: EventExportSummary | None = None - try: - extensions: DiscoveredRuntimeExtensions = ( - DiscoveredRuntimeExtensions() - if args.command == LINEAGE_COMMAND - else discover_runtime_extensions(project_dir=project_dir) - ) - providers: tuple[DiscoveredProvider, ...] = extensions.providers - event_exporters: tuple[DiscoveredEventExporter, ...] = extensions.event_exporters - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ( - extensions.command_output_sinks - ) - if event_exporters or command_output_sinks: - withhold_launch_record() - exporter_delivery: EventExporterDispatcher = EventExporterDispatcher( - shutdown_timeout_seconds=( - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS - if extensions.lifecycle_shutdown_timeout_seconds is None - else float(extensions.lifecycle_shutdown_timeout_seconds) - ), - failure_callback=_log_exporter_failure, - summary_callback=_log_exporter_summary, - ) - exporter_scope = EventExporterCommandScope(dispatcher=exporter_delivery) - exporter_scope.configure_extensions( - project_dir=project_dir, - providers=providers, - event_exporters=event_exporters, - command_output_sinks=command_output_sinks, - ) - unsubscribe_exporters = dispatcher.subscribe_lifecycle( - subscriber=exporter_delivery.enqueue, - accepts_opaque=False, - ) - unsubscribe_invocation_outcome = dispatcher.subscribe_lifecycle( - subscriber=invocation_outcome.consume, - accepts_opaque=False, - ) - with ExitStack() as stack: - _ = stack.enter_context(dispatcher_scope(dispatcher)) - _ = stack.enter_context(terminal_event_index_scope(terminal_index)) - if exporter_scope is not None: - _ = stack.enter_context(event_exporter_command_scope(exporter_scope)) - identity: ExecutionIdentity | None = current_execution_identity() - if identity is not None and command_output_sinks: - _ = stack.enter_context( - configured_output_capture_scope( - exporter_scope=exporter_scope, - identity=identity, - failure_callback=_log_command_output_failure, - ) - ) - yield dispatcher - finally: - if unsubscribe_exporters is not None: - _run_cleanup(action=unsubscribe_exporters, phase="event_exporter_unsubscribe") - if unsubscribe_invocation_outcome is not None: - _run_cleanup( - action=unsubscribe_invocation_outcome, phase="invocation_outcome_unsubscribe" - ) - if exporter_scope is not None: - final_export_summary = _close_exporter_scope(exporter_scope) - _run_cleanup(action=unsubscribe_terminal_index, phase="terminal_index_unsubscribe") - _run_cleanup(action=unsubscribe_progress, phase="progress_unsubscribe") - _run_cleanup(action=projector.close, phase="progress_close") - _run_cleanup( - action=lambda: projector.restore(projector_token), - phase="progress_context_restore", - ) - if final_export_summary is not None: - _run_cleanup( - action=partial( - _warn_incomplete_export, - summary=final_export_summary, - command_succeeded=invocation_outcome.succeeded, - ), - phase="event_exporter_warning", - ) - - -def format_event_export_warning( - *, summary: EventExportSummary, command_succeeded: bool -) -> str | None: - """Return one stderr warning line when lifecycle events were dropped or failed.""" - - if summary.dropped == 0 and summary.failed == 0: - return None - affected: tuple[EventExporterAccounting, ...] = tuple( - exporter - for exporter in summary.per_exporter - if exporter.counts.dropped > 0 or exporter.counts.failed > 0 - ) - names: str = ", ".join(f"'{exporter.exporter_name}'" for exporter in affected) - sink_label: str = "sink" if len(affected) == 1 else "sinks" - unit: str = "events" if len(summary.per_exporter) == 1 else "event deliveries" - prefix: str = ( - "Warning: command completed successfully, but lifecycle event export was incomplete" - if command_succeeded - else "Warning: lifecycle event export was incomplete" - ) - return ( - f"{prefix}: {summary.dropped} of {summary.accepted} {unit} dropped, " - f"{summary.failed} failed ({sink_label} {names}). " - f"Increase {_SHUTDOWN_TIMEOUT_SETTING} or check sink health." - ) - - -class _InvocationOutcome: - """Record whether the invocation published a successful terminal fact.""" - - def __init__(self) -> None: - self.succeeded: bool = False - - def consume(self, event: LifecycleEvent) -> None: - if event.event_type in INVOCATION_TERMINAL_EVENT_TYPES: - self.succeeded = event.event_type == _INVOCATION_COMPLETED_EVENT_TYPE - - -def _warn_incomplete_export(*, summary: EventExportSummary, command_succeeded: bool) -> None: - message: str | None = format_event_export_warning( - summary=summary, command_succeeded=command_succeeded - ) - if message is not None: - print(message, file=sys.stderr, flush=True) - - -def _close_exporter_scope(scope: EventExporterCommandScope) -> EventExportSummary | None: - try: - return scope.close() - except BaseException as error: - _report_cleanup_failure(error=error, phase="event_exporter_shutdown") - return None - - -def _run_cleanup(*, action: Callable[[], object], phase: str) -> None: - try: - _ = action() - except BaseException as error: - _report_cleanup_failure(error=error, phase=phase) - - -def _report_cleanup_failure(*, error: BaseException, phase: str) -> None: - try: - log_debug_event( - logger=_LOGGER, - message="CLI observability cleanup failed", - error_type=type(error).__name__, - phase=phase, - ) - except BaseException: - pass - - -def _log_dispatch_failure(failure: DispatchFailure) -> None: - log_debug_event( - logger=_LOGGER, - message="Lifecycle event subscriber failed", - error_type=failure.error_type, - channel=failure.channel, - subscriber=failure.subscriber, - ) - - -def _log_exporter_failure(failure: EventExporterFailure) -> None: - log_debug_event( - logger=_LOGGER, - message="Event exporter delivery failed", - exporter_name=failure.exporter_name, - error_type=failure.error_type, - event_kind=failure.event_kind, - event_severity=failure.event_severity, - ) - - -def _log_command_output_failure(error: BaseException) -> None: - log_debug_event( - logger=_LOGGER, - message="Command-output sink delivery failed", - error_type=type(error).__name__, - ) - - -def _log_exporter_summary(summary: EventExportSummary) -> None: - log_debug_event( - logger=_LOGGER, - message="Event exporter delivery summary", - accepted=summary.accepted, - filtered=summary.filtered, - delivered=summary.delivered, - failed=summary.failed, - dropped=summary.dropped, - queue_depth=summary.queue_depth, - queue_capacity=summary.queue_capacity, - flush_complete=summary.flush_complete, - ) - for exporter in summary.per_exporter: - counts: EventExporterCounts = exporter.counts - log_debug_event( - logger=_LOGGER, - message="Event exporter accounting", - exporter_name=exporter.exporter_name, - accepted=counts.accepted, - filtered=counts.filtered, - delivered=counts.delivered, - failed=counts.failed, - dropped=counts.dropped, - queue_depth=summary.queue_depth, - queue_capacity=summary.queue_capacity, - flush_complete=summary.flush_complete, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py deleted file mode 100644 index a3bbfef78b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ /dev/null @@ -1,33 +0,0 @@ -"""CLI argument parsing through the native argv parser, with post-parse normalization.""" - -from __future__ import annotations - -import shutil -import sys -from collections.abc import Sequence - -import sqlbuild._native as _native -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace -from sqlbuild.cli.entry.models import ParsedCliInvocation - - -def parse_cli_invocation(*, argv: Sequence[str] | None, use_color: bool) -> ParsedCliInvocation: - """Parse CLI arguments into a namespace; help, version and errors print and exit.""" - - args: CliNamespace = CliNamespace() - raw_args: list[str] = list(sys.argv[1:] if argv is None else argv) - rows: list[tuple[str, object]] | None - exit_code: int | None - rows, exit_code = _native.parse_cli_arguments( - raw_args, - list(vars(args).items()), - use_color=use_color, - columns=shutil.get_terminal_size().columns, - ) - if rows is None: - return ParsedCliInvocation(args=None, exit_code=exit_code) - name: str - value: object - for name, value in rows: - setattr(args, name, value) - return ParsedCliInvocation(args=args, exit_code=None) diff --git a/src/sqlbuild/cli/commands/_helpers/input/__init__.py b/src/sqlbuild/cli/commands/_helpers/input/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/input/parsing.py b/src/sqlbuild/cli/commands/_helpers/input/parsing.py deleted file mode 100644 index f248468e33..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/input/parsing.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Neutral command input conversion.""" - -from __future__ import annotations - -from datetime import datetime -from decimal import Decimal - -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar - - -def parse_cursor_timestamp(value: str | CursorScalar | None) -> datetime | None: - if value is None: - return None - if isinstance(value, TimestampValue): - return value.value - if isinstance(value, DateValue): - return datetime.combine(value.value, datetime.min.time()) - if isinstance(value, IntegerValue): - raise CursorAlgebraError("an integer cursor cannot be used as a timestamp") - return datetime.fromisoformat(value) - - -def parse_cursor_integer(value: str | CursorScalar | None) -> int | None: - if value is None: - return None - if isinstance(value, IntegerValue): - return value.value - if isinstance(value, DateValue | TimestampValue): - raise CursorAlgebraError("a timestamp cursor cannot be used as an integer") - return int(Decimal(value)) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/__init__.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py deleted file mode 100644 index 17f5b4df22..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Janitor command compilation phase.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild._native import PlanningProgressReporter, build_compiled_project -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.models import ( - AdapterConnectionContext, - JanitorCompileContext, - JanitorInvocation, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileContext: - """Compile for the janitor or ``--as`` target and resolve the active target connection.""" - - adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=invocation.discovered_inputs, - effective_project_dir=invocation.effective_project_dir, - selected_target=invocation.selected_target, - cli_vars=None, - ) - compile_start: float = time.perf_counter() - status: PlanningProgressReporter = PlanningProgressReporter( - stream=sys.stdout, - use_color=invocation.use_color, - ) - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - status.on_progress("Compiling project...") - project: CompiledProjectView = build_compiled_project( - discovered_inputs=invocation.discovered_inputs, - adapter=adapter_context.adapter, - selected_target=( - invocation.as_target - if invocation.as_target is not None - else invocation.selected_target - ), - resolved_connection=adapter_context.connection_config, - ) - status.on_progress(f"Compiled project. ({time.perf_counter() - compile_start:.2f}s)") - return JanitorCompileContext( - adapter_name=adapter_context.adapter_name, - adapter=adapter_context.adapter, - project=project, - connection_config=adapter_context.connection_config, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py deleted file mode 100644 index 0c9e5f71f9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Janitor command connection phases.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild._native import ConnectionProgressReporter -from sqlbuild.cli.commands.models import ( - JanitorCompileContext, - JanitorConnectionContext, - JanitorInvocation, -) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def connect_janitor_warehouse( - *, invocation: JanitorInvocation, compile_context: JanitorCompileContext -) -> JanitorConnectionContext: - """Connect to the janitor target warehouse.""" - - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=compile_context.adapter_name, - stream=sys.stdout, - use_color=invocation.use_color, - ) - connection_start: float = time.perf_counter() - connection_progress.on_connection_start(1) - try: - with OperationLifecycle( - operation_kind="janitor", operation_name="janitor_target_connection" - ): - connection: object = compile_context.adapter.connect(compile_context.connection_config) - except BaseException: - connection_progress.on_connection_error( - connection_count=1, elapsed_seconds=time.perf_counter() - connection_start - ) - raise - connection_progress.on_connection_complete( - connection_count=1, elapsed_seconds=time.perf_counter() - connection_start - ) - return JanitorConnectionContext(connection=connection) - - -def close_janitor_warehouse( - *, compile_context: JanitorCompileContext, connection_context: JanitorConnectionContext -) -> None: - """Close the janitor warehouse connection.""" - - compile_context.adapter.close(connection_context.connection) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py deleted file mode 100644 index 06820c93fd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Janitor command warehouse cleanup execution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.models import ( - JanitorCompileContext, - JanitorConnectionContext, - JanitorPlanningResult, -) -from sqlbuild.executor.janitor.main.execute import execute_janitor_plan -from sqlbuild.executor.janitor.models import JanitorExecutionResult - - -def execute_janitor_cleanup( - *, - compile_context: JanitorCompileContext, - connection_context: JanitorConnectionContext, - planning_result: JanitorPlanningResult, -) -> JanitorExecutionResult: - """Execute the warehouse cleanup plan.""" - - return execute_janitor_plan( - plan=planning_result.plan, - adapter=compile_context.adapter, - connection=connection_context.connection, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py deleted file mode 100644 index 2a0a9f7fc9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Janitor command warehouse inspection and planning phases.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild._native import TransientStatusReporter -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - JanitorCompileContext, - JanitorConnectionContext, - JanitorInvocation, - JanitorPlanningResult, - JanitorSettings, -) -from sqlbuild.executor.janitor.main.plan import build_janitor_plan -from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def build_janitor_execution_plan( - *, - invocation: JanitorInvocation, - settings: JanitorSettings, - compile_context: JanitorCompileContext, - connection_context: JanitorConnectionContext, -) -> JanitorPlanningResult: - """Build the janitor execution plan from warehouse facts.""" - - inspect_start: float = time.perf_counter() - status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stdout, - use_color=invocation.use_color, - ) - status.start("Inspecting warehouse state...") - with OperationLifecycle( - operation_kind="janitor", operation_name="janitor_candidate_planning" - ) as lifecycle: - plan: JanitorPlan = build_janitor_plan( - project=compile_context.project, - adapter=compile_context.adapter, - connection=connection_context.connection, - retention_days=settings.retention_days, - delete_tracked_only=invocation.discovered_inputs.project_config.janitor.delete_tracked_only, - exclude_patterns=invocation.discovered_inputs.project_config.janitor.exclude_patterns, - direct_settings=JanitorDirectModeSettings( - enabled=True, - state_history_versions=settings.direct_state_history_versions, - archive_retention_days=settings.archive_retention_days, - ), - early_old_name_view_drops=settings.drop_old_name_views, - ) - lifecycle.completed(metadata={"item_count": _janitor_candidate_count(plan)}) - status.complete( - message=f"Inspected warehouse state. ({time.perf_counter() - inspect_start:.2f}s)", - blank_line_after=True, - ) - if plan.old_name_views.unknown_requests: - raise CliUserError( - "no compatibility view is recorded at " - + ", ".join(plan.old_name_views.unknown_requests), - code="C503", - help="Run sqb janitor without --drop-old-name-view to list compatibility views.", - ) - return JanitorPlanningResult(plan=plan) - - -def _janitor_candidate_count(plan: JanitorPlan) -> int: - return sum( - len(candidates) - for candidates in ( - plan.candidates, - plan.archive_deletion_candidates, - plan.query_diff_artifact_candidates, - plan.direct_state_prune_candidates, - plan.old_name_views.drops, - ) - ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/__init__.py b/src/sqlbuild/cli/commands/_helpers/plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py deleted file mode 100644 index 15aebce087..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Plan command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - build_effective_settings_config, - discover_project_inputs, - validate_preview_target, -) -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.models import ( - AdapterConnectionContext, - PlanCommandRequest, - PlanInvocation, -) -from sqlbuild.cli.progress.main._build_command_progress_reporters import ( - build_command_progress_reporters, -) -from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color - - -def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: - """Resolve discovery, validations, adapter, connection, and reporters for plan.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir, extract_output_column_locations=False - ) - validate_preview_target( - discovered_inputs=discovered_inputs, as_target=request.as_target, command_name="plan" - ) - adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=discovered_inputs, - effective_project_dir=effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - use_color: bool = not request.no_color and not request.json_output and supports_color() - progress_stream: TextIO = sys.stderr if request.json_output else sys.stdout - reporters: CommandProgressReporters = build_command_progress_reporters( - adapter_name=adapter_context.adapter_name, - stream=progress_stream, - use_color=use_color, - ) - should_load_sources: bool = ( - request.load_sources - if request.load_sources is not None - else build_effective_settings_config(discovered_inputs=discovered_inputs).auto_load_sources - ) - return PlanInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter_context.adapter, - connection_config=adapter_context.connection_config, - use_color=use_color, - progress_stream=progress_stream, - connection_progress=reporters.connection, - planning_progress=reporters.planning, - should_load_sources=should_load_sources, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py deleted file mode 100644 index c513c58271..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Plan command output writing phase.""" - -from __future__ import annotations - -from sqlbuild._native import format_plan, format_plan_json -from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions - - -def write_plan_command_output( - *, - request: PlanCommandRequest, - invocation: PlanInvocation, - pipeline_result: CompilePipelineResult, -) -> None: - """Write the plan output as JSON or formatted text.""" - - plan_output: PlanOutput = pipeline_result.plan_output - if request.json_output: - print( - format_plan_json( - plan=plan_output, - python_plan_entries=pipeline_result.python_plan_entries, - ) - ) - return - display_options: DisplayOptions = DisplayOptions( - max_entries_per_section=None if request.verbose else 50 - ) - print( - "\n" - + format_plan( - plan=plan_output, - full_refresh=request.full_refresh, - use_color=invocation.use_color, - display_options=display_options, - python_plan_entries=pipeline_result.python_plan_entries, - include_direct_freshness_diagnostics=(False), - include_completion=True, - ) - ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py deleted file mode 100644 index b52075934b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Plan command pipeline compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import run_compile_pipeline -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, -) -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def compile_plan_pipeline( - *, - request: PlanCommandRequest, - invocation: PlanInvocation, -) -> CompilePipelineResult: - """Compile the plan pipeline.""" - - external_sql_reference_resolver: object | None = resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ) - if request.as_target is not None: - invocation.progress_stream.write( - f"Previewing plan as target '{request.as_target}' through the active target's " - "connection (inspection only).\n" - ) - invocation.progress_stream.flush() - result: CompilePipelineResult = run_compile_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=CompilePipelineOptions( - selected_target=( - request.as_target if request.as_target is not None else request.selected_target - ), - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - defer_to=request.defer_to, - defer_sources_to=request.defer_sources_to, - cursor_overrides=request.cursor_overrides, - full_refresh=request.full_refresh, - auto_load_sources=invocation.should_load_sources, - select=request.select, - exclude=request.exclude, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - resolve_python_run_selectors=request.include_python, - max_microbatches=request.max_microbatches, - selection_diagnostics=request.selection_diagnostics, - record_migration_fingerprints=False, - ), - hooks=ConnectionHooks( - on_progress=invocation.planning_progress.on_progress, - on_connection_start=invocation.connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - return result diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py deleted file mode 100644 index d158cfa71a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Focused compiler-rule command implementation.""" - -from __future__ import annotations - -import json -import sys -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - build_project_graph, - discover_project_inputs, - resolve_effective_adapter_name, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.constants import ( - RULES_LIST_ACTION, - RULES_SHOW_ACTION, - RULES_SKILLS_ACTION, -) -from sqlbuild.cli.commands.models import RulesCommandRequest -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine.main.build_catalogue import build_catalogue -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.render_result import format_result -from sqlbuild.rule_engine.main.render_rule import format_rule -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -from sqlbuild.rule_engine.main.run_rules import run_rules -from sqlbuild.rule_engine.main.skills import install_skills -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult, RulesRunResult - - -def run_rules_command(request: RulesCommandRequest) -> int: - """List, inspect, execute, or generate guidance for compiler rules.""" - - project_dir: Path = ( - Path.cwd() if request.project_dir is None else request.project_dir - ).resolve() - config: RulesConfig = load_rules_config(project_dir=project_dir) - if request.action == RULES_SKILLS_ACTION: - return _run_skills(request=request, config=config, project_dir=project_dir) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) - if request.action == RULES_LIST_ACTION: - return _list_rules(catalogue=catalogue, json_output=request.json_output) - return _run_catalogue_action( - request=request, - catalogue=catalogue, - config=config, - project_dir=project_dir, - ) - - -def _run_skills(*, request: RulesCommandRequest, config: RulesConfig, project_dir: Path) -> int: - fresh: bool = install_skills(config=config, project_dir=project_dir, check=request.skills_check) - if request.skills_check: - print("Rules skills are fresh" if fresh else "Rules skills are stale") - return 0 if fresh else 1 - print("Installed Rules skills") - return 0 - - -def _list_rules(*, catalogue: tuple[Rule, ...], json_output: bool) -> int: - if json_output: - payload: dict[str, object] = { - "rules": [ - { - "code": rule.code, - "family": rule.family, - "message": rule.message, - "custom": rule.custom, - "subject": rule.subject.value if rule.subject is not None else None, - } - for rule in catalogue - ] - } - print(json.dumps(payload, sort_keys=True)) - else: - print("\n".join(f"{rule.code} {rule.message}" for rule in catalogue)) - return 0 - - -def _run_catalogue_action( - *, - request: RulesCommandRequest, - catalogue: tuple[Rule, ...], - config: RulesConfig, - project_dir: Path, -) -> int: - selector: str | None = request.rule_selector - if selector is None: - print("A rule code or family prefix is required") - return 2 - if request.action == RULES_SHOW_ACTION: - return _show_rule(selector=selector, catalogue=catalogue, config=config) - return _run_rule_selection( - request=request, - selector=selector, - catalogue=catalogue, - config=config, - project_dir=project_dir, - ) - - -def _show_rule(*, selector: str, catalogue: tuple[Rule, ...], config: RulesConfig) -> int: - matching: tuple[Rule, ...] = tuple(rule for rule in catalogue if rule.code == selector) - if not matching: - print(f"Unknown rule: {selector}") - return 2 - print(format_rule(rule=matching[0], config=config)) - return 0 - - -def _run_rule_selection( - *, - request: RulesCommandRequest, - selector: str, - catalogue: tuple[Rule, ...], - config: RulesConfig, - project_dir: Path, -) -> int: - if not any(rule.code.startswith(selector) for rule in catalogue): - print(f"Unknown rule or family: {selector}") - return 2 - print(f"Evaluating rule selection {selector}...", file=sys.stderr) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered.project_config, - local_config=discovered.local_config, - ), - project_dir=project_dir, - ) - graph: ProjectGraph = build_project_graph(discovered_inputs=discovered, adapter=adapter) - selected_keys: frozenset[CompiledObjectKey] = select_project_graph( - graph=graph, select=request.select, exclude=request.exclude - ) - result: RulesRunResult = run_rules( - graph=graph, - discovered_inputs=discovered, - config=replace(config, select=(selector,), ignore=()), - project_dir=project_dir, - dialect=adapter.sql_analysis_dialect() or "generic", - selected_keys=selected_keys if request.select or request.exclude else None, - ) - skipped_rules_note: str | None = format_skipped_type_proof_rules( - codes=result.skipped_type_proof_rules - ) - if skipped_rules_note is not None: - print(f"note: {skipped_rules_note}", file=sys.stderr) - print( - format_result( - result=RulesResult( - findings=result.findings, - evaluated_models=result.evaluated_models, - unevaluated_resources=result.unevaluated_resources, - cache_hits=result.cache_hits, - cache_misses=result.cache_misses, - ), - json_output=request.json_output, - ) - ) - if result.findings: - print( - f"Rule selection {selector} failed with {len(result.findings)} finding(s).", - file=sys.stderr, - ) - return 1 - print(f"Rule selection {selector} passed.", file=sys.stderr) - return 0 diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py deleted file mode 100644 index 155eaae005..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Adapter and connection-context resolution for CLI commands.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import resolve_effective_adapter_name, resolve_project_connection_config -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.models import AdapterConnectionContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def resolve_adapter_connection_context( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_project_dir: Path, - selected_target: str | None, - cli_vars: dict[str, object] | None, - require_connection: bool = True, -) -> AdapterConnectionContext: - """Resolve the effective adapter and project connection configuration.""" - - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, - project_dir=effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - selected_target=selected_target, - cli_vars=cli_vars, - require_connection=require_connection, - ) - return AdapterConnectionContext( - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapters.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapters.py deleted file mode 100644 index 24c2b6be1f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapters.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Adapter resolution from project configuration.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.discovery.main.resolve_adapter import ( - resolve_adapter as resolve_registered_adapter, -) -from sqlbuild.cli.commands.exceptions import CliUserError - - -def resolve_adapter(*, adapter_name: str, project_dir: Path | None = None) -> BaseAdapter: - """Resolve an adapter name from project config to an adapter instance.""" - - try: - return resolve_registered_adapter(adapter_name=adapter_name, project_dir=project_dir) - except AdapterUserError as error: - raise CliUserError(error.message, code="C601", help=error.help) from error diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py deleted file mode 100644 index 180f99d001..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Shared invocation resolution for project commands that connect to one target.""" - -from __future__ import annotations - -import sys -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path -from typing import Protocol, TextIO - -from sqlbuild._native import discover_project_inputs -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.models import AdapterConnectionContext -from sqlbuild.cli.progress.main._build_command_progress_reporters import ( - build_command_progress_reporters, -) -from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color - - -class _CommandInvocationRequest(Protocol): - """CLI request fields needed to resolve a project command invocation.""" - - @property - def project_dir(self) -> Path | None: ... - - @property - def selected_target(self) -> str | None: ... - - @property - def cli_vars(self) -> dict[str, object] | None: ... - - @property - def json_output(self) -> bool: ... - - @property - def no_color(self) -> bool: ... - - -@dataclass(frozen=True) -class _CommandInvocationContext: - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_context: AdapterConnectionContext - use_color: bool - progress_stream: TextIO - - -def resolve_command_invocation[InvocationT]( - *, request: _CommandInvocationRequest, invocation_type: Callable[..., InvocationT] -) -> InvocationT: - """Resolve discovery, adapter, connection, and output context for one command.""" - - context: _CommandInvocationContext = _resolve_command_invocation_context( - request=request, require_connection=True - ) - return invocation_type( - effective_project_dir=context.effective_project_dir, - discovered_inputs=context.discovered_inputs, - adapter_name=context.adapter_context.adapter_name, - adapter=context.adapter_context.adapter, - connection_config=context.adapter_context.connection_config, - use_color=context.use_color, - progress_stream=context.progress_stream, - ) - - -def resolve_reported_command_invocation[InvocationT]( - *, - request: _CommandInvocationRequest, - invocation_type: Callable[..., InvocationT], - require_connection: bool = True, -) -> InvocationT: - """Resolve command context plus shared connection and planning progress reporters.""" - - context: _CommandInvocationContext = _resolve_command_invocation_context( - request=request, require_connection=require_connection - ) - reporters: CommandProgressReporters = build_command_progress_reporters( - adapter_name=context.adapter_context.adapter_name, - stream=context.progress_stream, - use_color=context.use_color, - ) - return invocation_type( - effective_project_dir=context.effective_project_dir, - discovered_inputs=context.discovered_inputs, - adapter_name=context.adapter_context.adapter_name, - adapter=context.adapter_context.adapter, - connection_config=context.adapter_context.connection_config, - use_color=context.use_color, - progress_stream=context.progress_stream, - connection_progress=reporters.connection, - planning_progress=reporters.planning, - ) - - -def _resolve_command_invocation_context( - *, request: _CommandInvocationRequest, require_connection: bool -) -> _CommandInvocationContext: - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=discovered_inputs, - effective_project_dir=effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - require_connection=require_connection, - ) - machine_output: bool = request.json_output - use_color: bool = not request.no_color and not machine_output and supports_color() - return _CommandInvocationContext( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter_context=adapter_context, - use_color=use_color, - progress_stream=sys.stderr if machine_output else sys.stdout, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py b/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py deleted file mode 100644 index a068f97c5f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Active command warehouse scope for CLI warehouse connections.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token - -from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS -from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.cli.entry.models import CommandWarehouseScope - -_CURRENT_SCOPE: ContextVar[CommandWarehouseScope | None] = ContextVar( - "sqlbuild_command_warehouse_scope", default=None -) - - -@contextmanager -def command_warehouse_scope( - *, - command: CliCommand | str | None, - cli_warehouse: str | None, - cli_vars: dict[str, object] | None = None, -) -> Iterator[CommandWarehouseScope]: - """Install the warehouse group, CLI override, and CLI vars for one command invocation.""" - - scope: CommandWarehouseScope = CommandWarehouseScope( - group=None if command is None else COMMAND_WAREHOUSE_GROUPS.get(CliCommand(command)), - cli_warehouse=cli_warehouse, - cli_vars={} if cli_vars is None else dict(cli_vars), - ) - token: Token[CommandWarehouseScope | None] = _CURRENT_SCOPE.set(scope) - try: - yield scope - finally: - _CURRENT_SCOPE.reset(token) - - -def current_command_warehouse_scope() -> CommandWarehouseScope: - """Return the active command's warehouse scope, or an empty one outside the CLI.""" - - scope: CommandWarehouseScope | None = _CURRENT_SCOPE.get() - return scope if scope is not None else CommandWarehouseScope(group=None) diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 6f7ff03df6..29671a69ed 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from pathlib import Path +from sqlbuild._native import supports_color from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( log_compute_capture_failure, ) @@ -13,7 +14,6 @@ from sqlbuild.diagnostics.classes.invocation_diagnostic_routing import InvocationDiagnosticRouting from sqlbuild.diagnostics.main.configure import configure_diagnostics from sqlbuild.diagnostics.models import DiagnosticRoutingOptions -from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.compute_logs.classes.local_filesystem_compute_log_storage import ( LocalFilesystemComputeLogStorage, ) diff --git a/src/sqlbuild/cli/commands/main/execution/__init__.py b/src/sqlbuild/cli/commands/main/execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/execution/connection_progress.py b/src/sqlbuild/cli/commands/main/execution/connection_progress.py deleted file mode 100644 index 3d6ae93e2a..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/connection_progress.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Connection progress public entry for cross-domain orchestration.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild._native import ConnectionProgressReporter - - -def build_connection_progress_reporter( - *, - adapter_name: str, - stream: TextIO, - blank_line_after_complete: bool = False, - use_color: bool = False, -) -> ConnectionProgressReporter: - """Build a connection progress reporter.""" - - return ConnectionProgressReporter( - adapter_name=adapter_name, - stream=stream, - blank_line_after_complete=blank_line_after_complete, - use_color=use_color, - ) diff --git a/src/sqlbuild/cli/commands/main/project/_plan.py b/src/sqlbuild/cli/commands/main/project/_plan.py deleted file mode 100644 index 6f47ec724c..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_plan.py +++ /dev/null @@ -1,28 +0,0 @@ -"""CLI plan command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.plan.invocation import resolve_plan_invocation -from sqlbuild.cli.commands._helpers.plan.outputs import write_plan_command_output -from sqlbuild.cli.commands._helpers.plan.planning import compile_plan_pipeline -from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output -from sqlbuild.compiler.pipeline.models import CompilePipelineResult - - -def run_plan(request: PlanCommandRequest) -> int: - """Execute the plan command.""" - - invocation: PlanInvocation = resolve_plan_invocation(request=request) - if not request.json_output: - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - pipeline_result: CompilePipelineResult - with verbose_inspection_output(enabled=request.verbose): - pipeline_result = compile_plan_pipeline(request=request, invocation=invocation) - write_plan_command_output( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - return 0 diff --git a/src/sqlbuild/cli/commands/main/project/_rules.py b/src/sqlbuild/cli/commands/main/project/_rules.py deleted file mode 100644 index 63a6589f08..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_rules.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Focused compiler-rule CLI entrypoint.""" - -from sqlbuild.cli.commands._helpers.rules.command import run_rules_command as execute_rules_command -from sqlbuild.cli.commands.models import RulesCommandRequest - - -def run_rules_command(request: RulesCommandRequest) -> int: - """Execute one focused Rules command.""" - - return execute_rules_command(request) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index cab2bdd5c2..08aab9280d 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -46,11 +46,19 @@ from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) - from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.refactoring.models import RefactorProject - from sqlbuild.executor.janitor.models import JanitorPlan + + +@dataclass(frozen=True) +class NestedProgressChildRow: + """One child row rendered below a completed nested progress item.""" + + label: str + name: str + status_text: str + detail: str = "" @dataclass(frozen=True) @@ -385,30 +393,6 @@ class JanitorSettings: drop_old_name_views: tuple[str, ...] = () -@dataclass(frozen=True) -class JanitorCompileContext: - """Compiled project and adapter context for janitor.""" - - adapter_name: str - adapter: BaseAdapter - project: CompiledProjectView - connection_config: dict[str, object] - - -@dataclass(frozen=True) -class JanitorConnectionContext: - """Janitor warehouse connection handle.""" - - connection: object - - -@dataclass(frozen=True) -class JanitorPlanningResult: - """Built janitor plan.""" - - plan: JanitorPlan - - @dataclass(frozen=True) class LoadCommandRequest: """CLI inputs for one load command invocation.""" @@ -451,21 +435,6 @@ class PlanCommandRequest: selection_diagnostics: bool = False -@dataclass(frozen=True) -class PlanInvocation: - """Resolved project, adapter, and reporter context for the plan command.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - connection_progress: ConnectionProgressReporter - planning_progress: PlanningProgressReporter - should_load_sources: bool - - @dataclass(frozen=True) class PlaygroundCommandRequest: """CLI inputs for one playground command invocation.""" diff --git a/src/sqlbuild/cli/compile/main/run.py b/src/sqlbuild/cli/compile/main/run.py index 8dda17e2f1..52f9307154 100644 --- a/src/sqlbuild/cli/compile/main/run.py +++ b/src/sqlbuild/cli/compile/main/run.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.cli.compile_reuse.main._attempt_compile_reuse import attempt_compile_reuse +from sqlbuild._native import attempt_compile_reuse from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py deleted file mode 100644 index 90bc5d2537..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Decide whether a compile can replay the stored result, and replay it on a hit.""" - -from __future__ import annotations - -import os -import time - -from sqlbuild.cli.compile_reuse._helpers.launch_records import write_launch_record -from sqlbuild.cli.compile_reuse._helpers.native_reuse import native_attempt -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome - - -def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: - """Replay a stored compile when every input matches, otherwise prepare to store one.""" - - started: float = time.monotonic() - project_dir: str = os.path.abspath( - os.getcwd() if request.project_dir is None else request.project_dir - ) - attempt: CompileReuseAttempt = native_attempt( - request=request, project_dir=project_dir, started=started - ) - if attempt.outcome is CompileReuseOutcome.HIT: - write_launch_record(attempt=attempt) - return attempt diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py b/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py deleted file mode 100644 index 32fd04594f..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Leave the native `sqb` executable what it needs to serve the next unchanged compile itself.""" - -from __future__ import annotations - -import contextlib -import json -import logging -import os -import sys -import uuid -from pathlib import Path - -from sqlbuild._native import NativeReuseAttempt -from sqlbuild.cli.compile_reuse._helpers.native_reuse import reuse_rules -from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs -from sqlbuild.cli.compile_reuse.constants import ( - COMPILE_COMMAND_NAME, - COMPUTE_LOG_DIRECTORY_NAME, - LAUNCH_RECORD_DIRECTORY, - LAUNCH_RECORD_FORMAT, - LAUNCH_RECORD_LIMIT, - REUSE_BYPASS_TIMING, - REUSE_CHECK_TIMING, - REUSE_HIT_MESSAGE, - REUSE_HIT_TIMING, - REUSE_LOGGER_NAME, - REUSE_MISS_TIMING, - TOTAL_TIMING, -) -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.entry.main._launch_record_path import launch_record_path -from sqlbuild.runtime.compute_logs.classes.text_tee import TextComputeLogTee -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV - -_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) -_SECONDS_FIELD: str = "{seconds:.1f}" -_OUTPUT_FILES_RULE: str = "output_files" - - -def write_launch_record(*, attempt: CompileReuseAttempt) -> None: - """Record this reusable compile for `sqb` when nothing it cannot replay took part.""" - - path: Path | None = launch_record_path() - native: NativeReuseAttempt | None = attempt.native - if path is None or native is None or not _replayable_natively(): - return - directory: Path = attempt.project_dir.joinpath(*LAUNCH_RECORD_DIRECTORY) - if path.parent.resolve() != directory.resolve(): - return - try: - _write(path=path, payload=_payload(attempt=attempt, native=native, key=path.stem)) - _prune(directory=directory) - except OSError: - _LOGGER.debug("Compile reuse did not leave a launch record", exc_info=True) - - -def _replayable_natively() -> bool: - return INVOCATION_CONTEXT_ENV not in os.environ - - -def _payload( - *, attempt: CompileReuseAttempt, native: NativeReuseAttempt, key: str -) -> dict[str, object]: - invocation_digest, runtime, search_path = native.launch_identity() - rules: dict[str, object] = { - name: value for name, value in reuse_rules().items() if name != _OUTPUT_FILES_RULE - } - project_dir: Path = attempt.project_dir.resolve() - prefix, suffix = REUSE_HIT_MESSAGE.split(_SECONDS_FIELD) - return { - "format": LAUNCH_RECORD_FORMAT, - "key": key, - "project_dir": str(attempt.project_dir), - "store_path": native.store_path(), - "invocation_digest": invocation_digest, - "runtime": runtime, - "search_path": search_path, - "startup_inputs": startup_inputs(), - "json_output": attempt.json_output, - "rules": rules, - "capture": { - "root": str((project_dir / COMPUTE_LOG_DIRECTORY_NAME).resolve()), - "command": COMPILE_COMMAND_NAME, - "project_dir": str(project_dir), - "target": attempt.selected_target, - } - if isinstance(sys.stdout, TextComputeLogTee) - else None, - "hit_message": [prefix, suffix], - "timings": { - "hit": REUSE_HIT_TIMING, - "miss": REUSE_MISS_TIMING, - "bypass": REUSE_BYPASS_TIMING, - "check": REUSE_CHECK_TIMING, - "total": TOTAL_TIMING, - }, - } - - -def _write(*, path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - _ = temporary.write_text(json.dumps(payload), encoding="utf-8") - os.replace(temporary, path) - finally: - with contextlib.suppress(OSError): - temporary.unlink(missing_ok=True) - - -def _prune(*, directory: Path) -> None: - records: list[tuple[int, Path]] = [ - (entry.stat().st_mtime_ns, entry) for entry in directory.glob("*.json") - ] - records.sort(reverse=True) - for _, stale in records[LAUNCH_RECORD_LIMIT:]: - with contextlib.suppress(OSError): - stale.unlink() diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py deleted file mode 100644 index abb322f09a..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Check, replay and record whole-project reuse through the native store.""" - -from __future__ import annotations - -import logging -import os -import sys -from dataclasses import replace -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import NativeReuseAttempt -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( - invocation_identity, - redirected_output_files, - runtime_identity, -) -from sqlbuild.cli.compile_reuse._helpers.timings import elapsed_ms, reuse_timings -from sqlbuild.cli.compile_reuse.constants import ( - COMPILED_DIRECTORY_NAME, - EXCLUDED_DIRECTORIES, - EXCLUDED_ROOT_DIRECTORIES, - MILLISECONDS_PER_SECOND, - MISSING_ENVIRONMENT_VALUE, - MISSING_FILE_DIGEST, - MISSING_PATH_MTIME_NS, - NATIVE_REUSE_DIRECTORY_NAME, - PRESENCE_ONLY_FILE_SUFFIXES, - PROJECT_CONFIG_FILENAMES, - PROJECT_ROOT_PATH_MTIME_NS, - RACY_WINDOW_NS, - RETIRED_REUSE_DIRECTORY_NAME, - REUSE_DISABLE_ENV_VAR, - REUSE_DISABLE_VALUE, - REUSE_HIT_MESSAGE, - REUSE_LOGGER_NAME, - REUSE_MAX_ENTRY_BYTES, - REUSE_MAX_STORED_ENTRIES, - TARGET_DIRECTORY_NAME, - TOTAL_TIMING, - TRACKED_ENVIRONMENT_PREFIXES, - UNTRACKED_ENVIRONMENT_NAMES, -) -from sqlbuild.cli.compile_reuse.models import ( - CompileReuseAttempt, - CompileReuseRequest, -) -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.presentation.main.supports_color import supports_color - -_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) -_OUTCOMES: dict[str, CompileReuseOutcome] = { - outcome.value: outcome for outcome in CompileReuseOutcome -} -_STORED: str = "stored" -_SKIPPED: str = "skipped" - - -def native_attempt( - *, request: CompileReuseRequest, project_dir: str, started: float -) -> CompileReuseAttempt: - """Check reuse natively, replaying on a hit.""" - - attempt: NativeReuseAttempt = _native.check_compile_reuse( - { - "project_dir": project_dir, - "store_directory": compiler_cache_directory(Path(project_dir)) - / NATIVE_REUSE_DIRECTORY_NAME, - "selected_target": request.selected_target, - "invocation": invocation_identity( - request=request, - project_dir=project_dir, - use_color=(not request.no_color) and supports_color(), - ), - "runtime": runtime_identity(), - "search_path": list(sys.path), - "bypass_requested": request.no_cache - or request.manifest - or request.profiling - or request.debug, - "json_output": request.json_output, - "rules": reuse_rules(), - } - ) - outcome: CompileReuseOutcome = _OUTCOMES[attempt.outcome()] - base: CompileReuseAttempt = CompileReuseAttempt( - outcome=outcome, - project_dir=Path(project_dir), - started=started, - selected_target=request.selected_target, - json_output=request.json_output, - ) - if outcome is CompileReuseOutcome.BYPASS: - return base - _LOGGER.debug("Compile reuse check digested %d project files", attempt.digested_files()) - missed: CompileReuseAttempt = replace( - base, - outcome=CompileReuseOutcome.MISS, - entry_path=Path(attempt.store_path()), - native=attempt, - check_ms=0 if attempt.replay_failed() else elapsed_ms(started=started), - ) - if outcome is CompileReuseOutcome.HIT: - return _replayed(attempt=attempt, missed=missed, started=started, json=request.json_output) - return missed - - -def _replayed( - *, - attempt: NativeReuseAttempt, - missed: CompileReuseAttempt, - started: float, - json: bool, -) -> CompileReuseAttempt: - check_ms: int = elapsed_ms(started=started) - timings: dict[str, int] = { - **reuse_timings(outcome=CompileReuseOutcome.HIT, check_ms=check_ms), - TOTAL_TIMING: elapsed_ms(started=started), - } - replay: tuple[str, list[str], int] | None = attempt.replay( - list(timings.items()) if json else [] - ) - if replay is None: - return replace(missed, check_ms=0) - stdout, stderr_lines, exit_code = replay - print(REUSE_HIT_MESSAGE.format(seconds=check_ms / MILLISECONDS_PER_SECOND), file=sys.stderr) - for line in stderr_lines: - print(line, file=sys.stderr) - _ = sys.stdout.write(stdout) - sys.stdout.flush() - return replace(missed, outcome=CompileReuseOutcome.HIT, check_ms=check_ms, exit_code=exit_code) - - -def reuse_rules() -> dict[str, object]: - """The reuse constants the native check and record share with the Python implementation.""" - - return { - "excluded": sorted(EXCLUDED_DIRECTORIES), - "excluded_root": sorted(EXCLUDED_ROOT_DIRECTORIES), - "presence_suffixes": list(PRESENCE_ONLY_FILE_SUFFIXES), - "output_files": redirected_output_files(), - "tracked_prefixes": list(TRACKED_ENVIRONMENT_PREFIXES), - "untracked_names": sorted(UNTRACKED_ENVIRONMENT_NAMES), - "disabling_variables": [ - (COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE), - (REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE), - ], - "config_filenames": list(PROJECT_CONFIG_FILENAMES), - "racy_window_ns": RACY_WINDOW_NS, - "max_stored_entries": REUSE_MAX_STORED_ENTRIES, - "max_entry_bytes": REUSE_MAX_ENTRY_BYTES, - "missing_environment_value": MISSING_ENVIRONMENT_VALUE, - "missing_file_digest": MISSING_FILE_DIGEST, - "missing_path_mtime_ns": MISSING_PATH_MTIME_NS, - "project_root_path_mtime_ns": PROJECT_ROOT_PATH_MTIME_NS, - "compiled_root": os.path.join(TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME), - "retired_directories": [RETIRED_REUSE_DIRECTORY_NAME], - } diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py b/src/sqlbuild/cli/compile_reuse/_helpers/timings.py deleted file mode 100644 index e33f7438b9..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Compile reuse counters and elapsed time reported in compile_timings.""" - -from __future__ import annotations - -import time - -from sqlbuild.cli.compile_reuse.constants import ( - MILLISECONDS_PER_SECOND, - REUSE_BYPASS_TIMING, - REUSE_CHECK_TIMING, - REUSE_HIT_TIMING, - REUSE_MISS_TIMING, -) -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome - - -def reuse_timings(*, outcome: CompileReuseOutcome, check_ms: int) -> dict[str, int]: - """Return the reuse counters and check time reported in compile_timings.""" - - return { - REUSE_HIT_TIMING: int(outcome is CompileReuseOutcome.HIT), - REUSE_MISS_TIMING: int(outcome is CompileReuseOutcome.MISS), - REUSE_BYPASS_TIMING: int(outcome is CompileReuseOutcome.BYPASS), - REUSE_CHECK_TIMING: check_ms, - } - - -def elapsed_ms(*, started: float) -> int: - """Return whole milliseconds elapsed since a monotonic start time.""" - - return int((time.monotonic() - started) * MILLISECONDS_PER_SECOND) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 0980a46ebe..a6c9d8cca0 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -76,6 +76,5 @@ ) LAUNCH_RECORD_FORMAT: int = 1 LAUNCH_RECORD_DIRECTORY: tuple[str, ...] = (TARGET_DIRECTORY_NAME, "cache", "launcher-v1") -LAUNCH_RECORD_LIMIT: int = 32 COMPUTE_LOG_DIRECTORY_NAME: str = "logs" COMPILE_COMMAND_NAME: str = "compile" diff --git a/src/sqlbuild/cli/compile_reuse/main/__init__.py b/src/sqlbuild/cli/compile_reuse/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py b/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py deleted file mode 100644 index 587fd42003..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public entry for replaying the previous compile when no compile input changed.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.attempt import attempt_reuse -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest - - -def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: - """Replay the stored compile on a hit, otherwise return the state a full compile stores.""" - - return attempt_reuse(request=request) diff --git a/src/sqlbuild/cli/paths/_helpers/artifact_path.py b/src/sqlbuild/cli/paths/_helpers/artifact_path.py deleted file mode 100644 index a5528bf8f1..0000000000 --- a/src/sqlbuild/cli/paths/_helpers/artifact_path.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Stable, filesystem-safe CLI artifact paths.""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -from sqlbuild.compiler.planner.models import SqlTestPlanEntry - -_CHAIN_DIR: str = "_chain_" -_SQL_FILE_SUFFIX: str = ".sql" -_MAX_CHAIN_COMPONENT_BYTES: int = 200 -_CHAIN_DIGEST_LENGTH: int = 12 -_CHAIN_EDGE_NAME_BYTES: int = 80 - - -def build_sql_test_output_path(entry: SqlTestPlanEntry) -> Path: - """Return a readable test path with bounded chain directory components.""" - - return build_sql_test_output_path_from_values( - name=entry.name, - source_path=entry.source_path, - block_index=entry.block_index, - case_name=entry.case_name, - model_names=tuple(step.model_name for step in entry.chain), - ) - - -def build_sql_test_output_path_from_values( - *, - name: str, - source_path: Path | None, - block_index: int | None, - case_name: str | None, - model_names: tuple[str, ...], -) -> Path: - """Return the stable test path from compact planner artifact metadata.""" - - if case_name is None or source_path is None: - return _test_folder_from_names(name=name, model_names=model_names) / ( - f"{name}{_SQL_FILE_SUFFIX}" - ) - source_stem: Path = source_path.with_suffix("") - return source_stem / f"block_{block_index}__{case_name}{_SQL_FILE_SUFFIX}" - - -def _test_folder_from_names(*, name: str, model_names: tuple[str, ...]) -> Path: - unique_names: list[str] = sorted(set(model_names)) - if len(unique_names) <= 1: - return Path(unique_names[0] if unique_names else name) - chain_name: str = "__".join(unique_names) - if len(chain_name.encode()) <= _MAX_CHAIN_COMPONENT_BYTES: - return Path(_CHAIN_DIR) / chain_name - digest: str = hashlib.sha256(chain_name.encode()).hexdigest()[:_CHAIN_DIGEST_LENGTH] - first_name: str = _truncate_utf8(value=unique_names[0], max_bytes=_CHAIN_EDGE_NAME_BYTES) - last_name: str = _truncate_utf8(value=unique_names[-1], max_bytes=_CHAIN_EDGE_NAME_BYTES) - return Path(_CHAIN_DIR) / f"{first_name}__{last_name}__{digest}" - - -def _truncate_utf8(*, value: str, max_bytes: int) -> str: - return value.encode()[:max_bytes].decode(errors="ignore") diff --git a/src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py b/src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py deleted file mode 100644 index 15a43b417c..0000000000 --- a/src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Compiled SQL test artifact path entrypoint.""" - -from pathlib import Path - -from sqlbuild.cli.paths._helpers.artifact_path import build_sql_test_output_path_from_values -from sqlbuild.compiler.compile.models import CompiledSqlTest - - -def compiled_sql_test_output_path(*, test: CompiledSqlTest, model_names: tuple[str, ...]) -> Path: - """Return a test path from a compiled test and compact native plan facts.""" - - return build_sql_test_output_path_from_values( - name=test.name, - source_path=test.source_path, - block_index=test.block_index, - case_name=test.case_name, - model_names=model_names, - ) diff --git a/src/sqlbuild/cli/paths/main/_sql_test_output_path.py b/src/sqlbuild/cli/paths/main/_sql_test_output_path.py deleted file mode 100644 index e28da2e1ba..0000000000 --- a/src/sqlbuild/cli/paths/main/_sql_test_output_path.py +++ /dev/null @@ -1,12 +0,0 @@ -"""SQL test artifact path entrypoint.""" - -from pathlib import Path - -from sqlbuild.cli.paths._helpers.artifact_path import build_sql_test_output_path -from sqlbuild.compiler.planner.models import SqlTestPlanEntry - - -def sql_test_output_path(entry: SqlTestPlanEntry) -> Path: - """Return the filesystem-safe target path for one SQL test artifact.""" - - return build_sql_test_output_path(entry) diff --git a/src/sqlbuild/cli/progress/__init__.py b/src/sqlbuild/cli/progress/__init__.py deleted file mode 100644 index c202eed76a..0000000000 --- a/src/sqlbuild/cli/progress/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI progress rendering.""" diff --git a/src/sqlbuild/cli/progress/main/__init__.py b/src/sqlbuild/cli/progress/main/__init__.py deleted file mode 100644 index afe461d975..0000000000 --- a/src/sqlbuild/cli/progress/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI progress entry functions.""" diff --git a/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py b/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py deleted file mode 100644 index 826778e69f..0000000000 --- a/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Progress reporter construction for CLI commands.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter -from sqlbuild.cli.progress.models import CommandProgressReporters - - -def build_command_progress_reporters( - *, - adapter_name: str, - stream: TextIO, - use_color: bool, -) -> CommandProgressReporters: - """Build connection and planning progress reporters for one command.""" - - return CommandProgressReporters( - connection=ConnectionProgressReporter( - adapter_name=adapter_name, - stream=stream, - use_color=use_color, - ), - planning=PlanningProgressReporter( - stream=stream, - use_color=use_color, - ), - ) diff --git a/src/sqlbuild/cli/progress/main/verbose_inspection_output.py b/src/sqlbuild/cli/progress/main/verbose_inspection_output.py deleted file mode 100644 index 84d16e9534..0000000000 --- a/src/sqlbuild/cli/progress/main/verbose_inspection_output.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Verbose per-query warehouse inspection output on stderr.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild._native import VerboseInspectionWriter -from sqlbuild.adapter.relations.main.inspection_query_sink import inspection_query_sink - - -@contextmanager -def verbose_inspection_output(*, enabled: bool) -> Iterator[None]: - """Write each inspection read, then a total, to stderr while ``enabled``.""" - - if not enabled: - yield - return - writer: VerboseInspectionWriter = VerboseInspectionWriter() - with inspection_query_sink(writer.write): - yield - writer.write_total() diff --git a/src/sqlbuild/cli/progress/models.py b/src/sqlbuild/cli/progress/models.py deleted file mode 100644 index 91c51b4c99..0000000000 --- a/src/sqlbuild/cli/progress/models.py +++ /dev/null @@ -1,25 +0,0 @@ -"""CLI progress models.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter - - -@dataclass(frozen=True) -class CommandProgressReporters: - """Connection and planning progress reporters bound to one output stream.""" - - connection: ConnectionProgressReporter - planning: PlanningProgressReporter - - -@dataclass(frozen=True) -class NestedProgressChildRow: - """One child row rendered below a completed nested progress item.""" - - label: str - name: str - status_text: str - detail: str = "" diff --git a/src/sqlbuild/cli/target_artifacts/__init__.py b/src/sqlbuild/cli/target_artifacts/__init__.py deleted file mode 100644 index 1011f89980..0000000000 --- a/src/sqlbuild/cli/target_artifacts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI target artifact capabilities.""" diff --git a/src/sqlbuild/cli/target_artifacts/_helpers/runtime.py b/src/sqlbuild/cli/target_artifacts/_helpers/runtime.py deleted file mode 100644 index a3e02a8e92..0000000000 --- a/src/sqlbuild/cli/target_artifacts/_helpers/runtime.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Write runtime SQL artifacts to target/run.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.paths.main._sql_test_output_path import sql_test_output_path -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import ( - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SqlTestPlanEntry, -) -from sqlbuild.executor.build.models import BuildExecutionResult, FunctionExecutionResult -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.models import SqlTestExecutionResult - -_RUN_DIR: str = "run" -_MODELS_DIR: str = "models" -_FUNCTIONS_DIR: str = "functions" -_TESTS_DIR: str = "tests" -_CHECKS_DIR: str = "checks" -_SQL_FILE_SUFFIX: str = ".sql" - - -def write_runtime_target( - *, - target_dir: Path, - plan_output: PlanOutput, - result: BuildExecutionResult, -) -> None: - """Write executed model lifecycle SQL under target/run.""" - - run_dir: Path = target_dir / _RUN_DIR - - model_entry_map: dict[str, ModelPlanEntry] = { - entry.name: entry for entry in plan_output.model_entries - } - function_entry_map: dict[str, FunctionPlanEntry] = { - entry.name: entry for entry in plan_output.function_entries - } - - model_result: ModelExecutionResult - for model_result in result.model_results: - if not model_result.lifecycle_events: - continue - sql_events: tuple[LifeCycleEvent, ...] = tuple( - e for e in model_result.lifecycle_events if e.kind == LifeCycleEventKind.SQL - ) - if not sql_events: - continue - entry: ModelPlanEntry | None = model_entry_map.get(model_result.model_name) - if entry is None: - continue - run_path: Path = run_dir / _model_output_path(entry.relative_path) - _write_sql( - path=run_path, - sql="\n\n".join(_format_statement(e.content) for e in sql_events), - ) - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if not function_result.lifecycle_events: - continue - function_sql_events: tuple[LifeCycleEvent, ...] = tuple( - e for e in function_result.lifecycle_events if e.kind == LifeCycleEventKind.SQL - ) - if not function_sql_events: - continue - function_entry: FunctionPlanEntry | None = function_entry_map.get( - function_result.function_name - ) - if function_entry is None: - continue - function_run_path: Path = run_dir / _function_output_path( - relative_path=function_entry.relative_path, - language=function_entry.language, - ) - _write_sql( - path=function_run_path, - sql="\n\n".join(_format_statement(e.content) for e in function_sql_events), - ) - - -def write_test_runtime_target( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - results: tuple[SqlTestExecutionResult, ...], -) -> None: - """Write executed SQL unit-test statements under target/run/tests.""" - - run_dir: Path = target_dir / _RUN_DIR - result_keys: frozenset[tuple[str, str | None, int | None, str | None]] = frozenset( - ( - result.test_name, - result.source_path.as_posix() if result.source_path is not None else None, - result.block_index, - result.case_name, - ) - for result in results - ) - entry: SqlTestPlanEntry - for entry in plan_output.test_entries: - entry_key: tuple[str, str | None, int | None, str | None] = ( - entry.name, - entry.source_path.as_posix() if entry.source_path is not None else None, - entry.block_index, - entry.case_name, - ) - if entry_key not in result_keys: - continue - test_run_path: Path = run_dir / _TESTS_DIR / sql_test_output_path(entry) - _write_sql( - path=test_run_path, - sql=build_sql_test_comparison_sql( - test_entry=entry, - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), - ), - ) - - -def write_python_check_runtime_target( - *, target_dir: Path, results: tuple[PythonCheckExecutionResult, ...] -) -> None: - """Write Python check runtime results under target/run/checks.""" - - if not results: - return - run_path: Path = target_dir / _RUN_DIR / _CHECKS_DIR / "python_checks.json" - run_path.parent.mkdir(parents=True, exist_ok=True) - payload: dict[str, object] = { - "checks": [ - { - "kind": "python_check", - "name": result.node_name, - "display_name": result.node_name, - "check_id": f"python_check:{result.node_name}", - "status": "pass" if result.passed else "warn" if result.warned else "fail", - "passed": result.passed, - "severity": result.severity.value, - "message": result.message, - "error_message": result.error_message, - "metadata": result.metadata, - } - for result in results - ] - } - run_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _write_sql(*, path: Path, sql: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(sql.rstrip() + "\n", encoding="utf-8") - - -def _model_output_path(relative_path: Path) -> Path: - parts: tuple[str, ...] = relative_path.parts - if parts and parts[0] == _MODELS_DIR: - return Path(*parts) - return Path(_MODELS_DIR) / relative_path - - -def _function_output_path(*, relative_path: Path, language: FunctionLanguage) -> Path: - parts: tuple[str, ...] = relative_path.parts - language_dir: str = language.value - function_language_path_part_count: int = 2 - if ( - len(parts) >= function_language_path_part_count - and parts[0] == _FUNCTIONS_DIR - and parts[1] == language_dir - ): - return Path(*parts).with_suffix(_SQL_FILE_SUFFIX) - return (Path(_FUNCTIONS_DIR) / language_dir / relative_path).with_suffix(_SQL_FILE_SUFFIX) - - -def _format_statement(statement: str) -> str: - stripped: str = statement.rstrip() - if not stripped: - return statement - if stripped.endswith(";"): - return stripped - return f"{stripped};" diff --git a/src/sqlbuild/cli/target_artifacts/main/__init__.py b/src/sqlbuild/cli/target_artifacts/main/__init__.py deleted file mode 100644 index 5978e9c4e4..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public CLI target artifact entrypoints.""" diff --git a/src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py b/src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py deleted file mode 100644 index cb92880475..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public Python check runtime target writing entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.target_artifacts._helpers.runtime import ( - write_python_check_runtime_target as _write_python_check_runtime_target, -) -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult - - -def write_python_check_runtime_target( - *, target_dir: Path, results: tuple[PythonCheckExecutionResult, ...] -) -> None: - """Write Python check runtime results under target/run/checks.""" - - _write_python_check_runtime_target(target_dir=target_dir, results=results) diff --git a/src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py b/src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py deleted file mode 100644 index 292133e1de..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public runtime target writing entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.target_artifacts._helpers.runtime import ( - write_runtime_target as _write_runtime_target, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.build.models import BuildExecutionResult - - -def write_runtime_target( - *, target_dir: Path, plan_output: PlanOutput, result: BuildExecutionResult -) -> None: - """Write executed model lifecycle SQL under target/run.""" - - _write_runtime_target(target_dir=target_dir, plan_output=plan_output, result=result) diff --git a/src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py b/src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py deleted file mode 100644 index 06d8b1fd2d..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public SQL test runtime target writing entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.target_artifacts._helpers.runtime import ( - write_test_runtime_target as _write_test_runtime_target, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.testing.models import SqlTestExecutionResult - - -def write_test_runtime_target( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - results: tuple[SqlTestExecutionResult, ...], -) -> None: - """Write executed SQL unit-test statements under target/run/tests.""" - - _write_test_runtime_target( - target_dir=target_dir, - adapter=adapter, - plan_output=plan_output, - results=results, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py b/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py deleted file mode 100644 index 8a5017538f..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Selector resolution over a project graph.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection._graph_selection import resolve_graph_selection - - -def select_project_graph( - *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> frozenset[CompiledObjectKey]: - """Resolve `--select`/`--exclude` on the graph, adding the functions they need to build.""" - - return resolve_graph_selection(graph=graph.native, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/pipeline/main/reject_unit_test_selectors.py b/src/sqlbuild/compiler/pipeline/main/reject_unit_test_selectors.py deleted file mode 100644 index adedaf289c..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/reject_unit_test_selectors.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public validation that a command which never runs unit tests selects none.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.main.selection._split_unit_test_selectors import ( - split_project_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import UnitTestSelectorSplit - - -def reject_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, -) -> UnitTestSelectorSplit: - """Raise when a selector names a unit test; return the unchanged resource selectors.""" - - return split_project_unit_test_selectors( - select=select, - exclude=exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=False, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/unit_test_selection.py b/src/sqlbuild/compiler/pipeline/main/unit_test_selection.py deleted file mode 100644 index 064b50b1b4..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/unit_test_selection.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public split of unit-test selectors from resource selectors for one command.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.main.selection._split_unit_test_selectors import ( - split_project_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import UnitTestSelectorSplit - - -def split_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, -) -> UnitTestSelectorSplit: - """Split unit-test selectors from resource selectors for a command that runs unit tests.""" - - return split_project_unit_test_selectors( - select=select, - exclude=exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=True, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index fa198254ff..642a82b8af 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -2,15 +2,15 @@ from __future__ import annotations -from sqlbuild._native import get_materialization_type +from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, + build_seed_identity, + get_materialization_type, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) -from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity from sqlbuild.compiler.planner._helpers.output.plan_entry import ( build_planner_relations_context, extract_seed_columns, diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py b/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py deleted file mode 100644 index 8b347bc534..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Planner helpers for source auto-load selection.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType - - -def managed_source_upstream_keys( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - project: CompiledProjectView, -) -> frozenset[CompiledObjectKey]: - """Return selected managed source keys reachable from the current selection.""" - - managed_source_names: frozenset[str] = frozenset( - source.name for source in project.sources if source.source_entry.loader is not None - ) - direct_upstream_sources: set[CompiledObjectKey] = set() - selected_key: CompiledObjectKey - for selected_key in selected_keys: - if selected_key.resource_type != CompiledResourceType.MODEL: - continue - upstream_key: CompiledObjectKey - for upstream_key in upstream_deps.get(selected_key, ()): - if upstream_key.resource_type == CompiledResourceType.SOURCE: - direct_upstream_sources.add(upstream_key) - return frozenset(key for key in direct_upstream_sources if key.name in managed_source_names) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/buildability.py b/src/sqlbuild/compiler/planner/_helpers/graph/buildability.py deleted file mode 100644 index 7bd34e5d8e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/buildability.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Buildability validation for selected scope against warehouse state.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import MissingUpstream, WarehouseSnapshot - -_MESSAGE_ITEM_LIMIT: int = 5 - - -def check_buildability( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - snapshot: WarehouseSnapshot, - deferred_relations: dict[str, RelationInfo] | None = None, - satisfied_keys: frozenset[CompiledObjectKey] = frozenset(), -) -> tuple[MissingUpstream, ...]: - """Validate that all upstream deps for selected keys exist in scope or warehouse.""" - - missing_map: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - - selected_key: CompiledObjectKey - for selected_key in selected_keys: - dep_keys: tuple[CompiledObjectKey, ...] = upstream_deps.get(selected_key, ()) - dep_key: CompiledObjectKey - for dep_key in dep_keys: - if dep_key.resource_type == CompiledResourceType.SQL_TEST: - continue - if dep_key.resource_type == CompiledResourceType.DBT_REF: - continue - if dep_key.resource_type == CompiledResourceType.SOURCE: - continue - if dep_key in selected_keys: - continue - if dep_key in satisfied_keys: - continue - if dep_key.name in snapshot.existing_relations: - continue - if deferred_relations is not None and dep_key.name in deferred_relations: - continue - missing_map.setdefault(dep_key, []).append(selected_key) - - missing: list[MissingUpstream] = [ - MissingUpstream( - key=key, - required_by=tuple(sorted(dependents, key=lambda k: (k.resource_type, k.name))), - ) - for key, dependents in sorted( - missing_map.items(), - key=lambda item: (-len(item[1]), item[0].resource_type, item[0].name), - ) - ] - - return tuple(missing) - - -def missing_upstream_message( - *, - missing: tuple[MissingUpstream, ...], - edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str], -) -> str: - """Describe missing upstream keys, naming the audit when only an audit read requires one.""" - - lineage_missing: list[MissingUpstream] = [] - audit_reads: dict[str, None] = {} - entry: MissingUpstream - for entry in missing: - origins: tuple[str, ...] = tuple( - edge_origins[(dependent, entry.key)] - for dependent in entry.required_by - if (dependent, entry.key) in edge_origins - ) - if len(origins) < len(entry.required_by): - lineage_missing.append(entry) - continue - audit_reads.update(dict.fromkeys(origins)) - parts: list[str] = [] - if lineage_missing: - names: str = ", ".join(m.key.name for m in lineage_missing[:_MESSAGE_ITEM_LIMIT]) - parts.append(f"{len(lineage_missing)} missing upstream dependencies ({names})") - read_names: tuple[str, ...] = tuple(audit_reads) - if len(read_names) == 1: - parts.append( - f"{read_names[0]}, which is not selected and does not exist in the warehouse; " - "select it or build it first" - ) - elif read_names: - shown: str = "; ".join(read_names[:_MESSAGE_ITEM_LIMIT]) - more: str = "; ..." if len(read_names) > _MESSAGE_ITEM_LIMIT else "" - parts.append( - f"{len(read_names)} audit reads are not selected and do not exist in the warehouse " - f"({shown}{more}); select them or build them first" - ) - return "cannot build selected scope: " + "; ".join(parts) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/core.py b/src/sqlbuild/compiler/planner/_helpers/graph/core.py deleted file mode 100644 index d83d9f744c..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/core.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Graph traversal helpers for planning over compiled project state.""" - -from __future__ import annotations - -import heapq - -from sqlbuild._native import ( - attached_audit_gate_edges, - hook_read_edges, - invert_edges, - native_graph_key, - transitive_closure, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import ( - CompiledProjectView, - CompiledResourceType, - SqlTestMode, -) -from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge -from sqlbuild.compiler.planner.exceptions import PlannerInputError - - -def _key_sort_key(key: CompiledObjectKey) -> tuple[str, str]: - return (key.resource_type, key.name) - - -def build_execution_upstream_deps( - project: CompiledProjectView, -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Return execution-order edges from lineage and woven SQL test nodes.""" - - upstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - upstream.update({model.key: list(model.deps) for model in project.models}) - upstream.update({source.key: list(source.deps) for source in project.sources}) - upstream.update({seed.key: list(seed.deps) for seed in project.seeds}) - upstream.update({function.key: list(function.deps) for function in project.functions}) - - test: CompiledSqlTest - for test in project.sql_tests: - if test.mode == SqlTestMode.TABLE_FN: - upstream[test.key] = list(_function_scope_deps_for_test(test=test)) - continue - upstream[test.key] = list(_function_deps_for_test(test=test, upstream=upstream)) - for target_key in test.scope_deps: - if target_key in upstream: - upstream[target_key].append(test.key) - - edge: AttachedAuditGateEdge - for edge in attached_audit_gate_edges(project=project): - gated_upstream: list[CompiledObjectKey] = upstream.setdefault(edge.gated, []) - if edge.read not in gated_upstream: - gated_upstream.append(edge.read) - - hook_edge: HookReadEdge - for hook_edge in hook_read_edges(project=project): - hook_upstream: list[CompiledObjectKey] = upstream.setdefault(hook_edge.gated, []) - if hook_edge.read not in hook_upstream: - hook_upstream.append(hook_edge.read) - - return {k: tuple(v) for k, v in upstream.items()} - - -def build_execution_edge_origins( - project: CompiledProjectView, -) -> dict[tuple[CompiledObjectKey, CompiledObjectKey], str]: - """Return human-readable origins for execution edges injected beyond pure lineage.""" - - origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str] = {} - test: CompiledSqlTest - for test in project.sql_tests: - for target_key in test.scope_deps: - origins[(target_key, test.key)] = ( - f"SQL test '{test.name}' runs before '{target_key.name}'" - ) - edge: AttachedAuditGateEdge - for edge in attached_audit_gate_edges(project=project): - if _is_lineage_edge(project=project, gated=edge.gated, read=edge.read): - continue - origins.setdefault( - (edge.gated, edge.read), - f"audit '{edge.audit_name}' on '{edge.target.name}' reads '{edge.read.name}'", - ) - hook_edge: HookReadEdge - for hook_edge in hook_read_edges(project=project): - if _is_lineage_edge(project=project, gated=hook_edge.gated, read=hook_edge.read): - continue - origins.setdefault( - (hook_edge.gated, hook_edge.read), - f"{hook_edge.label} on '{hook_edge.gated.name}' reads '{hook_edge.read.name}'", - ) - return origins - - -def _is_lineage_edge( - *, project: CompiledProjectView, gated: CompiledObjectKey, read: CompiledObjectKey -) -> bool: - return native_graph_key(read) in project.lineage_graph.upstream(native_graph_key(gated)) - - -def _function_scope_deps_for_test(*, test: CompiledSqlTest) -> tuple[CompiledObjectKey, ...]: - return tuple( - scope_dep - for scope_dep in test.scope_deps - if scope_dep.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN} - ) - - -def _function_deps_for_test( - *, - test: CompiledSqlTest, - upstream: dict[CompiledObjectKey, list[CompiledObjectKey]], -) -> tuple[CompiledObjectKey, ...]: - deps: list[CompiledObjectKey] = [] - seen: set[CompiledObjectKey] = set() - target_key: CompiledObjectKey - for target_key in test.scope_deps: - dep_key: CompiledObjectKey - for dep_key in upstream.get(target_key, ()): - if dep_key.resource_type not in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - continue - if dep_key in seen: - continue - seen.add(dep_key) - deps.append(dep_key) - return tuple(deps) - - -def build_downstream_deps( - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Return downstream edges keyed by upstream object key.""" - - return invert_edges(edges=upstream, sort_key=_key_sort_key) - - -def topologically_order_keys( - *, - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - injected_edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str] | None = None, -) -> tuple[CompiledObjectKey, ...]: - """Return all keys in stable dependency order using Kahn's algorithm.""" - - node_keys: set[CompiledObjectKey] = set(upstream) - dep_keys: tuple[CompiledObjectKey, ...] - for dep_keys in upstream.values(): - dep_key: CompiledObjectKey - for dep_key in dep_keys: - node_keys.add(dep_key) - - indegree: dict[CompiledObjectKey, int] = {key: 0 for key in node_keys} - key: CompiledObjectKey - for key, dep_keys in upstream.items(): - indegree[key] = len(dep_keys) - - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream - ) - ready: list[tuple[tuple[str, str], CompiledObjectKey]] = [ - (_key_sort_key(key), key) for key, count in indegree.items() if count == 0 - ] - heapq.heapify(ready) - ordered: list[CompiledObjectKey] = [] - - while ready: - current: CompiledObjectKey = heapq.heappop(ready)[1] - ordered.append(current) - downstream_key: CompiledObjectKey - for downstream_key in downstream.get(current, ()): - if downstream_key not in indegree: - continue - indegree[downstream_key] -= 1 - if indegree[downstream_key] == 0: - heapq.heappush(ready, (_key_sort_key(downstream_key), downstream_key)) - - if len(ordered) != len(node_keys): - ordered_set: set[CompiledObjectKey] = set(ordered) - cycle_keys: tuple[CompiledObjectKey, ...] = tuple( - sorted( - (key for key in node_keys if key not in ordered_set), - key=_key_sort_key, - ) - ) - cycle_names: str = ", ".join(f"{k.resource_type}:{k.name}" for k in cycle_keys) - origin_notes: str = _cycle_origin_notes( - cycle_keys=cycle_keys, - upstream=upstream, - injected_edge_origins=injected_edge_origins or {}, - ) - raise PlannerInputError(f"Dependency cycle detected involving: {cycle_names}{origin_notes}") - - return tuple(ordered) - - -def _cycle_origin_notes( - *, - cycle_keys: tuple[CompiledObjectKey, ...], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - injected_edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str], -) -> str: - """Describe injected (non-lineage) edges between cycle keys so users see the closure cause.""" - - if not injected_edge_origins: - return "" - cycle_key_set: set[CompiledObjectKey] = set(cycle_keys) - notes: list[str] = [] - key: CompiledObjectKey - for key in cycle_keys: - dep_key: CompiledObjectKey - for dep_key in upstream.get(key, ()): - if dep_key not in cycle_key_set: - continue - origin: str | None = injected_edge_origins.get((key, dep_key)) - if origin is not None: - notes.append(origin) - if not notes: - return "" - return " (via " + "; ".join(sorted(set(notes))) + ")" - - -def expand_upstream( - *, - key: CompiledObjectKey, - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return all transitive upstream keys reachable from the given key.""" - - return transitive_closure(start=key, edges=upstream) - - -def expand_downstream( - *, - key: CompiledObjectKey, - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return all transitive downstream keys reachable from the given key.""" - - return transitive_closure(start=key, edges=downstream) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py deleted file mode 100644 index f921cf634e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Loader DAG planning helpers.""" - -from __future__ import annotations - -from sqlbuild._native import loader_destination_parts -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry - - -def expand_selected_loader_dependencies( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - executable_dependency_source_keys: frozenset[CompiledObjectKey] = frozenset(), -) -> tuple[frozenset[CompiledObjectKey], dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]]: - """Add intermediate loader nodes needed by selected managed source loads.""" - - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in project.loader_functions - } - source_by_loader: dict[str, SourceEntry] = { - source.source_entry.loader: source.source_entry - for source in project.sources - if source.source_entry.loader is not None - } - expanded_upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = dict(upstream_deps) - expanded_selected: set[CompiledObjectKey] = set(selected_keys) - - source_by_name: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry for source in project.sources - } - selected_key: CompiledObjectKey - for selected_key in selected_keys: - if selected_key.resource_type != CompiledResourceType.SOURCE: - continue - source_entry: SourceEntry | None = source_by_name.get(selected_key.name) - if source_entry is None: - if selected_key.name in loader_by_name: - expanded_upstream.setdefault(selected_key, ()) - continue - if source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(source_entry.loader) - if loader_function is None: - continue - expanded_upstream[selected_key] = _dependency_keys( - loader_function=loader_function, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - expanded_upstream.update( - _dependency_closure_edges( - loader_function=loader_function, - loader_by_name=loader_by_name, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - ) - if selected_key not in executable_dependency_source_keys: - continue - dependency_loader_name: str - for dependency_loader_name in upstream_loader_dependency_names( - loader_function=loader_function, - loader_functions=project.loader_functions, - ): - expanded_selected.add( - _loader_node_key( - loader_name=dependency_loader_name, - source_by_loader=source_by_loader, - ) - ) - - return frozenset(expanded_selected), expanded_upstream - - -def build_intermediate_source_map( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] -) -> dict[str, SourceEntry]: - """Return synthetic source entries for selected intermediate loader nodes.""" - - source_by_loader: dict[str, SourceEntry] = { - source.source_entry.loader: source.source_entry - for source in project.sources - if source.source_entry.loader is not None - } - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - source_names: frozenset[str] = frozenset(source.source_entry.name for source in project.sources) - entries: dict[str, SourceEntry] = {} - key: CompiledObjectKey - for key in selected_keys: - if key.resource_type != CompiledResourceType.SOURCE: - continue - if key.name in entries or key.name in source_names: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(key.name) - if loader_function is None or loader_function.name in source_by_loader: - continue - entries[key.name] = loader_to_source_entry(project=project, loader=loader_function) - return entries - - -def build_upstream_intermediate_source_map( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] -) -> dict[str, SourceEntry]: - """Return synthetic source entries for upstream intermediate loaders of selected sources.""" - - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - source_by_name: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry for source in project.sources - } - source_by_loader: dict[str, SourceEntry] = { - source.source_entry.loader: source.source_entry - for source in project.sources - if source.source_entry.loader is not None - } - entries: dict[str, SourceEntry] = {} - key: CompiledObjectKey - for key in selected_keys: - if key.resource_type != CompiledResourceType.SOURCE: - continue - source_entry: SourceEntry | None = source_by_name.get(key.name) - if source_entry is None or source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(source_entry.loader) - if loader_function is None: - continue - dependency_name: str - for dependency_name in upstream_loader_dependency_names( - loader_function=loader_function, - loader_functions=project.loader_functions, - ): - if dependency_name in source_by_loader: - continue - dependency_loader: DiscoveredLoaderFunction | None = loader_by_name.get(dependency_name) - if dependency_loader is None: - continue - entries.setdefault( - dependency_name, - loader_to_source_entry(project=project, loader=dependency_loader), - ) - return entries - - -def upstream_loader_dependency_names( - *, - loader_function: DiscoveredLoaderFunction, - loader_functions: tuple[DiscoveredLoaderFunction, ...], -) -> tuple[str, ...]: - """Return upstream loader dependency names for one loader function.""" - - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in loader_functions - } - names: list[str] = [] - pending: list[str] = list( - _dependency_loader_names( - loader_function=loader_function, - loader_name_by_function=loader_name_by_function, - ) - ) - while pending: - current: str = pending.pop(0) - if current in names: - continue - names.append(current) - dependency_loader: DiscoveredLoaderFunction | None = loader_by_name.get(current) - if dependency_loader is None: - continue - pending.extend( - _dependency_loader_names( - loader_function=dependency_loader, - loader_name_by_function=loader_name_by_function, - ) - ) - return tuple(names) - - -def loader_to_source_entry( - *, project: CompiledProjectView, loader: DiscoveredLoaderFunction -) -> SourceEntry: - destination: LoaderDestinationParts = loader_destination_parts( - destination=( - loader.destination if loader.destination is not None else f"__loader__{loader.name}" - ), - default_database=project.effective_target_database, - default_schema=project.effective_target_schema, - ) - return SourceEntry( - name=loader.name, - database=destination.database, - schema=destination.schema, - table=destination.table, - loader=loader.name, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - contract=loader.contract, - meta={"sqlbuild_loader_node": True}, - columns=loader.columns, - ) - - -def _dependency_closure_edges( - *, - loader_function: DiscoveredLoaderFunction, - loader_by_name: dict[str, DiscoveredLoaderFunction], - loader_name_by_function: dict[object, str], - source_by_loader: dict[str, SourceEntry], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - edges: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = {} - dependency_name: str - for dependency_name in _dependency_loader_names( - loader_function=loader_function, loader_name_by_function=loader_name_by_function - ): - dependency_key: CompiledObjectKey = _loader_node_key( - loader_name=dependency_name, source_by_loader=source_by_loader - ) - dependency_function: DiscoveredLoaderFunction = loader_by_name[dependency_name] - edges[dependency_key] = _dependency_keys( - loader_function=dependency_function, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - edges.update( - _dependency_closure_edges( - loader_function=dependency_function, - loader_by_name=loader_by_name, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - ) - return edges - - -def _dependency_keys( - *, - loader_function: DiscoveredLoaderFunction, - loader_name_by_function: dict[object, str], - source_by_loader: dict[str, SourceEntry], -) -> tuple[CompiledObjectKey, ...]: - return tuple( - _loader_node_key(loader_name=loader_name, source_by_loader=source_by_loader) - for loader_name in _dependency_loader_names( - loader_function=loader_function, - loader_name_by_function=loader_name_by_function, - ) - ) - - -def _dependency_loader_names( - *, - loader_function: DiscoveredLoaderFunction, - loader_name_by_function: dict[object, str], -) -> tuple[str, ...]: - return tuple( - loader_name_by_function[dependency] - for dependency in loader_function.depends_on - if dependency in loader_name_by_function - ) - - -def _loader_node_key( - *, loader_name: str, source_by_loader: dict[str, SourceEntry] -) -> CompiledObjectKey: - source_entry: SourceEntry | None = source_by_loader.get(loader_name) - return CompiledObjectKey( - resource_type=CompiledResourceType.SOURCE, - name=source_entry.name if source_entry is not None else loader_name, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py b/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py deleted file mode 100644 index a64410fd46..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Pure model-name closure helpers for planner dependency graphs.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure_many -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType - - -def build_downstream_model_name_closure( - *, - start_keys: tuple[CompiledObjectKey, ...], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - """Return model names reachable downstream from the given keys, including model roots.""" - - return _build_model_name_closure(start_keys=start_keys, deps=downstream_deps) - - -def _build_model_name_closure( - *, - start_keys: tuple[CompiledObjectKey, ...], - deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - return frozenset( - key.name - for key in transitive_closure_many(starts=start_keys, edges=deps, include_starts=True) - if key.resource_type == CompiledResourceType.MODEL - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py deleted file mode 100644 index 1bca778617..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Planner scope resolution helpers.""" - -from __future__ import annotations - -from sqlbuild._native import lineage_graph_names -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_edge_origins, - build_execution_upstream_deps, - topologically_order_keys, -) -from sqlbuild.compiler.planner._helpers.graph.loader_dag import expand_selected_loader_dependencies -from sqlbuild.compiler.planner._helpers.graph.selectors import ( - parse_selector, - resolve_graph_selectors, -) -from sqlbuild.compiler.planner.models import ( - ParsedSelector, - PathSelector, - PlannerScope, - SqlTestSelection, -) -from sqlbuild.compiler.planner.types import SelectorKind - - -def build_planner_scope( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], - auto_load_sources: bool, - selected_keys: frozenset[CompiledObjectKey] | None = None, - python_read_source_names: frozenset[str] = frozenset(), - sql_test_selection: SqlTestSelection | None = None, -) -> PlannerScope: - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream_deps - ) - all_keys: dict[str, CompiledObjectKey] = lineage_graph_names(project.lineage_graph) - resolved_selected_keys: frozenset[CompiledObjectKey] = ( - selected_keys - if selected_keys is not None - else resolve_graph_selectors(graph=project.lineage_graph, select=select, exclude=exclude) - ) - executable_dependency_source_keys: frozenset[CompiledObjectKey] = ( - _upstream_source_selector_keys( - select=select, - all_keys=all_keys, - selected_keys=resolved_selected_keys, - ) - ) - if auto_load_sources: - auto_loaded_source_keys: frozenset[CompiledObjectKey] = managed_source_upstream_keys( - selected_keys=resolved_selected_keys, - upstream_deps=upstream_deps, - project=project, - ) - resolved_selected_keys = resolved_selected_keys | auto_loaded_source_keys - executable_dependency_source_keys = ( - executable_dependency_source_keys | auto_loaded_source_keys - ) - executable_dependency_source_keys = executable_dependency_source_keys & resolved_selected_keys - if auto_load_sources or any( - key.resource_type == CompiledResourceType.SOURCE for key in resolved_selected_keys - ): - resolved_selected_keys, upstream_deps = expand_selected_loader_dependencies( - project=project, - selected_keys=resolved_selected_keys, - upstream_deps=upstream_deps, - executable_dependency_source_keys=executable_dependency_source_keys, - ) - downstream_deps = build_downstream_deps(upstream_deps) - return PlannerScope( - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - all_keys=all_keys, - models_by_name={model.name: model for model in project.models}, - selected_keys=resolved_selected_keys, - execution_order=topologically_order_keys( - upstream=upstream_deps, - injected_edge_origins=build_execution_edge_origins(project), - ), - user_selected_keys=resolved_selected_keys, - python_read_source_names=python_read_source_names, - sql_test_selection=sql_test_selection or SqlTestSelection(), - ) - - -def _upstream_source_selector_keys( - *, - select: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - selected_keys: frozenset[CompiledObjectKey], -) -> frozenset[CompiledObjectKey]: - keys: set[CompiledObjectKey] = set() - raw_select: str - for raw_select in select: - token: str - for token in raw_select.split(): - part: str - for part in token.split(","): - parsed: ParsedSelector | PathSelector = parse_selector(part) - if isinstance(parsed, PathSelector) or not parsed.upstream: - continue - if parsed.kind not in {SelectorKind.NAME, SelectorKind.SOURCE}: - continue - key: CompiledObjectKey | None = all_keys.get(parsed.value) - if key is not None and key.resource_type == CompiledResourceType.SOURCE: - keys.add(key) - return frozenset(key for key in keys if key in selected_keys) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py b/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py deleted file mode 100644 index 9d6412f3fe..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Planner selector boundary over the native selector grammar and graph resolution.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable, Iterator, Mapping - -import sqlbuild._native as _native -from sqlbuild._native import compiled_graph_keys, native_graph_from_views, native_graph_keys -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.constants import ( - UNIT_TEST_SELECTOR_ERROR_CODE, - UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind - -_KIND_SELECTOR: str = "kind" -_SURROGATE_START: int = 0xD800 -_SURROGATE_END: int = 0xDFFF -_PRIVATE_USE_START: int = 0xF0000 -_PRIVATE_USE_END: int = 0xFFFFE - -type _Edges = Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] -type _Failure = tuple[str, str, str | None] -type _Outcome = tuple[list[tuple[str, str]] | None, _Failure | None] - - -class _SurrogateCarrier: - """Carry lone surrogates through the UTF-8 grammar as unused private-use characters.""" - - def __init__(self, *, texts: Iterable[str], reserved: Callable[[], Iterable[str]]) -> None: - joined: str = "".join(texts) - surrogates: list[str] = sorted( - {character for character in joined if _is_surrogate(character)} - ) - self._sent: dict[str, str] = {} - self._back: dict[str, str] = {} - if not surrogates: - return - taken: set[str] = set(joined).union(*map(set, reserved())) - candidates: Iterator[str] = ( - chr(point) for point in range(_PRIVATE_USE_START, _PRIVATE_USE_END) - ) - for surrogate, carrier in zip( - surrogates, - (character for character in candidates if character not in taken), - strict=False, - ): - self._sent[surrogate] = carrier - self._back[carrier] = surrogate - - def send(self, text: str) -> str: - return text.translate(str.maketrans(self._sent)) if self._sent else text - - def back(self, text: str | None) -> str | None: - if text is None or not self._back: - return text - return text.translate(str.maketrans(self._back)) - - def failure(self, failure: _Failure | None) -> _Failure | None: - if failure is None: - return None - code, message, help_text = failure - return (code, self.back(message) or "", self.back(help_text)) - - -def _is_surrogate(character: str) -> bool: - return _SURROGATE_START <= ord(character) <= _SURROGATE_END - - -def _graph_names(graph: _native.NativeProjectGraph) -> Callable[[], Iterable[str]]: - return lambda: (name for name, _ in graph.names()) - - -def parse_selector(raw: str) -> ParsedSelector | PathSelector: - """Parse one raw selector token into a structured form.""" - - carrier: _SurrogateCarrier = _SurrogateCarrier(texts=(raw,), reserved=tuple) - parsed, failure = _native.parse_project_selector(carrier.send(raw)) - if parsed is None: - raise _selector_error(carrier.failure(failure)) - shape, first, second, upstream, downstream = parsed - if shape == _KIND_SELECTOR: - return ParsedSelector( - kind=SelectorKind(first), - value=carrier.back(second) or "", - upstream=upstream, - downstream=downstream, - ) - return PathSelector( - start_name=carrier.back(first) or "", - end_name=carrier.back(second) or "", - upstream=upstream, - downstream=downstream, - ) - - -def resolve_graph_selectors( - *, - graph: _native.NativeProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[CompiledObjectKey]: - """Resolve select/exclude strings against a native graph, adding required functions.""" - - carrier: _SurrogateCarrier = _SurrogateCarrier( - texts=(*select, *exclude), reserved=_graph_names(graph) - ) - return _selected( - outcome=graph.resolve(list(map(carrier.send, select)), list(map(carrier.send, exclude))), - carrier=carrier, - ) - - -def resolve_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_keys: Mapping[str, CompiledObjectKey], - upstream: _Edges, - downstream: _Edges, - tag_index: Mapping[str, Iterable[CompiledObjectKey]] | None = None, - path_index: Mapping[CompiledObjectKey, str] | None = None, -) -> frozenset[CompiledObjectKey]: - """Resolve raw select/exclude strings into a final set of object keys.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index or {}, - path_index=path_index or {}, - ) - return resolve_graph_selectors(graph=graph, select=select, exclude=exclude) - - -def resolve_selector_tokens( - *, - selectors: tuple[str, ...], - all_keys: Mapping[str, CompiledObjectKey], - upstream: _Edges, - downstream: _Edges, - tag_index: Mapping[str, Iterable[CompiledObjectKey]], - path_index: Mapping[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Union the keys matched by whitespace-separated selector tokens, without build expansion.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - carrier: _SurrogateCarrier = _SurrogateCarrier(texts=selectors, reserved=_graph_names(graph)) - return _selected(outcome=graph.tokens(list(map(carrier.send, selectors))), carrier=carrier) - - -def match_selector_keys( - *, - parsed: ParsedSelector, - all_keys: Mapping[str, CompiledObjectKey], - tag_index: Mapping[str, Iterable[CompiledObjectKey]], - path_index: Mapping[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Return the keys one parsed selector matches before `+` graph expansion.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys=all_keys, upstream={}, downstream={}, tag_index=tag_index, path_index=path_index - ) - carrier: _SurrogateCarrier = _SurrogateCarrier( - texts=(parsed.value,), reserved=_graph_names(graph) - ) - return _selected( - outcome=graph.matched(str(parsed.kind), carrier.send(parsed.value)), carrier=carrier - ) - - -def expand_required_build_resources( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream: _Edges, - downstream: _Edges, - include_upstream_functions: bool = True, - include_upstream_seeds: bool = False, - include_downstream_functions: bool = False, -) -> frozenset[CompiledObjectKey]: - """Add non-model resources needed to build a coherent selected model scope.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys={}, upstream=upstream, downstream=downstream, tag_index={}, path_index={} - ) - return compiled_graph_keys( - graph.build_resources( - native_graph_keys(selected_keys), - (include_upstream_functions, include_upstream_seeds, include_downstream_functions), - ) - ) - - -def selector_name_help(*, value: str, candidates: Iterable[str]) -> str | None: - """Suggest the nearest selectable names for an unknown selector name.""" - - names: list[str] = list(candidates) - carrier: _SurrogateCarrier = _SurrogateCarrier(texts=(value,), reserved=lambda: names) - return carrier.back(_native.project_selector_name_help(carrier.send(value), names)) - - -def unit_test_selector_rejection(*, selector: str) -> PlannerInputError: - """The error for a unit-test selector given to a command that does not run unit tests.""" - - return PlannerInputError( - f"selector '{selector}' selects a unit test; {UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD}", - code=UNIT_TEST_SELECTOR_ERROR_CODE, - ) - - -def _selected(*, outcome: _Outcome, carrier: _SurrogateCarrier) -> frozenset[CompiledObjectKey]: - keys, failure = outcome - if keys is None: - raise _selector_error(carrier.failure(failure)) - return compiled_graph_keys(keys) - - -def _selector_error(failure: _Failure | None) -> PlannerInputError: - if failure is None: - return PlannerInputError("selector resolution returned neither keys nor an error") - code, message, help_text = failure - return PlannerInputError(message, code=code, help=help_text) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py b/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py deleted file mode 100644 index 1544e1189e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Planner helpers for executable source-load nodes.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure_many -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_intermediate_source_map, - build_upstream_intermediate_source_map, -) -from sqlbuild.compiler.planner.models import SourceLoadPlanEntry -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_source_load_map( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, -) -> dict[str, SourceEntry]: - """Return source entries available to source-load and source-read planning.""" - - relevant_keys: frozenset[CompiledObjectKey] | None = ( - transitive_closure_many(starts=selected_keys, edges=upstream_deps, include_starts=True) - if upstream_deps is not None - else None - ) - source_map: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry - for source in project.sources - if relevant_keys is None or source.key in relevant_keys - } - source_map.update(build_intermediate_source_map(project=project, selected_keys=selected_keys)) - source_map.update( - build_upstream_intermediate_source_map(project=project, selected_keys=selected_keys) - ) - return source_map - - -def build_source_load_entries( - *, - execution_order: tuple[CompiledObjectKey, ...], - selected_keys: frozenset[CompiledObjectKey], - source_map: dict[str, SourceEntry], - is_reload: bool, -) -> tuple[SourceLoadPlanEntry, ...]: - """Return executable source-load plan entries in DAG execution order.""" - - entries: list[SourceLoadPlanEntry] = [] - key: CompiledObjectKey - for key in execution_order: - if key not in selected_keys: - continue - if key.resource_type != CompiledResourceType.SOURCE: - continue - source_entry: SourceEntry | None = source_map.get(key.name) - if source_entry is None or source_entry.loader is None: - continue - entries.append( - SourceLoadPlanEntry( - key=key, - name=source_entry.name, - loader=source_entry.loader, - destination=source_entry.table or source_entry.name, - resource_kind=_source_load_resource_kind(source_entry), - write_strategy=source_entry.write_strategy, - cursor_column=source_entry.cursor_column, - unique_key=source_entry.unique_key, - is_reload=is_reload, - integration_kind=( - source_entry.integration_loader.kind - if source_entry.integration_loader is not None - else None - ), - ) - ) - return tuple(entries) - - -def _source_load_resource_kind(source_entry: SourceEntry) -> ExecutionResourceKind: - return ( - ExecutionResourceKind.LOADER - if source_entry.meta.get("sqlbuild_loader_node") is True - else ExecutionResourceKind.SOURCE - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py deleted file mode 100644 index 44b861581e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Direct planner model version identity helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledResourceType, - CompiledSeedView, -) -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - build_model_local_identity_hash, - compose_native_graph_identity, - graph_key_for_compiled_resource, - model_definition_hash, -) -from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity -from sqlbuild.compiler.planner.main.identity._graph_identity import ( - build_expected_graph_identity_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) -from sqlbuild.compiler.planner.models import ( - DirectModelVersionIdentities, - GraphIdentityNode, - GraphNodeKey, - PlannerScope, -) -from sqlbuild.compiler.planner.types import GraphResourceKind -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities - - -def build_direct_model_version_identities( - *, - functions: tuple[CompiledFunctionView, ...], - seeds: tuple[CompiledSeedView, ...] = (), - scope: PlannerScope, - source_version_hashes: dict[str, str] | None = None, - hook_functions: tuple[DiscoveredHookFunction, ...] = (), - dialect: str | None, -) -> DirectModelVersionIdentities: - """Compute current direct model identities from code and upstream identities.""" - - function_local_hashes: dict[str, str] = build_function_local_hashes( - functions=functions, dialect=dialect - ) - hook_version_hashes: dict[str, str] = { - name: identity.version_hash - for name, identity in build_hook_identities(hook_functions).items() - } - seed_version_hashes: dict[str, str] = {} - seed_metadata_jsons: dict[str, str] = {} - seed: CompiledSeedView - for seed in seeds: - seed_hash: str - seed_metadata_json: str - seed_hash, seed_metadata_json = build_seed_identity(seed) - seed_version_hashes[seed.name] = seed_hash - seed_metadata_jsons[seed.name] = seed_metadata_json - model_metadata_jsons: dict[str, str] = {} - model_local_hashes: dict[str, str] = {} - graph_nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - function: CompiledFunctionView - for function in functions: - function_hash: str | None = function_local_hashes.get(function.name) - if function_hash is None: - continue - function_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=function.key.resource_type, - name=function.name, - ) - graph_nodes[function_key] = GraphIdentityNode( - key=function_key, - resource_kind=GraphResourceKind.FUNCTION, - upstream_keys=tuple( - graph_key_for_compiled_resource( - resource_type=dep.resource_type, - name=dep.name, - ) - for dep in function.deps - ), - local_hash=function_hash, - ) - seed_name: str - for seed_name, seed_hash in seed_version_hashes.items(): - seed_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - graph_nodes[seed_key] = GraphIdentityNode( - key=seed_key, - resource_kind=GraphResourceKind.SEED, - upstream_keys=(), - local_hash=seed_hash, - ) - source_hashes: dict[str, str] = source_version_hashes or {} - source_name: str - source_hash: str - for source_name, source_hash in source_hashes.items(): - source_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SOURCE, - name=source_name, - ) - graph_nodes[source_key] = GraphIdentityNode( - key=source_key, - resource_kind=GraphResourceKind.SOURCE, - upstream_keys=(), - local_hash=source_hash, - ) - - key: object - for key in scope.execution_order: - if not hasattr(key, "resource_type") or key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if model is None: - continue - metadata_json: str = build_model_version_identity_metadata_json( - model=model, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, - ) - model_metadata_jsons[model.name] = metadata_json - local_hash: str = build_model_local_identity_hash( - query_fingerprint=model_definition_hash( - model_name=model.name, query_sql=model.query_sql, dialect=dialect - ), - metadata_json=metadata_json, - ) - model_local_hashes[model.name] = local_hash - model_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model.name, - ) - graph_nodes[model_key] = GraphIdentityNode( - key=model_key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=tuple( - graph_key_for_compiled_resource(resource_type=dep.resource_type, name=dep.name) - for dep in model.deps - ), - local_hash=local_hash, - ) - - graph_hashes: dict[GraphNodeKey, str | None] = build_expected_graph_identity_hashes( - nodes=graph_nodes, - execution_order=tuple( - graph_key_for_compiled_resource(resource_type=key.resource_type, name=key.name) - for key in scope.execution_order - if hasattr(key, "resource_type") - ), - compose_identity=compose_native_graph_identity, - ) - model_version_hashes: dict[str, str] = {**function_local_hashes, **seed_version_hashes} - model_name: str - for model_name in model_local_hashes: - model_hash: str | None = graph_hashes.get( - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model_name, - ) - ) - if model_hash is not None: - model_version_hashes[model_name] = model_hash - - return DirectModelVersionIdentities( - function_local_hashes=function_local_hashes, - seed_version_hashes=seed_version_hashes, - seed_metadata_jsons=seed_metadata_jsons, - model_metadata_jsons=model_metadata_jsons, - model_local_hashes=model_local_hashes, - model_version_hashes=model_version_hashes, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py deleted file mode 100644 index ac2bb6b279..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Function fingerprint source helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - FunctionArgument, - FunctionReturnColumn, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash -from sqlbuild.compiler.planner.types import PlanReason - - -def build_compiled_function_fingerprint_sql(function: CompiledFunctionView) -> str: - """Build stable function definition text used for change fingerprints.""" - - return build_function_fingerprint_sql( - arguments=function.arguments, - returns=function.returns, - return_columns=function.return_columns, - body_sql=function.body_sql, - language=str(function.language), - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - - -def detect_function_change( - *, - function: CompiledFunctionView, - fingerprint_sql: str, - fingerprint: Fingerprint | None, - query_change_tracking: bool, - full_refresh: bool, - dialect: str | None, -) -> PlanReason: - """Resolve why a function is redeployed; callers decide their own replay.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - if fingerprint is None: - return PlanReason.FIRST_RUN - if query_change_tracking and ( - function_definition_hash( - function_name=function.name, - fingerprint_sql=fingerprint_sql, - language=str(function.language), - dialect=dialect, - ) - != fingerprint.definition_hash - ): - return PlanReason.QUERY_CHANGED - return PlanReason.NO_CHANGE - - -def build_function_fingerprint_sql( - *, - arguments: tuple[FunctionArgument | object, ...], - returns: str, - body_sql: str, - language: str, - runtime_version: str | None, - entry_point: str | None, - packages: tuple[str, ...], - return_columns: tuple[FunctionReturnColumn | object, ...] = (), -) -> str: - """Build stable function definition text used for change fingerprints.""" - - rendered_arguments: str = ",".join(_render_argument(arg) for arg in arguments) - rendered_return_columns: str = ",".join( - _render_return_column(column) for column in return_columns - ) - rendered_packages: str = ",".join(packages) - return "\n".join( - ( - f"language={language}", - f"arguments={rendered_arguments}", - f"returns={returns}", - f"return_columns={rendered_return_columns}", - f"runtime_version={runtime_version or ''}", - f"entry_point={entry_point or ''}", - f"packages={rendered_packages}", - "body=", - body_sql, - ) - ) - - -def _render_argument(argument: FunctionArgument | object) -> str: - name: object | None = getattr(argument, "name", None) - arg_type: object | None = getattr(argument, "type", None) - return f"{name or ''}:{arg_type or ''}" - - -def _render_return_column(column: FunctionReturnColumn | object) -> str: - name: object | None = getattr(column, "name", None) - col_type: object | None = getattr(column, "type", None) - return f"{name or ''}:{col_type or ''}" diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/graph.py b/src/sqlbuild/compiler/planner/_helpers/identity/graph.py deleted file mode 100644 index be520d01e0..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/graph.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Pure graph identity resolution helpers.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner.classes.graph_identity_resolver import GraphIdentityResolver -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - - -def build_expected_graph_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str | None]: - resolver: GraphIdentityResolver = GraphIdentityResolver( - nodes=nodes, compose_identity=compose_identity, base_hashes={}, selected_keys=None - ) - key: GraphNodeKey - for key in execution_order: - _ = resolver.resolve_expected(key) - return resolver.hashes - - -def build_graph_write_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - selected_keys: frozenset[GraphNodeKey], - base_identity_hashes: Mapping[GraphNodeKey, str], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str]: - resolver: GraphIdentityResolver = GraphIdentityResolver( - nodes=nodes, - compose_identity=compose_identity, - base_hashes=base_identity_hashes, - selected_keys=selected_keys, - ) - key: GraphNodeKey - for key in execution_order: - _ = resolver.resolve_write(key) - return {hashed_key: value for hashed_key, value in resolver.hashes.items() if value is not None} diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py b/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py deleted file mode 100644 index 6160a72aac..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Model version identity hashing helpers.""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable -from functools import partial - -from sqlbuild._native import ( - compute_function_definition_hash, - compute_query_hash, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.fingerprints.constants import ( - NODE_TYPE_MODEL, - NODE_TYPE_SEED, - NODE_TYPE_TABLE_FN, - NODE_TYPE_UDF, -) -from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError -from sqlbuild.compiler.planner.constants import QUERY_FINGERPRINT_FAILED -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import GraphNodeKey - -_FINGERPRINT_HELP: str = ( - "SQLBuild fingerprints the dialect token stream of compiled SQL to detect changes. " - "Fix the SQL so the warehouse dialect tokenizer accepts it." -) - - -def stable_version_identity_hash(value: str) -> str: - """Return the stable SHA-256 hash used for local and composed identities.""" - - return hashlib.sha256(value.encode("utf-8")).hexdigest() - - -def model_definition_hash(*, model_name: str, query_sql: str, dialect: str | None) -> str: - """Fingerprint a model query for change detection, naming the model when it cannot.""" - - return _planner_fingerprint( - subject=f"Model '{model_name}' SQL", - compute=partial(compute_query_hash, query_sql=query_sql, dialect=dialect), - ) - - -def function_definition_hash( - *, function_name: str, fingerprint_sql: str, language: str, dialect: str | None -) -> str: - """Fingerprint a function definition for change detection, naming it when it cannot.""" - - return _planner_fingerprint( - subject=f"Function '{function_name}'", - compute=partial( - compute_function_definition_hash, - fingerprint_sql=fingerprint_sql, - language=language, - dialect=dialect, - ), - ) - - -def _planner_fingerprint(*, subject: str, compute: Callable[[], str]) -> str: - try: - return compute() - except QueryFingerprintError as error: - raise PlannerInputError( - f"{subject} cannot be fingerprinted for change detection: {error}", - code=QUERY_FINGERPRINT_FAILED, - help=_FINGERPRINT_HELP, - ) from None - - -def build_model_local_identity_hash(*, query_fingerprint: str, metadata_json: str) -> str: - """Build a model's local identity hash from its query fingerprint and non-query metadata.""" - - return stable_version_identity_hash("\n".join((query_fingerprint, metadata_json))) - - -def graph_key_for_compiled_resource( - *, resource_type: str | CompiledResourceType, name: str -) -> GraphNodeKey: - normalized: CompiledResourceType | None = None - if isinstance(resource_type, CompiledResourceType): - normalized = resource_type - else: - try: - normalized = CompiledResourceType(resource_type) - except ValueError: - normalized = None - node_type: str = ( - resource_type.value if isinstance(resource_type, CompiledResourceType) else resource_type - ) - if normalized == CompiledResourceType.MODEL: - node_type = NODE_TYPE_MODEL - elif normalized == CompiledResourceType.SEED: - node_type = NODE_TYPE_SEED - elif normalized == CompiledResourceType.UDF: - node_type = NODE_TYPE_UDF - elif normalized == CompiledResourceType.TABLE_FN: - node_type = NODE_TYPE_TABLE_FN - return GraphNodeKey(node_type=node_type, node_name=name) - - -def compose_native_graph_identity( - *, local_hash: str, upstream_hashes: tuple[tuple[GraphNodeKey, str], ...] -) -> str: - return stable_version_identity_hash( - "\n".join((local_hash, *(upstream_hash for _, upstream_hash in upstream_hashes))) - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py b/src/sqlbuild/compiler/planner/_helpers/identity/honest.py deleted file mode 100644 index 2e8cb7ee6f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Build direct model write hashes from actually available upstream identities.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - compose_native_graph_identity, - graph_key_for_compiled_resource, -) -from sqlbuild.compiler.planner.main.identity._graph_write_identity import ( - build_graph_write_identity_hashes, -) -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DirectModelVersionIdentities, - GraphIdentityNode, - GraphNodeKey, - PlannerChangeResults, - PlannerResolvedActions, - PlannerScope, - ResolvedModelAction, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import GraphResourceKind - - -def with_honest_model_write_hashes( - *, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - changes: PlannerChangeResults, - version_identities: DirectModelVersionIdentities, - available_model_hashes: dict[str, str] | None = None, -) -> PlannerChangeResults: - """Replace planned write hashes with hashes based on upstreams available this run.""" - - base_hashes: dict[GraphNodeKey, str] = _base_write_hashes( - scope=scope, - snapshot=snapshot, - version_identities=version_identities, - available_model_hashes=available_model_hashes, - ) - graph_nodes: dict[GraphNodeKey, GraphIdentityNode] = _graph_identity_nodes( - scope=scope, - version_identities=version_identities, - ) - write_hashes: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=graph_nodes, - execution_order=tuple( - graph_key_for_compiled_resource(resource_type=key.resource_type, name=key.name) - for key in scope.execution_order - ), - selected_keys=frozenset( - graph_key_for_compiled_resource(resource_type=key.resource_type, name=key.name) - for key in scope.selected_keys - ), - base_identity_hashes=base_hashes, - compose_identity=compose_native_graph_identity, - ) - models_by_name: dict[str, CompiledModelView] = scope.models_by_name - model_changes: dict[str, ChangeDetectionResult] = dict(changes.models) - key: CompiledObjectKey - for key in scope.execution_order: - if key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = models_by_name.get(key.name) - if model is None: - continue - if key not in scope.selected_keys: - continue - local_hash: str | None = version_identities.model_local_hashes.get(model.name) - if local_hash is None: - continue - graph_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model.name, - ) - write_hash: str | None = write_hashes.get(graph_key) - if write_hash is None: - continue - change: ChangeDetectionResult | None = model_changes.get(model.name) - if change is None: - continue - model_changes[model.name] = replace(change, fingerprint_version_hash=write_hash) - return replace(changes, models=model_changes) - - -def merge_recomputed_model_changes( - *, - resolved_actions: PlannerResolvedActions, - changes: PlannerChangeResults, -) -> PlannerResolvedActions: - """Patch recomputed model changes into resolved actions, keeping run-despite-unchanged kinds.""" - - merged_models: dict[str, ResolvedModelAction] = {} - model_name: str - resolved: ResolvedModelAction - for model_name, resolved in resolved_actions.models.items(): - merged_models[model_name] = replace( - resolved, - change=_merged_recomputed_change( - resolved_change=resolved.change, - recomputed_change=changes.models.get(model_name), - ), - ) - return replace(resolved_actions, models=merged_models) - - -def _merged_recomputed_change( - *, - resolved_change: ChangeDetectionResult, - recomputed_change: ChangeDetectionResult | None, -) -> ChangeDetectionResult: - """Prefer a recomputed change when available.""" - - if recomputed_change is None: - return resolved_change - return recomputed_change - - -def _graph_identity_nodes( - *, scope: PlannerScope, version_identities: DirectModelVersionIdentities -) -> dict[GraphNodeKey, GraphIdentityNode]: - nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - function_name: str - function_hash: str - for function_name, function_hash in version_identities.function_local_hashes.items(): - for resource_type in (CompiledResourceType.UDF, CompiledResourceType.TABLE_FN): - function_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=resource_type, - name=function_name, - ) - nodes[function_key] = GraphIdentityNode( - key=function_key, - resource_kind=GraphResourceKind.FUNCTION, - upstream_keys=tuple( - graph_key_for_compiled_resource( - resource_type=dep.resource_type, - name=dep.name, - ) - for dep in scope.upstream_deps.get( - CompiledObjectKey(resource_type=resource_type, name=function_name), - (), - ) - ), - local_hash=function_hash, - ) - seed_name: str - seed_hash: str - for seed_name, seed_hash in version_identities.seed_version_hashes.items(): - seed_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - nodes[seed_key] = GraphIdentityNode( - key=seed_key, - resource_kind=GraphResourceKind.SEED, - upstream_keys=(), - local_hash=seed_hash, - ) - model: CompiledModelView - for model in scope.models_by_name.values(): - model_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model.name, - ) - nodes[model_key] = GraphIdentityNode( - key=model_key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=tuple( - graph_key_for_compiled_resource(resource_type=dep.resource_type, name=dep.name) - for dep in model.deps - ), - local_hash=version_identities.model_local_hashes.get(model.name), - ) - return nodes - - -def _base_write_hashes( - *, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - available_model_hashes: dict[str, str] | None, -) -> dict[GraphNodeKey, str]: - hashes: dict[GraphNodeKey, str] = {} - function_name: str - function_hash: str - for function_name, function_hash in version_identities.function_local_hashes.items(): - for resource_type in (CompiledResourceType.UDF, CompiledResourceType.TABLE_FN): - hashes[ - graph_key_for_compiled_resource(resource_type=resource_type, name=function_name) - ] = function_hash - seed_name: str - seed_hash: str - for seed_name, seed_hash in _seed_write_hashes( - scope=scope, - snapshot=snapshot, - version_identities=version_identities, - ).items(): - hashes[ - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - ] = seed_hash - model_name: str - model_hash: str - for model_name, model_hash in _built_model_hashes(snapshot).items(): - hashes[ - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model_name, - ) - ] = model_hash - for model_name, model_hash in (available_model_hashes or {}).items(): - hashes[ - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model_name, - ) - ] = model_hash - return hashes - - -def _seed_write_hashes( - *, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, -) -> dict[str, str]: - hashes: dict[str, str] = {} - seed_name: str - expected_hash: str - for seed_name, expected_hash in version_identities.seed_version_hashes.items(): - seed_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SEED, name=seed_name - ) - if seed_key in scope.selected_keys: - hashes[seed_name] = expected_hash - continue - fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) - hashes[seed_name] = fingerprint.version_hash if fingerprint is not None else expected_hash - return hashes - - -def _built_model_hashes(snapshot: WarehouseSnapshot) -> dict[str, str]: - return { - model_name: fingerprint.version_hash - for model_name, fingerprint in snapshot.fingerprints.models.items() - if fingerprint.version_hash - } diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py b/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py deleted file mode 100644 index 00ec07931c..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Model contract metadata used by version identity.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import EnumDeclaration -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry - - -def contract_output_signature(*, model: CompiledModelView) -> dict[str, object] | None: - """Build required model output metadata that participates in execution identity.""" - - schema_entry: SchemaModelEntry | None = model.schema_entry - if schema_entry is None or not (schema_entry.columns or schema_entry.dynamic_columns): - return None - enforced: bool = model.config.values.get("contract") == ContractPolicy.ENFORCED - if not enforced and schema_entry.model_schema is None: - return None - signature: dict[str, object] = { - "enforced": enforced, - "columns": [ - _column_output_signature(model=model, column=column) for column in schema_entry.columns - ], - } - if schema_entry.dynamic_columns: - signature["dynamic_columns"] = [ - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - } - for family in schema_entry.dynamic_columns - ] - return signature - - -def _column_output_signature( - *, model: CompiledModelView, column: SchemaColumn -) -> dict[str, object]: - signature: dict[str, object] = { - "name": column.name, - "type": column.type, - "nullable": column.nullable, - } - enum_declaration: EnumDeclaration | None = model.enum_columns.get(column.name) - if enum_declaration is not None: - signature["enum"] = { - "name": enum_declaration.name, - "members": [ - {"name": member.name, "value": member.value} for member in enum_declaration.members - ], - } - return signature - - -def declared_columns_hash(*, model: CompiledModelView) -> str: - """Hash the model's own declared columns, column families, and contract setting.""" - - schema_entry: SchemaModelEntry | None = model.schema_entry - payload: dict[str, object] = { - "contract": model.config.values.get("contract"), - "columns": [ - {"name": column.name, "type": column.type, "nullable": column.nullable} - for column in (schema_entry.columns if schema_entry is not None else ()) - ], - "dynamic_columns": [ - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - } - for family in (schema_entry.dynamic_columns if schema_entry is not None else ()) - ], - "type_enforcement": schema_entry.type_enforcement if schema_entry is not None else None, - } - return hashlib.sha256( - json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") - ).hexdigest() diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/seed.py b/src/sqlbuild/compiler/planner/_helpers/identity/seed.py deleted file mode 100644 index e957ed8126..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/seed.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Stable seed identity helpers for direct changes-only planning.""" - -from __future__ import annotations - -import csv -import json -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.planner._helpers.identity.hashing import stable_version_identity_hash -from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings - - -def build_seed_identity(seed: CompiledSeedView) -> tuple[str, str]: - """Return ``(identity_hash, metadata_json)`` for one compiled seed.""" - - metadata_json: str = build_seed_identity_metadata_json(seed) - return stable_version_identity_hash(metadata_json), metadata_json - - -def build_seed_identity_metadata_json(seed: CompiledSeedView) -> str: - """Build deterministic seed identity metadata from content and load-affecting config.""" - - payload: dict[str, object] = { - "name": seed.name, - "csv_settings": _normalize_json_value(asdict(seed.schema_entry.csv_settings)), - "columns": [_column_identity(column) for column in seed.schema_entry.columns], - "rows": [] - if seed.external - else _read_seed_rows( - file_path=seed.seed_file.file_path, - csv_settings=seed.schema_entry.csv_settings, - ), - } - return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - - -def _read_seed_rows(*, file_path: Path, csv_settings: SeedCsvSettings) -> list[list[str]]: - encoding: str = csv_settings.encoding or "utf-8-sig" - dialect_options: dict[str, Any] = {} - if csv_settings.delimiter is not None: - dialect_options["delimiter"] = csv_settings.delimiter - if csv_settings.quotechar is not None: - dialect_options["quotechar"] = csv_settings.quotechar - if csv_settings.doublequote is not None: - dialect_options["doublequote"] = csv_settings.doublequote - if csv_settings.escapechar is not None: - dialect_options["escapechar"] = csv_settings.escapechar - if csv_settings.skipinitialspace is not None: - dialect_options["skipinitialspace"] = csv_settings.skipinitialspace - - rows: list[list[str]] = [] - with file_path.open("r", encoding=encoding, newline="") as handle: - reader: Any = csv.reader(handle, **dialect_options) - for row in reader: - rows.append(list(row)) - return rows - - -def _column_identity(column: SchemaColumn) -> dict[str, object]: - return { - "name": column.name, - "type": column.type, - "nullable": column.nullable, - } - - -def _normalize_json_value(value: object) -> object: - if isinstance(value, dict): - return {str(k): _normalize_json_value(v) for k, v in sorted(value.items())} - if isinstance(value, (list, tuple)): - return [_normalize_json_value(v) for v in value] - return value diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index dd3f39880d..c4b55e4ef3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from dataclasses import dataclass, replace -from sqlbuild._native import get_config_str +from sqlbuild._native import get_config_str, model_definition_hash from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType @@ -16,7 +16,6 @@ from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery -from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( identical_renames, rename_hints, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index 5ddc97cc42..d454e09865 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -6,7 +6,11 @@ from collections import defaultdict from typing import Any -from sqlbuild._native import get_config_str +from sqlbuild._native import ( + build_function_local_hashes, + build_model_version_identity_metadata_json, + get_config_str, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -26,12 +30,6 @@ MigrationStateInspection, ) from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import ( ModelMigrationDeclaration, ModelMigrationDiscovery, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 7a9d8f542e..1d7271da0a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( build_model_warnings, + build_source_load_map, cap_microbatch_bounds, check_cursor_type_consistency, count_microbatches, @@ -57,7 +58,6 @@ CompiledSeedView, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( check_full_rebuild_allowed, describe_full_rebuild_cause, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index a8bdb7ae5f..61c803dd83 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -2,7 +2,11 @@ from __future__ import annotations -from sqlbuild._native import project_lineage_views +from sqlbuild._native import ( + build_source_load_entries, + project_lineage_views, + upstream_loader_dependency_names, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -25,8 +29,6 @@ from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_context, ) -from sqlbuild.compiler.planner._helpers.graph.loader_dag import upstream_loader_dependency_names -from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_entries from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner._helpers.output.plan_entry import ( audit_is_selected, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py index fc1eb330af..1e6d2a951d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py @@ -2,13 +2,13 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.buildability import ( +from sqlbuild._native import ( + build_execution_edge_origins, check_buildability, missing_upstream_message, ) -from sqlbuild.compiler.planner._helpers.graph.core import build_execution_edge_origins +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.warehouse.source_tables import ( check_selected_source_tables_exist, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py index 1be7c7f043..30bcab34c4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py @@ -4,12 +4,9 @@ from dataclasses import replace -from sqlbuild._native import detect_model_changes_in_scope +from sqlbuild._native import build_direct_model_version_identities, detect_model_changes_in_scope from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.identity.direct import ( - build_direct_model_version_identities, -) from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index 5b8870b3da..b103f79c02 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -4,15 +4,13 @@ from dataclasses import replace +from sqlbuild._native import build_stale_out_of_selection_warnings from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner._helpers.migrations.entry_fingerprints import ( with_migration_fingerprints, ) from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types -from sqlbuild.compiler.planner._helpers.pruning.selection_staleness import ( - build_stale_out_of_selection_warnings, -) from sqlbuild.compiler.planner._helpers.warehouse.source_freshness_warnings import ( build_source_freshness_unknown_warnings, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py b/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py index 1524814beb..d1070fcc6b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py @@ -2,10 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner._helpers.identity.honest import ( - merge_recomputed_model_changes, - with_honest_model_write_hashes, -) +from sqlbuild._native import merge_recomputed_model_changes, with_honest_model_write_hashes from sqlbuild.compiler.planner.models import ( PlannerChangeReconciliation, PlannerChangeResults, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py index 707f22d76c..22bb97e84b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py @@ -4,10 +4,9 @@ from dataclasses import replace -from sqlbuild._native import transitive_closure_many +from sqlbuild._native import build_planner_scope, transitive_closure_many from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope from sqlbuild.compiler.planner.models import ( PlannerPolicies, PlannerScope, diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py deleted file mode 100644 index 01e499e278..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Pure selection-aware staleness classification helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - SelectionStalenessGraph, - SelectionStalenessNodeKey, - SelectionStalenessWarning, -) - -_STALE_TRIGGER_DISPLAY_LIMIT: int = 5 - - -def format_stale_upstream_warning_message( - *, - model_label: str, - model_name: str, - trigger_label: str, - trigger_names: tuple[str, ...], -) -> str: - """Build a multi-line stale-selection warning: summary, capped bullet list, single hint.""" - - count: int = len(trigger_names) - displayed: tuple[str, ...] = trigger_names[:_STALE_TRIGGER_DISPLAY_LIMIT] - lines: list[str] = [ - f"{model_label} '{model_name}' will build on {count} stale {trigger_label} " - "not selected for rebuild:" - ] - name: str - for name in displayed: - lines.append(f" - {name}") - if count > _STALE_TRIGGER_DISPLAY_LIMIT: - lines.append(f" +{count - _STALE_TRIGGER_DISPLAY_LIMIT} more") - lines.append(f" rebuild the closure to refresh them: --select +{model_name}") - return "\n".join(lines) - - -def classify_selection_staleness_warnings( - graph: SelectionStalenessGraph, -) -> tuple[SelectionStalenessWarning, ...]: - """Return stale warnings for selected models with changed upstreams outside the run set.""" - - warnings: list[SelectionStalenessWarning] = [] - for model_name in sorted(graph.selected_model_names): - triggers: tuple[str, ...] = _changed_upstream_names( - graph=graph, - model_key=SelectionStalenessNodeKey( - resource_type=CompiledResourceType.MODEL.value, - name=model_name, - ), - ) - if triggers: - warnings.append( - SelectionStalenessWarning(model_name=model_name, trigger_names=triggers) - ) - return tuple(warnings) - - -def _changed_upstream_names( - *, - graph: SelectionStalenessGraph, - model_key: SelectionStalenessNodeKey, -) -> tuple[str, ...]: - names: set[str] = set() - visiting: set[SelectionStalenessNodeKey] = set() - stale_by_key: dict[SelectionStalenessNodeKey, bool] = {} - - def visit( - *, - upstream_key: SelectionStalenessNodeKey, - found_names: set[str], - visiting_keys: set[SelectionStalenessNodeKey], - stale_cache: dict[SelectionStalenessNodeKey, bool], - ) -> tuple[ - bool, set[str], set[SelectionStalenessNodeKey], dict[SelectionStalenessNodeKey, bool] - ]: - cached: bool | None = stale_cache.get(upstream_key) - if cached is not None: - return cached, found_names, visiting_keys, stale_cache - if upstream_key in visiting_keys: - return False, found_names, visiting_keys, stale_cache - visiting_keys = visiting_keys | {upstream_key} - is_stale: bool - if upstream_key.resource_type == CompiledResourceType.MODEL.value: - in_run_set: bool = upstream_key.name in graph.run_model_names - own_changed: bool = upstream_key.name in graph.changed_model_names - if own_changed and not in_run_set: - is_stale = True - found_names = found_names | {upstream_key.name} - else: - ancestor_stale: bool = False - parent_key: SelectionStalenessNodeKey - for parent_key in graph.upstream_deps.get(upstream_key, ()): - parent_stale, found_names, visiting_keys, stale_cache = visit( - upstream_key=parent_key, - found_names=found_names, - visiting_keys=visiting_keys, - stale_cache=stale_cache, - ) - ancestor_stale = parent_stale or ancestor_stale - if _run_parent_changed(graph=graph, parent_key=parent_key): - ancestor_stale = True - is_stale = ancestor_stale - if ancestor_stale and not in_run_set and upstream_key != model_key: - found_names = found_names | {upstream_key.name} - elif upstream_key.resource_type == CompiledResourceType.SEED.value: - changed: bool = upstream_key.name in graph.changed_seed_names - if changed and upstream_key.name not in graph.run_seed_names: - found_names = found_names | {upstream_key.name} - is_stale = changed - elif upstream_key.resource_type == CompiledResourceType.SOURCE.value: - changed = upstream_key.name in graph.changed_source_names - if changed and upstream_key.name not in graph.run_source_names: - found_names = found_names | {upstream_key.name} - is_stale = changed - else: - is_stale = False - visiting_keys = visiting_keys - {upstream_key} - return is_stale, found_names, visiting_keys, {**stale_cache, upstream_key: is_stale} - - upstream_key: SelectionStalenessNodeKey - for upstream_key in graph.upstream_deps.get(model_key, ()): - _, names, visiting, stale_by_key = visit( - upstream_key=upstream_key, - found_names=names, - visiting_keys=visiting, - stale_cache=stale_by_key, - ) - return tuple(sorted(names)) - - -def _run_parent_changed( - *, graph: SelectionStalenessGraph, parent_key: SelectionStalenessNodeKey -) -> bool: - if parent_key.resource_type == CompiledResourceType.MODEL.value: - return ( - parent_key.name in graph.run_model_names - and parent_key.name in graph.changed_model_names - ) - if parent_key.resource_type == CompiledResourceType.SEED.value: - return ( - parent_key.name in graph.run_seed_names and parent_key.name in graph.changed_seed_names - ) - if parent_key.resource_type == CompiledResourceType.SOURCE.value: - return ( - parent_key.name in graph.run_source_names - and parent_key.name in graph.changed_source_names - ) - return False diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py deleted file mode 100644 index 74d5c886b7..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Selection-aware stale warning helpers for direct planning.""" - -from __future__ import annotations - -from sqlbuild._native import detect_model_changes -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( - classify_selection_staleness_warnings, - format_stale_upstream_warning_message, -) -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DirectModelVersionIdentities, - PlannerChangeResults, - PlannerScope, - PlanWarning, - SelectionStalenessGraph, - SelectionStalenessNodeKey, - SelectionStalenessWarning, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ChangeKind, WarningSeverity -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult - -_OWN_CHANGE_KINDS: frozenset[ChangeKind] = frozenset( - { - ChangeKind.FIRST_RUN, - ChangeKind.QUERY_CHANGED, - ChangeKind.FUNCTION_CHANGED, - ChangeKind.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED, - } -) - - -def build_stale_out_of_selection_warnings( - *, - original_scope: PlannerScope, - execution_scope: PlannerScope, - changes: PlannerChangeResults, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - source_freshness: DirectSourceFreshnessPlanningResult | None, - include_sources: bool = True, - model_changes_complete: bool = False, -) -> tuple[PlanWarning, ...]: - """Warn for selected models stale through changed upstreams outside the run set.""" - - run_model_names: frozenset[str] = frozenset( - key.name - for key in execution_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - ) - run_seed_names: frozenset[str] = frozenset( - key.name - for key in execution_scope.selected_keys - if key.resource_type == CompiledResourceType.SEED - ) - run_source_names: frozenset[str] = frozenset( - key.name - for key in execution_scope.selected_keys - if key.resource_type == CompiledResourceType.SOURCE - ) - neutral_graph: SelectionStalenessGraph = SelectionStalenessGraph( - upstream_deps=_neutral_upstream_deps(original_scope.upstream_deps), - selected_model_names=frozenset( - key.name - for key in original_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - ), - run_model_names=run_model_names, - run_seed_names=run_seed_names, - run_source_names=run_source_names, - changed_model_names=_changed_model_names( - original_scope=original_scope, - changes=changes, - snapshot=snapshot, - version_identities=version_identities, - model_changes_complete=model_changes_complete, - ), - changed_seed_names=_changed_seed_names( - snapshot=snapshot, - version_identities=version_identities, - ), - changed_source_names=( - _changed_source_names(source_freshness) if include_sources else frozenset() - ), - ) - warnings: list[PlanWarning] = [] - warning: SelectionStalenessWarning - for warning in classify_selection_staleness_warnings(neutral_graph): - warnings.append( - PlanWarning( - model_name=warning.model_name, - severity=WarningSeverity.WARNING, - message=format_stale_upstream_warning_message( - model_label="selected model", - model_name=warning.model_name, - trigger_label="upstream(s)", - trigger_names=warning.trigger_names, - ), - ) - ) - return tuple(warnings) - - -def _neutral_upstream_deps( - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]]: - neutral_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]] = {} - for key, upstream_keys in upstream_deps.items(): - neutral_upstreams: list[SelectionStalenessNodeKey] = [] - for upstream_key in upstream_keys: - neutral_upstreams.append(_neutral_key(upstream_key)) - neutral_deps[_neutral_key(key)] = tuple(neutral_upstreams) - return neutral_deps - - -def _neutral_key(key: CompiledObjectKey) -> SelectionStalenessNodeKey: - return SelectionStalenessNodeKey( - resource_type=_neutral_resource_type(key.resource_type), - name=key.name, - ) - - -def _neutral_resource_type(resource_type: str) -> str: - if resource_type == CompiledResourceType.MODEL: - return "model" - if resource_type == CompiledResourceType.SEED: - return "seed" - if resource_type == CompiledResourceType.SOURCE: - return "source" - return str(resource_type) - - -def _changed_model_names( - *, - original_scope: PlannerScope, - changes: PlannerChangeResults, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - model_changes_complete: bool, -) -> frozenset[str]: - changed: set[str] = { - model_name - for model_name, change in changes.models.items() - if change.change_kind in _OWN_CHANGE_KINDS - } - changed.update( - model_name - for model_name in original_scope.models_by_name - if _model_own_identity_changed( - model_name=model_name, - original_scope=original_scope, - changes=changes, - snapshot=snapshot, - version_identities=version_identities, - model_changes_complete=model_changes_complete, - ) - ) - return frozenset(changed) - - -def _changed_seed_names( - *, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, -) -> frozenset[str]: - return frozenset( - seed_name - for seed_name in version_identities.seed_version_hashes - if _seed_identity_changed( - seed_name=seed_name, - snapshot=snapshot, - version_identities=version_identities, - ) - ) - - -def _changed_source_names( - source_freshness: DirectSourceFreshnessPlanningResult | None, -) -> frozenset[str]: - if source_freshness is None: - return frozenset() - return frozenset(identity.source_name for identity in source_freshness.changed_identities) - - -def _model_own_identity_changed( - *, - model_name: str, - original_scope: PlannerScope, - changes: PlannerChangeResults, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - model_changes_complete: bool, -) -> bool: - selected_change: ChangeDetectionResult | None = changes.models.get(model_name) - if selected_change is not None and ( - model_changes_complete or selected_change.change_kind in _OWN_CHANGE_KINDS - ): - return selected_change.change_kind in _OWN_CHANGE_KINDS - model: CompiledModelView | None = original_scope.models_by_name.get(model_name) - if model is None: - return False - change: ChangeDetectionResult = detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=False, - query_change_tracking=True, - full_refresh=False, - expected_version_hash=version_identities.model_version_hashes.get(model_name), - expected_metadata_json=version_identities.model_metadata_jsons.get(model_name), - ) - return change.change_kind in _OWN_CHANGE_KINDS - - -def _seed_identity_changed( - *, - seed_name: str, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, -) -> bool: - expected_hash: str | None = version_identities.seed_version_hashes.get(seed_name) - if expected_hash is None: - return False - fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) - return fingerprint is None or fingerprint.version_hash != expected_hash diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index 17d3d343e8..b735c1ee9d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -4,7 +4,14 @@ from typing import Any -from sqlbuild._native import get_materialization_type +from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, + build_downstream_deps, + build_execution_upstream_deps, + get_materialization_type, + resolve_graph_selection, + topologically_order_keys, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -16,15 +23,6 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, - topologically_order_keys, -) -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, build_model_locations, @@ -67,7 +65,7 @@ def build_clone_plan_output( downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( upstream_deps ) - selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selectors( + selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selection( graph=project.lineage_graph, select=select, exclude=exclude ) return PlanOutput( diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index 385e1dcffb..9b7e1d46ad 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -7,8 +7,11 @@ from dataclasses import replace from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, + build_execution_upstream_deps, compiled_model_record, render_source_relation, + topologically_order_keys, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo @@ -35,13 +38,6 @@ from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, ) -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_execution_upstream_deps, - topologically_order_keys, -) -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) from sqlbuild.compiler.planner._helpers.output.plan_entry import extract_seed_columns, plan_model from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, diff --git a/src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py b/src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py deleted file mode 100644 index 17b39561dc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Unit-test selectors: split them from resource selectors or reject them where tests never run.""" - -from __future__ import annotations - -from fnmatch import fnmatchcase - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.graph.selectors import ( - parse_selector, - selector_name_help, - unit_test_selector_rejection, -) -from sqlbuild.compiler.planner.constants import ( - UNIT_TEST_SELECTOR_ERROR_CODE, - UNKNOWN_SELECTOR_ERROR_CODE, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ParsedSelector, - PathSelector, - SqlTestSelection, - UnitTestSelectorNames, - UnitTestSelectorSplit, -) -from sqlbuild.compiler.planner.types import SelectorKind - - -def split_discovered_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, - accepts_unit_tests: bool, -) -> UnitTestSelectorSplit: - """Split unit-test selectors from resource selectors using discovered project names.""" - - return split_unit_test_selectors( - select=select, - exclude=exclude, - names=UnitTestSelectorNames( - tests=_unit_test_names(discovered_inputs), - resources=_resource_names(discovered_inputs), - ), - accepts_unit_tests=accepts_unit_tests, - ) - - -def split_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - names: UnitTestSelectorNames, - accepts_unit_tests: bool, -) -> UnitTestSelectorSplit: - """Move unit-test selectors out of resource selectors, or reject them for this command.""" - - selected_tests, resource_select = _split_groups( - groups=select, names=names, accepts_unit_tests=accepts_unit_tests - ) - excluded_tests, resource_exclude = _split_groups( - groups=exclude, names=names, accepts_unit_tests=accepts_unit_tests - ) - return UnitTestSelectorSplit( - select=resource_select, - exclude=resource_exclude, - sql_test_selection=SqlTestSelection( - names=selected_tests, - excluded_names=excluded_tests, - models_selected=not select or bool(resource_select) or not selected_tests, - ), - ) - - -def _split_groups( - *, groups: tuple[str, ...], names: UnitTestSelectorNames, accepts_unit_tests: bool -) -> tuple[frozenset[str], tuple[str, ...]]: - tests: set[str] = set() - kept_groups: list[str] = [] - for group in groups: - tokens: list[str] = group.split() - kept_tokens: list[str] = [] - for token in tokens: - matched, keep = _split_token( - token=token, names=names, accepts_unit_tests=accepts_unit_tests - ) - tests.update(matched) - if keep: - kept_tokens.append(token) - if len(kept_tokens) == len(tokens): - kept_groups.append(group) - elif kept_tokens: - kept_groups.append(" ".join(kept_tokens)) - return frozenset(tests), tuple(kept_groups) - - -def _split_token( - *, token: str, names: UnitTestSelectorNames, accepts_unit_tests: bool -) -> tuple[frozenset[str], bool]: - parts: list[str] = token.split(",") - classified: list[tuple[frozenset[str], bool]] = [ - _classify(part=part, names=names, accepts_unit_tests=accepts_unit_tests) for part in parts - ] - selects_tests: bool = any(matched and not keep for matched, keep in classified) - if len(parts) > 1 and selects_tests: - for part in parts: - parse_selector(part) - raise PlannerInputError( - f"selector '{token}' intersects a unit test with ','; unit tests can only be " - "selected whole, by name", - code=UNIT_TEST_SELECTOR_ERROR_CODE, - ) - if len(parts) > 1: - return frozenset(), True - return classified[0] - - -def _classify( - *, part: str, names: UnitTestSelectorNames, accepts_unit_tests: bool -) -> tuple[frozenset[str], bool]: - """Return the unit tests one selector selects and whether it also selects resources.""" - - try: - parsed: ParsedSelector | PathSelector = parse_selector(part) - except PlannerInputError: - return frozenset(), True - if isinstance(parsed, PathSelector) or parsed.kind not in { - SelectorKind.NAME, - SelectorKind.TEST, - }: - return frozenset(), True - matched_tests: frozenset[str] = _matching(value=parsed.value, names=names.tests) - selects_resources: bool = parsed.kind == SelectorKind.NAME and bool( - _matching(value=parsed.value, names=names.resources) - ) - if selects_resources: - return frozenset(), True - if not matched_tests: - if parsed.kind == SelectorKind.TEST or accepts_unit_tests: - _raise_unknown(parsed=parsed, names=names) - return frozenset(), True - if not accepts_unit_tests: - raise unit_test_selector_rejection(selector=part) - if parsed.upstream or parsed.downstream: - raise PlannerInputError( - f"selector '{part}' expands a unit test with '+'; unit tests have no lineage, so " - "select them by name without '+'", - code=UNIT_TEST_SELECTOR_ERROR_CODE, - ) - return matched_tests, False - - -def _raise_unknown(*, parsed: ParsedSelector, names: UnitTestSelectorNames) -> None: - label: str = "pattern" if _is_pattern(parsed.value) else "name" - candidates: frozenset[str] = names.tests - subject: str = "unit test" - if parsed.kind == SelectorKind.NAME: - candidates = names.tests | names.resources - subject = "selector" - raise PlannerInputError( - f"unknown {subject} {label} '{parsed.value}'", - code=UNKNOWN_SELECTOR_ERROR_CODE, - help=selector_name_help(value=parsed.value, candidates=candidates), - ) - - -def _matching(*, value: str, names: frozenset[str]) -> frozenset[str]: - if not _is_pattern(value): - return frozenset({value}) & names - return frozenset(name for name in names if fnmatchcase(name, value)) - - -def _is_pattern(value: str) -> bool: - return any(character in value for character in "*?[") - - -def _unit_test_names(discovered_inputs: DiscoveredProjectInputs) -> frozenset[str]: - names: set[str] = set() - for test_file in discovered_inputs.test_files: - names.update(block.name or test_file.relative_path.stem for block in test_file.blocks) - return frozenset(names) - - -def _resource_names(discovered_inputs: DiscoveredProjectInputs) -> frozenset[str]: - names: set[str] = {model_file.file_path.stem for model_file in discovered_inputs.model_files} - for schema_file in discovered_inputs.schema_files: - names.update(entry.name for entry in schema_file.model_entries) - names.update(entry.name for entry in schema_file.seed_entries) - for source_file in discovered_inputs.source_files: - names.update(entry.name for entry in source_file.source_entries) - names.update(seed_file.file_path.stem for seed_file in discovered_inputs.seed_files) - names.update( - function_file.file_path.stem - for function_file in ( - *discovered_inputs.sql_function_files, - *discovered_inputs.python_function_files, - ) - ) - names.update( - node.name - for node in ( - *discovered_inputs.loader_functions, - *discovered_inputs.task_functions, - *discovered_inputs.asset_functions, - *discovered_inputs.check_functions, - ) - ) - return frozenset(names) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index ccff286819..b18690f71c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -10,6 +10,8 @@ from typing import Any from sqlbuild._native import ( + build_execution_upstream_deps, + build_upstream_intermediate_source_map, classify_cursor_sql_type, get_config_cursor_bound, get_config_str, @@ -62,12 +64,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_execution_upstream_deps, -) -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_upstream_intermediate_source_map, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) diff --git a/src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py b/src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py deleted file mode 100644 index 7d2f6e1fcf..0000000000 --- a/src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Memoized graph identity resolution over one dependency graph.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - -type _Resolve = Callable[[GraphNodeKey], str | None] - - -class GraphIdentityResolver: - """Resolve expected or selected-write identity hashes, each node once, in linear time.""" - - def __init__( - self, - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - compose_identity: GraphIdentityComposer, - base_hashes: Mapping[GraphNodeKey, str | None], - selected_keys: frozenset[GraphNodeKey] | None, - ) -> None: - self._nodes: Mapping[GraphNodeKey, GraphIdentityNode] = nodes - self._compose_identity: GraphIdentityComposer = compose_identity - self._selected_keys: frozenset[GraphNodeKey] | None = selected_keys - self._hashes: dict[GraphNodeKey, str | None] = dict(base_hashes) - self._resolved_selected: dict[GraphNodeKey, str] = {} - self._visiting: set[GraphNodeKey] = set() - - @property - def hashes(self) -> dict[GraphNodeKey, str | None]: - """Return every hash recorded so far, in first-recorded order.""" - - return self._hashes - - def resolve_expected(self, key: GraphNodeKey) -> str | None: - """Compose a node's expected hash from its upstream expected hashes.""" - - if key in self._hashes: - return self._hashes[key] - node: GraphIdentityNode | None = self._nodes.get(key) - if node is None or node.local_hash is None: - self._hashes[key] = None - return None - if key in self._visiting: - self._hashes[key] = node.local_hash - return node.local_hash - composed: str = self._compose( - key=key, local_hash=node.local_hash, node=node, resolve=self.resolve_expected - ) - self._hashes[key] = composed - return composed - - def resolve_write(self, key: GraphNodeKey) -> str | None: - """Recompute a selected node's hash from upstream write hashes; others keep theirs.""" - - if self._selected_keys is None or key not in self._selected_keys: - return self._hashes.get(key) - if key in self._resolved_selected: - return self._resolved_selected[key] - node: GraphIdentityNode | None = self._nodes.get(key) - if node is None or node.local_hash is None: - return self._hashes.get(key) - if key in self._visiting: - return node.local_hash - composed: str = self._compose( - key=key, local_hash=node.local_hash, node=node, resolve=self.resolve_write - ) - self._hashes[key] = composed - self._resolved_selected[key] = composed - return composed - - def _compose( - self, *, key: GraphNodeKey, local_hash: str, node: GraphIdentityNode, resolve: _Resolve - ) -> str: - self._visiting.add(key) - upstream_hashes: list[tuple[GraphNodeKey, str]] = [] - upstream_key: GraphNodeKey - for upstream_key in node.upstream_keys: - upstream_hash: str | None = resolve(upstream_key) - if upstream_hash is not None: - upstream_hashes.append((upstream_key, upstream_hash)) - self._visiting.discard(key) - return self._compose_identity(local_hash=local_hash, upstream_hashes=tuple(upstream_hashes)) diff --git a/src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py b/src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py deleted file mode 100644 index 78400f2196..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public selection-aware staleness classification entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( - classify_selection_staleness_warnings as _classify_selection_staleness_warnings, -) -from sqlbuild.compiler.planner.models import SelectionStalenessGraph, SelectionStalenessWarning - - -def classify_selection_staleness_warnings( - graph: SelectionStalenessGraph, -) -> tuple[SelectionStalenessWarning, ...]: - """Return stale warnings for selected models with changed upstreams outside the run set.""" - - return _classify_selection_staleness_warnings(graph) diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index 549d9a0158..224227c80b 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -2,15 +2,16 @@ from __future__ import annotations -from sqlbuild._native import get_config_str, get_materialization_type +from sqlbuild._native import ( + build_downstream_deps, + build_execution_upstream_deps, + get_config_str, + get_materialization_type, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import resolve_model_full_refresh from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import ( diff --git a/src/sqlbuild/compiler/planner/main/identity/__init__.py b/src/sqlbuild/compiler/planner/main/identity/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/identity/_graph_identity.py b/src/sqlbuild/compiler/planner/main/identity/_graph_identity.py deleted file mode 100644 index a60cf1a941..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_graph_identity.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public entrypoints for neutral graph identity resolution.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner._helpers.identity.graph import ( - build_expected_graph_identity_hashes as _build_expected_graph_identity_hashes, -) -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - - -def build_expected_graph_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str | None]: - """Build expected identity hashes for a neutral dependency graph.""" - - return _build_expected_graph_identity_hashes( - nodes=nodes, - execution_order=execution_order, - compose_identity=compose_identity, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py b/src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py deleted file mode 100644 index 775e5c0ae7..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public entrypoint for neutral graph write identity resolution.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner._helpers.identity.graph import ( - build_graph_write_identity_hashes as _build_graph_write_identity_hashes, -) -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - - -def build_graph_write_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - selected_keys: frozenset[GraphNodeKey], - base_identity_hashes: Mapping[GraphNodeKey, str], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str]: - """Build write identity hashes from caller-supplied available upstream hashes.""" - - return _build_graph_write_identity_hashes( - nodes=nodes, - execution_order=execution_order, - selected_keys=selected_keys, - base_identity_hashes=base_identity_hashes, - compose_identity=compose_identity, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py b/src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py deleted file mode 100644 index 9daac10832..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Build model config payloads that participate in version identity.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY - - -def build_version_identity_config(config_values: dict[str, object]) -> dict[str, object]: - """Return config fields that affect produced model version identity.""" - - identity: dict[str, object] = {} - if CURSOR_INPUTS_CONFIG_KEY in config_values: - identity[CURSOR_INPUTS_CONFIG_KEY] = config_values[CURSOR_INPUTS_CONFIG_KEY] - version_identity_config_keys: tuple[str, ...] = ( - "append_cursor_inclusive", - "batch_size", - "check_columns", - "cursor", - "cursor_grain", - "cursor_start", - "cursor_end", - "cursor_start_max_ahead", - "cursor_start_max_action", - "cursor_future_max_distance", - "cursor_future_action", - "cursor_type", - "historical_input", - "incremental_mode", - "microbatch_strategy", - "cursor_watermark_mode", - "max_microbatches", - "microbatch_limit", - "incremental_strategy", - "merge_exclude_columns", - "full_refresh", - "initial_valid_from", - "invalidate_hard_deletes", - "lookback", - "materialized", - "observed_at", - "on_schema_change", - "snapshot_full_refresh", - "snapshot_schema_change", - "snapshot_strategy", - "unique_key", - "updated_at", - "valid_from_column", - "valid_to_column", - ) - identity.update( - {key: config_values[key] for key in version_identity_config_keys if key in config_values} - ) - return identity diff --git a/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py b/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py deleted file mode 100644 index d5628746d8..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Build non-query metadata that participates in model version identity.""" - -from __future__ import annotations - -import json - -from sqlbuild.compiler.planner.constants import LOCAL_FUNCTION_HASHES_METADATA_KEY -from sqlbuild.compiler.planner.main.identity._version_identity_config import ( - build_version_identity_config, -) - - -def build_version_identity_metadata_json( - *, - model_name: str, - config_values: dict[str, object], - local_function_hashes: dict[str, str] | None = None, - execution_signature: dict[str, object] | None = None, -) -> str: - """Return deterministic metadata JSON for model version identity.""" - - return json.dumps( - { - "config": build_version_identity_config(config_values), - "execution_signature": execution_signature or {}, - LOCAL_FUNCTION_HASHES_METADATA_KEY: local_function_hashes or {}, - "model_name": model_name, - }, - sort_keys=True, - separators=(",", ":"), - default=str, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py deleted file mode 100644 index 0a17680710..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Build function hashes that participate in model version identity.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash - - -def build_function_local_hashes( - *, - functions: tuple[CompiledFunctionView, ...], - dialect: str | None, -) -> dict[str, str]: - """Derive local-only semantic hashes for functions; SQL bodies ignore layout and comments.""" - - hashes: dict[str, str] = {} - for function in functions: - arguments: list[tuple[str, str]] = [] - for argument in function.arguments: - arguments.append((argument.name, argument.type)) - return_columns: list[tuple[str, str]] = [] - for column in function.return_columns: - return_columns.append((column.name, column.type)) - hashes[function.name] = _stable_hash( - json.dumps( - { - "arguments": arguments, - "returns": function.returns, - "return_columns": return_columns, - "body_sql": function_definition_hash( - function_name=function.name, - fingerprint_sql=function.body_sql, - language=function.language.value, - dialect=dialect, - ), - "language": function.language.value, - "runtime_version": function.runtime_version, - "entry_point": function.entry_point, - "packages": function.packages, - }, - sort_keys=True, - default=str, - ) - ) - return hashes - - -def _stable_hash(value: str) -> str: - return hashlib.sha256(value.encode("utf-8")).hexdigest() diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py deleted file mode 100644 index 0cc9e3b64b..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Build model metadata that participates in version identity.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import serialize_hook_entries -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.discovery.constants import SQL_HOOK_IDENTITY_FIELDS -from sqlbuild.compiler.planner._helpers.identity.model_metadata import contract_output_signature -from sqlbuild.compiler.planner.constants import ( - MODEL_CUSTOM_CONFIG_KEY, - MODEL_PLACEHOLDERS_CONFIG_KEY, - MODEL_POST_HOOKS_CONFIG_KEY, - MODEL_PRE_HOOKS_CONFIG_KEY, -) -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) - - -def build_model_version_identity_metadata_json( - *, - model: CompiledModelView, - function_local_hashes: dict[str, str] | None = None, - hook_version_hashes: dict[str, str] | None = None, -) -> str: - """Build deterministic non-query model fingerprint metadata JSON.""" - - function_hashes: dict[str, str] = function_local_hashes or {} - local_function_hashes: dict[str, str] = {} - upstream_key: Any - for upstream_key in model.deps: - if upstream_key.resource_type not in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - continue - upstream_hash: str | None = function_hashes.get(upstream_key.name) - if upstream_hash is not None: - local_function_hashes[upstream_key.name] = upstream_hash - return build_version_identity_metadata_json( - model_name=model.name, - config_values=model.config.values, - local_function_hashes=local_function_hashes, - execution_signature=_model_execution_signature( - model=model, - hook_version_hashes=hook_version_hashes or {}, - ), - ) - - -def _model_execution_signature( - *, model: CompiledModelView, hook_version_hashes: dict[str, str] -) -> dict[str, object]: - signature: dict[str, object] = {} - contract_signature: dict[str, object] | None = contract_output_signature(model=model) - if contract_signature is not None: - signature["contract"] = contract_signature - if MODEL_CUSTOM_CONFIG_KEY in model.config.values: - signature["custom_config"] = model.config.values[MODEL_CUSTOM_CONFIG_KEY] - if MODEL_PLACEHOLDERS_CONFIG_KEY in model.config.values: - signature["custom_placeholders"] = model.config.values[MODEL_PLACEHOLDERS_CONFIG_KEY] - if MODEL_PRE_HOOKS_CONFIG_KEY in model.config.values: - signature["pre_hooks"] = _hook_execution_signature( - value=model.config.values[MODEL_PRE_HOOKS_CONFIG_KEY], - hook_version_hashes=hook_version_hashes, - ) - if MODEL_POST_HOOKS_CONFIG_KEY in model.config.values: - signature["post_hooks"] = _hook_execution_signature( - value=model.config.values[MODEL_POST_HOOKS_CONFIG_KEY], - hook_version_hashes=hook_version_hashes, - ) - return signature - - -def _hook_execution_signature( - *, value: object, hook_version_hashes: dict[str, str] -) -> list[dict[str, object]]: - return serialize_hook_entries( - value=value, - sql_fields=SQL_HOOK_IDENTITY_FIELDS, - python_hook_fields={ - hook_name: {"version_hash": version_hash} - for hook_name, version_hash in hook_version_hashes.items() - }, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_build_resources.py b/src/sqlbuild/compiler/planner/main/selection/_build_resources.py deleted file mode 100644 index c8f1499525..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_build_resources.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Compiler-internal required-build-resource expansion entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import expand_required_build_resources - - -def expand_build_resource_selection( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - include_upstream_functions: bool = True, - include_upstream_seeds: bool = False, - include_downstream_functions: bool = False, -) -> frozenset[CompiledObjectKey]: - """Expand a selected SQL scope with required build resources.""" - - return expand_required_build_resources( - selected_keys=selected_keys, - upstream=upstream, - downstream=downstream, - include_upstream_functions=include_upstream_functions, - include_upstream_seeds=include_upstream_seeds, - include_downstream_functions=include_downstream_functions, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py b/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py deleted file mode 100644 index 71c954d22f..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public selector resolution over a native project graph.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors - - -def resolve_graph_selection( - *, - graph: _native.NativeProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[CompiledObjectKey]: - """Resolve select/exclude strings natively, adding the functions selected models need.""" - - return resolve_graph_selectors(graph=graph, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py b/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py deleted file mode 100644 index 5455e59d6b..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public planner entrypoints for loader DAG helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_intermediate_source_map as _build_intermediate_source_map, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_intermediate_source_map( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] -) -> dict[str, SourceEntry]: - """Return synthetic source entries for selected intermediate loader nodes.""" - - return _build_intermediate_source_map(project=project, selected_keys=selected_keys) diff --git a/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py b/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py deleted file mode 100644 index ce4bd03d3d..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Compiler-internal downstream model-name closure entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.model_closure import ( - build_downstream_model_name_closure, -) - - -def build_downstream_model_names( - *, - start_keys: tuple[CompiledObjectKey, ...], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - return build_downstream_model_name_closure( - start_keys=start_keys, - downstream_deps=downstream_deps, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_selector_match.py b/src/sqlbuild/compiler/planner/main/selection/_selector_match.py deleted file mode 100644 index 88244d4e29..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_selector_match.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public entrypoint matching one parsed selector before graph expansion.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import match_selector_keys -from sqlbuild.compiler.planner.models import ParsedSelector - - -def match_project_selector( - *, - parsed: ParsedSelector, - all_keys: dict[str, CompiledObjectKey], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Return the keys one parsed selector matches before `+` graph expansion.""" - - return match_selector_keys( - parsed=parsed, - all_keys=all_keys, - tag_index=tag_index, - path_index=path_index, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py b/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py deleted file mode 100644 index a69d10ba0f..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public selector-token resolution entrypoint without required build resources.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selector_tokens - - -def resolve_project_selector_tokens( - *, - selectors: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Resolve selector strings to the keys they match, without adding required build resources.""" - - return resolve_selector_tokens( - selectors=selectors, - all_keys=all_keys, - upstream=upstream_deps, - downstream=downstream_deps, - tag_index=tag_index, - path_index=path_index, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py b/src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py deleted file mode 100644 index 4d7a8a7270..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Compiler-internal unit-test selector split entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.selection.unit_test_selectors import ( - split_discovered_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import UnitTestSelectorSplit - - -def split_project_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, - accepts_unit_tests: bool, -) -> UnitTestSelectorSplit: - """Move unit-test selectors out of resource selectors, or reject them for this command.""" - - return split_discovered_unit_test_selectors( - select=select, - exclude=exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=accepts_unit_tests, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_upstream.py b/src/sqlbuild/compiler/planner/main/selection/_upstream.py deleted file mode 100644 index a03105df3d..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_upstream.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public upstream graph expansion entrypoint for planner consumers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.core import expand_upstream - - -def expand_project_upstream_keys( - *, - key: CompiledObjectKey, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return the upstream key closure for one compiled graph key.""" - - return frozenset(expand_upstream(key=key, upstream=upstream_deps)) diff --git a/src/sqlbuild/compiler/planner/main/selection/scope.py b/src/sqlbuild/compiler/planner/main/selection/scope.py deleted file mode 100644 index 035d762d25..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/scope.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public planner scope entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope as _build -from sqlbuild.compiler.planner.models import PlannerScope - - -def build_planner_scope( - *, - project: CompiledProjectView, - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - auto_load_sources: bool = False, - selected_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerScope: - """Build the resolved planner graph scope for a project.""" - - return _build( - project=project, - select=select, - exclude=exclude, - auto_load_sources=auto_load_sources, - selected_keys=selected_keys, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/selection.py b/src/sqlbuild/compiler/planner/main/selection/selection.py deleted file mode 100644 index 8190db37a9..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/selection.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public selector-resolution entrypoints for planner consumers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selectors - - -def resolve_project_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Resolve model/source/seed/function selectors against a compiled graph.""" - - return resolve_selectors( - select=select, - exclude=exclude, - all_keys=all_keys, - upstream=upstream_deps, - downstream=downstream_deps, - tag_index=tag_index, - path_index=path_index, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/selector_expansion.py b/src/sqlbuild/compiler/planner/main/selection/selector_expansion.py deleted file mode 100644 index 2db5d73406..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/selector_expansion.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public selector expansion marker parsing entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.constants import SELECTOR_EXPANSION_MARKER -from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.errors.contracts.exceptions import SharedInputError - - -def split_selector_expansion(raw: str) -> SelectorExpansion: - """Split +selector+ marker syntax from the selector core without resolving the core.""" - - stripped: str = raw.strip() - if not stripped: - raise SharedInputError("empty selector") - upstream: bool = stripped.startswith("+") - downstream: bool = stripped.endswith("+") - core: str = stripped.lstrip("+").rstrip("+") - if not core: - raise SharedInputError(f"selector '{stripped}' has no name after removing '+' markers") - if SELECTOR_EXPANSION_MARKER in core: - raise SharedInputError(f"selector '{stripped}' contains '+' in an unsupported position") - return SelectorExpansion(core=core, upstream=upstream, downstream=downstream) diff --git a/src/sqlbuild/compiler/planner/main/selection/selector_parse.py b/src/sqlbuild/compiler/planner/main/selection/selector_parse.py deleted file mode 100644 index 4cdc4a7652..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/selector_parse.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public selector parsing entrypoint for planner consumers.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.graph.selectors import parse_selector -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector - - -def parse_project_selector(raw: str) -> ParsedSelector | PathSelector: - """Parse one project selector token into a structured form.""" - - return parse_selector(raw) diff --git a/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py b/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py deleted file mode 100644 index 6e662fdae3..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Public SQLBuild model selector resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView - - -def resolve_sqlbuild_model_selector_names( - *, project: CompiledProjectView, term: str -) -> tuple[tuple[str, ...], str | None]: - """Resolve one SQLBuild-owned model selector core to names and path translation.""" - - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - if term in models_by_name: - return (term,), None - if term.startswith("tag:"): - tag: str = term.removeprefix("tag:") - names: list[str] = [] - for model in project.models: - raw_tags: object = model.config.values.get("tags") - tags: tuple[str, ...] = () - if isinstance(raw_tags, tuple): - tags = tuple(str(item) for item in raw_tags) - elif isinstance(raw_tags, list): - tags = tuple(str(item) for item in raw_tags) - if tag in tags: - names.append(model.name) - return tuple(names), None - if term.startswith("path:"): - raw_path: str = term.removeprefix("path:") - translated_path: str = raw_path.replace("\\", "/") - names = [ - model.name - for model in project.models - if (model_path := model.relative_path.parent.as_posix()) == translated_path - or model_path.startswith(f"{translated_path}/") - ] - return tuple(names), f"path:{translated_path}" if translated_path != raw_path else None - return (), None diff --git a/src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py b/src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py deleted file mode 100644 index 2aab683cb6..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public error for unit-test selectors where unit tests never run.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.graph.selectors import unit_test_selector_rejection -from sqlbuild.compiler.planner.exceptions import PlannerInputError - - -def unit_test_selector_error(*, selector: str) -> PlannerInputError: - """The error for a unit-test selector given where unit tests never run.""" - - return unit_test_selector_rejection(selector=selector) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py index da64eb1d77..9368d38745 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py @@ -2,11 +2,9 @@ from __future__ import annotations +from sqlbuild._native import build_downstream_model_name_closure from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.main.selection._model_downstream_closure import ( - build_downstream_model_names, -) from sqlbuild.compiler.planner.models import PlannerScope from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, @@ -65,7 +63,7 @@ def _downstream_model_names(*, source_name: str, scope: PlannerScope) -> frozens resource_type=CompiledResourceType.SOURCE, name=source_name, ) - return build_downstream_model_names( + return build_downstream_model_name_closure( start_keys=(source_key,), downstream_deps=scope.downstream_deps, ) diff --git a/src/sqlbuild/integrations/dagster/_helpers/selection.py b/src/sqlbuild/integrations/dagster/_helpers/selection.py index 3007e05646..25ac702707 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/selection.py +++ b/src/sqlbuild/integrations/dagster/_helpers/selection.py @@ -5,12 +5,13 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlbuild._native import path_nodes, transitive_closure_many -from sqlbuild.compiler.planner.constants import SELECTOR_KIND_SEPARATOR -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.main.selection.unit_test_selector_error import ( +from sqlbuild._native import ( + parse_project_selector, + path_nodes, + transitive_closure_many, unit_test_selector_error, ) +from sqlbuild.compiler.planner.constants import SELECTOR_KIND_SEPARATOR from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.planner.types import SelectorKind from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 44a93ea5ed..b2c8f288f5 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -13,9 +13,6 @@ resolve_target_config, ) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work -from sqlbuild.cli.commands.main.execution.connection_progress import ( - build_connection_progress_reporter, -) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput @@ -145,7 +142,7 @@ def resolve_sqlbuild_execution_plan_output( return None invocation.output_stream.write("\n") invocation.output_stream.flush() - connection_progress: ConnectionProgressReporter = build_connection_progress_reporter( + connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( adapter_name=compiled.adapter_name, stream=invocation.output_stream, blank_line_after_complete=True, diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/plan.py b/src/sqlbuild/integrations/dbt/main/pipeline/plan.py index 35f46789c6..47b74d6a66 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/plan.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/plan.py @@ -7,9 +7,7 @@ from pathlib import Path from typing import Any, TextIO -from sqlbuild.cli.commands.main.execution.connection_progress import ( - build_connection_progress_reporter, -) +from sqlbuild._native import ConnectionProgressReporter from sqlbuild.integrations.dbt._helpers.pipeline.interop_prologue import ( compile_dbt_interop_project, load_compiled_dbt_manifest, @@ -75,7 +73,7 @@ def plan_dbt_interop_from_project( on_progress=on_progress, ) connection_progress: Any | None = ( - build_connection_progress_reporter( + ConnectionProgressReporter( adapter_name=compiled.adapter_name, stream=progress_stream, use_color=use_color, diff --git a/src/sqlbuild/presentation/_helpers/error_text.py b/src/sqlbuild/presentation/_helpers/error_text.py deleted file mode 100644 index 9e5e945b1a..0000000000 --- a/src/sqlbuild/presentation/_helpers/error_text.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Coded-error rendering implementation.""" - -from __future__ import annotations - -import textwrap - -from sqlbuild.presentation.classes.cli_style import CliStyle - -_MAX_INLINE_ERROR_LINES: int = 2 - - -def format_coded_error( - *, - code: str, - message: str, - help: str | None = None, - use_color: bool = False, - include_error_label: bool = True, -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - prefix_text: str = f"error[{code}]:" if include_error_label else f"[{code}]" - prefix: str = style.error_strong(prefix_text) - lines: list[str] = message.split("\n") - rendered_message: str = message - if len(lines) > 1: - rendered_message = "\n".join([lines[0], *[style.muted(line) for line in lines[1:]]]) - rendered: str = f"{prefix} {rendered_message}" - if help is not None: - help_label: str = style.muted("= help:") - rendered = f"{rendered}\n {help_label} {help}" - return rendered - - -def format_inline_error_lines( - *, - error_code: str | None, - error_message: str, - error_help: str | None, - content_width: int, - style: CliStyle, -) -> list[str]: - """Wrap a compact inline diagnostic while preserving semantic code styling.""" - - code_prefix: str = f"[{error_code}]" if error_code is not None else "" - message: str = f"{code_prefix} {error_message}" if code_prefix else error_message - if error_help is not None: - message = f"{message}\n= help: {error_help}" - wrapped_lines: list[str] = [] - paragraph: str - for paragraph in message.splitlines() or [message]: - wrapped_lines.extend( - textwrap.wrap( - paragraph, - width=content_width, - break_long_words=True, - break_on_hyphens=False, - ) - or [""] - ) - truncated: bool = len(wrapped_lines) > _MAX_INLINE_ERROR_LINES - visible_lines: list[str] = wrapped_lines[:_MAX_INLINE_ERROR_LINES] - if truncated: - final_line: str = visible_lines[-1] - ellipsis: str = "." * min(3, content_width) - visible_lines[-1] = ( - f"{final_line[: max(0, content_width - len(ellipsis))].rstrip()}{ellipsis}" - ) - if code_prefix and visible_lines: - visible_lines[0] = visible_lines[0].replace( - code_prefix, - style.error_strong(code_prefix), - 1, - ) - return visible_lines diff --git a/src/sqlbuild/presentation/_helpers/progress.py b/src/sqlbuild/presentation/_helpers/progress.py deleted file mode 100644 index 948369c498..0000000000 --- a/src/sqlbuild/presentation/_helpers/progress.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Progress environment implementations.""" - -from __future__ import annotations - -import os - -from sqlbuild.presentation.constants import PROGRESS_DISABLED_VALUES - - -def progress_spinners_disabled() -> bool: - value: str | None = os.environ.get("SQLBUILD_NO_PROGRESS") - return value is not None and value.strip().lower() in PROGRESS_DISABLED_VALUES diff --git a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py b/src/sqlbuild/presentation/_helpers/terminal_capabilities.py index bc8cc09b94..ec3b5e3cfa 100644 --- a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py +++ b/src/sqlbuild/presentation/_helpers/terminal_capabilities.py @@ -2,21 +2,11 @@ from __future__ import annotations -import os import shutil -import sys _DEFAULT_TERMINAL_COLUMNS: int = 120 -def supports_color() -> bool: - if os.environ.get("NO_COLOR") is not None: - return False - if not hasattr(sys.stdout, "isatty"): - return False - return sys.stdout.isatty() - - def terminal_columns() -> int: """Return the current terminal width in columns.""" diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index c79c8b6d41..a06117ca75 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -6,7 +6,6 @@ ERROR_STATUS_LABELS: frozenset[str] = frozenset({"ERROR", "FAIL", "FAILED"}) PHASE_FAIL_GLYPH: str = "\u2717" PHASE_OK_GLYPH: str = "\u2713" -PROGRESS_DISABLED_VALUES: frozenset[str] = frozenset({"1", "on", "true", "yes"}) SKIPPED_STATUS_LABELS: frozenset[str] = frozenset({"SKIP", "SKIPPED"}) SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" diff --git a/src/sqlbuild/presentation/main/_maybe_status.py b/src/sqlbuild/presentation/main/_maybe_status.py deleted file mode 100644 index d9cb7214cd..0000000000 --- a/src/sqlbuild/presentation/main/_maybe_status.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public transient stderr status context entry.""" - -from __future__ import annotations - -import sys -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild._native import TransientStatusReporter -from sqlbuild.presentation._helpers.progress import progress_spinners_disabled - - -@contextmanager -def maybe_status(*, message: str, enabled: bool) -> Iterator[None]: - """Render a stderr spinner for long-running interactive operations.""" - - if not enabled or progress_spinners_disabled() or not sys.stderr.isatty(): - yield - return - reporter: TransientStatusReporter = TransientStatusReporter(stream=sys.stderr, enabled=enabled) - reporter.start(message) - try: - yield - finally: - reporter.close() diff --git a/src/sqlbuild/presentation/main/_progress_spinners_disabled.py b/src/sqlbuild/presentation/main/_progress_spinners_disabled.py deleted file mode 100644 index 06ba947e68..0000000000 --- a/src/sqlbuild/presentation/main/_progress_spinners_disabled.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public progress spinner environment check entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.progress import ( - progress_spinners_disabled as _progress_spinners_disabled, -) - - -def progress_spinners_disabled() -> bool: - """Return whether transient spinners are disabled via the environment.""" - - return _progress_spinners_disabled() diff --git a/src/sqlbuild/presentation/main/coded_error_text.py b/src/sqlbuild/presentation/main/coded_error_text.py deleted file mode 100644 index 6fff672e59..0000000000 --- a/src/sqlbuild/presentation/main/coded_error_text.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public coded-error formatting entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.error_text import format_coded_error as _format_coded_error - - -def format_coded_error( - *, - code: str, - message: str, - help: str | None = None, - use_color: bool = False, - include_error_label: bool = True, -) -> str: - """Render a result-stored coded error consistently with CLI expected errors.""" - - return _format_coded_error( - code=code, - message=message, - help=help, - use_color=use_color, - include_error_label=include_error_label, - ) diff --git a/src/sqlbuild/presentation/main/inline_error_lines.py b/src/sqlbuild/presentation/main/inline_error_lines.py deleted file mode 100644 index 7735e70da9..0000000000 --- a/src/sqlbuild/presentation/main/inline_error_lines.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public inline error-line formatting operation.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.error_text import ( - format_inline_error_lines as _format_inline_error_lines, -) -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_inline_error_lines( - *, - error_code: str | None, - error_message: str, - error_help: str | None, - content_width: int, - style: CliStyle, -) -> list[str]: - """Wrap an inline error to two display lines before truncating.""" - - return _format_inline_error_lines( - error_code=error_code, - error_message=error_message, - error_help=error_help, - content_width=content_width, - style=style, - ) diff --git a/src/sqlbuild/presentation/main/supports_color.py b/src/sqlbuild/presentation/main/supports_color.py deleted file mode 100644 index c98d0ad2eb..0000000000 --- a/src/sqlbuild/presentation/main/supports_color.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public terminal color capability entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.terminal_capabilities import supports_color as _supports_color - - -def supports_color() -> bool: - """Return whether the active output terminal supports color.""" - - return _supports_color() diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py new file mode 100644 index 0000000000..c86d8bafe6 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -0,0 +1,76 @@ +"""Import the project's custom rule modules and collect their `@rule` functions.""" + +from __future__ import annotations + +import importlib.util +import sys +from dataclasses import replace +from importlib.machinery import ModuleSpec +from pathlib import Path +from types import ModuleType + +from sqlbuild.rule_engine._helpers.engine.definition import ( + resolve_rule_signature, + rule_from_value, +) +from sqlbuild.rule_engine.constants import INIT_MODULE_NAME, RULE_DECORATOR_TOKEN +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.rule_engine.models import Rule + + +def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: + """Every custom rule declared under the project's `rules/` directory, in file order.""" + + result: list[Rule] = [] + existing_modules: frozenset[str] = frozenset(sys.modules) + repository_path: str = str(project_dir.resolve()) + sys.path.insert(0, repository_path) + try: + for file_path in sorted((project_dir / "rules").resolve().rglob("*.py")): + if RULE_DECORATOR_TOKEN not in file_path.read_text(encoding="utf-8"): + continue + name: str = _rule_module_name(file_path=file_path, project_dir=project_dir) + spec: ModuleSpec | None = importlib.util.spec_from_file_location(name, file_path) + if spec is None or spec.loader is None: + raise RulesError(f"could not load rule file {file_path}") + module: ModuleType = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except Exception as error: + raise RulesError(f"could not import rule file {file_path}: {error}") from error + finally: + _ = sys.modules.pop(name, None) + result.extend(_module_rules(module=module, source=file_path)) + finally: + sys.path.remove(repository_path) + _remove_loaded_project_modules(existing_modules=existing_modules, project_dir=project_dir) + return tuple(result) + + +def _rule_module_name(*, file_path: Path, project_dir: Path) -> str: + relative: Path = file_path.relative_to(project_dir.resolve()).with_suffix("") + parts: tuple[str, ...] = relative.parts + if parts[-1] == INIT_MODULE_NAME: + parts = parts[:-1] + return ".".join(parts) + + +def _module_rules(*, module: ModuleType, source: Path) -> tuple[Rule, ...]: + rules: tuple[Rule, ...] = tuple( + resolve_rule_signature(rule=rule) + for value in vars(module).values() + if getattr(value, "__module__", None) == module.__name__ + and (rule := rule_from_value(value=value)) is not None + ) + return tuple(replace(rule, source=source.as_posix()) for rule in rules) + + +def _remove_loaded_project_modules(*, existing_modules: frozenset[str], project_dir: Path) -> None: + for name in tuple(sys.modules): + if name in existing_modules: + continue + module: object = sys.modules.get(name) + source: object = getattr(module, "__file__", None) + if isinstance(source, str) and Path(source).resolve().is_relative_to(project_dir.resolve()): + _ = sys.modules.pop(name, None) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py deleted file mode 100644 index 512d3f411e..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Built-in and custom rule catalogue construction.""" - -from __future__ import annotations - -import importlib.util -import sys -from collections.abc import Mapping -from dataclasses import replace -from importlib.machinery import ModuleSpec -from pathlib import Path -from types import ModuleType - -from sqlbuild._native import builtin_rules, selected_rule_codes -from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads -from sqlbuild.rule_engine._helpers.engine.definition import ( - resolve_rule_signature, - rule_from_value, -) -from sqlbuild.rule_engine.constants import ( - INIT_MODULE_NAME, - RULE_DECORATOR_TOKEN, -) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import ( - Finding, - Rule, - RuleOption, - RulesConfig, -) - - -def build_catalogue( - *, config: RulesConfig, project_dir: Path, include_custom: bool = True -) -> tuple[Rule, ...]: - """Build and validate the complete configured rule catalogue.""" - - rules_dir: Path = project_dir / "rules" - custom: tuple[Rule, ...] = ( - (_load_paths(paths=("rules",), project_dir=project_dir) if rules_dir.is_dir() else ()) - if include_custom - else () - ) - rules: tuple[Rule, ...] = (*_builtins(), *custom) - codes: tuple[str, ...] = tuple(rule.code for rule in rules) - duplicates: tuple[str, ...] = tuple( - sorted(code for code in set(codes) if codes.count(code) > 1) - ) - if duplicates: - raise RulesError(f"duplicate rule codes: {', '.join(duplicates)}") - for rule in rules: - _validate_options(rule=rule, configured=config.rule_options.get(rule.code, {})) - configured_codes: set[str] = set(config.rule_options) - unknown_option_codes: set[str] = configured_codes - set(codes) - if unknown_option_codes: - raise RulesError( - f"rule options target unknown codes: {', '.join(sorted(unknown_option_codes))}" - ) - return tuple(sorted(rules, key=lambda item: item.code)) - - -def _builtins() -> tuple[Rule, ...]: - return tuple(builtin_rules(check=_native_builtin)) - - -def _native_builtin(*, model: object, ctx: object) -> list[Finding]: - del model, ctx - raise RulesError("built-in rules execute only through the native engine") - - -def select_rules( - *, - catalogue: tuple[Rule, ...], - config: RulesConfig, - project_dir: Path, - custom_payloads: list[dict[str, object]] | None = None, -) -> tuple[Rule, ...]: - """Resolve the active catalogue through the native Fensu rules owner.""" - - by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} - codes: list[str] = selected_rule_codes( - config=config, - custom_payloads=( - custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) - if custom_payloads is None - else custom_payloads - ), - ) - try: - return tuple(by_code[code] for code in codes) - except KeyError as error: - raise RulesError(f"native compiler rules returned unknown rule {error.args[0]}") from error - - -def _load_paths(*, paths: tuple[str, ...], project_dir: Path) -> tuple[Rule, ...]: - result: list[Rule] = [] - existing_modules: frozenset[str] = frozenset(sys.modules) - repository_path: str = str(project_dir.resolve()) - sys.path.insert(0, repository_path) - try: - for configured in paths: - path: Path = (project_dir / configured).resolve() - if not path.is_relative_to(project_dir.resolve()): - raise RulesError(f"rule path escapes the project: {configured}") - configured_directory: bool = path.is_dir() - files: tuple[Path, ...] = ( - tuple(sorted(path.rglob("*.py"))) if path.is_dir() else (path,) - ) - for file_path in files: - if not file_path.is_file(): - raise RulesError(f"rule path does not exist: {file_path}") - if configured_directory and RULE_DECORATOR_TOKEN not in file_path.read_text( - encoding="utf-8" - ): - continue - name: str = _rule_module_name(file_path=file_path, project_dir=project_dir) - spec: ModuleSpec | None = importlib.util.spec_from_file_location(name, file_path) - if spec is None or spec.loader is None: - raise RulesError(f"could not load rule file {file_path}") - module: ModuleType = importlib.util.module_from_spec(spec) - sys.modules[name] = module - try: - spec.loader.exec_module(module) - except Exception as error: - raise RulesError(f"could not import rule file {file_path}: {error}") from error - finally: - _ = sys.modules.pop(name, None) - result.extend( - _module_rules( - module=module, - source=file_path, - require_rules=not configured_directory, - ) - ) - finally: - sys.path.remove(repository_path) - _remove_loaded_project_modules( - existing_modules=existing_modules, - project_dir=project_dir, - ) - return tuple(result) - - -def _rule_module_name(*, file_path: Path, project_dir: Path) -> str: - relative: Path = file_path.relative_to(project_dir.resolve()).with_suffix("") - parts: tuple[str, ...] = relative.parts - if parts[-1] == INIT_MODULE_NAME: - parts = parts[:-1] - return ".".join(parts) - - -def _module_rules( - *, module: ModuleType, source: Path, require_rules: bool = True -) -> tuple[Rule, ...]: - rules: tuple[Rule, ...] = tuple( - resolve_rule_signature(rule=rule) - for value in vars(module).values() - if getattr(value, "__module__", None) == module.__name__ - and (rule := rule_from_value(value=value)) is not None - ) - if not rules and require_rules: - raise RulesError(f"rule source exposes no @rule functions: {source}") - return tuple(replace(rule, source=source.as_posix()) for rule in rules) - - -def _remove_loaded_project_modules(*, existing_modules: frozenset[str], project_dir: Path) -> None: - for name in tuple(sys.modules): - if name in existing_modules: - continue - module: object = sys.modules.get(name) - source: object = getattr(module, "__file__", None) - if isinstance(source, str) and Path(source).resolve().is_relative_to(project_dir.resolve()): - _ = sys.modules.pop(name, None) - - -def _validate_options(*, rule: Rule, configured: Mapping[str, object]) -> None: - declarations: dict[str, RuleOption[object]] = {option.name: option for option in rule.options} - for option in rule.options: - _validate_option_declaration(rule=rule, option=option) - unknown: set[str] = set(configured) - set(declarations) - if unknown: - raise RulesError(f"rule {rule.code} has unknown options: {', '.join(sorted(unknown))}") - for name, value in configured.items(): - option: RuleOption[object] = declarations[name] - expected: type[object] = option.value_type - if expected is tuple: - valid: bool = isinstance(value, tuple) - else: - valid = isinstance(value, expected) and not ( - expected is int and isinstance(value, bool) - ) - if not valid: - raise RulesError(f"rule {rule.code} option {name} has the wrong type") - if option.choices and value not in option.choices: - raise RulesError(f"rule {rule.code} option {name} is not an allowed choice") - if isinstance(value, tuple): - if option.minimum_items is not None and len(value) < option.minimum_items: - raise RulesError(f"rule {rule.code} option {name} has too few items") - expected_item_type: type[object] | None = option.item_type - if expected_item_type is not None and any( - not isinstance(item, expected_item_type) for item in value - ): - raise RulesError( - f"rule {rule.code} option {name} has item values of the wrong type" - ) - if isinstance(value, int) and not isinstance(value, bool): - if option.minimum is not None and value < option.minimum: - raise RulesError(f"rule {rule.code} option {name} is below its minimum") - if option.maximum is not None and value > option.maximum: - raise RulesError(f"rule {rule.code} option {name} exceeds its maximum") - - -def _validate_option_declaration(*, rule: Rule, option: RuleOption[object]) -> None: - if not option.name.strip() or not option.description.strip(): - raise RulesError(f"rule {rule.code} declares an incomplete option") - default: object = option.default - expected: type[object] = option.value_type - valid_default: bool = isinstance(default, expected) and not ( - expected is int and isinstance(default, bool) - ) - if not valid_default: - raise RulesError(f"rule {rule.code} option {option.name} has a default of the wrong type") - if option.choices and default not in option.choices: - raise RulesError(f"rule {rule.code} option {option.name} default is not an allowed choice") - if isinstance(default, tuple): - if option.minimum_items is not None and len(default) < option.minimum_items: - raise RulesError(f"rule {rule.code} option {option.name} default has too few items") - if option.item_type is not None and any( - not isinstance(item, option.item_type) for item in default - ): - raise RulesError( - f"rule {rule.code} option {option.name} default has items of the wrong type" - ) - if isinstance(default, int) and not isinstance(default, bool): - if option.minimum is not None and default < option.minimum: - raise RulesError(f"rule {rule.code} option {option.name} default is below its minimum") - if option.maximum is not None and default > option.maximum: - raise RulesError(f"rule {rule.code} option {option.name} default exceeds its maximum") diff --git a/src/sqlbuild/rule_engine/_helpers/engine/config.py b/src/sqlbuild/rule_engine/_helpers/engine/config.py deleted file mode 100644 index 17c5eea811..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/config.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Resolve rule ignore selectors against the compiled project.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RuleIgnore, RulesConfig - - -def resolve_rule_ignore_selectors( - *, config: RulesConfig, project: CompiledProjectView -) -> RulesConfig: - """Resolve configured resource selectors into deterministic finding paths.""" - if not any(ignore.selectors for ignore in config.rule_ignores): - return config - path_by_key: dict[CompiledObjectKey, Path] = _resource_paths(project=project) - resolved: list[RuleIgnore] = [] - for ignore in config.rule_ignores: - if not ignore.selectors: - resolved.append(ignore) - continue - scope: PlannerScope = build_planner_scope(project=project, select=ignore.selectors) - selector_paths: set[str] = { - path_by_key[key].as_posix() for key in scope.selected_keys if key in path_by_key - } - if not selector_paths: - joined: str = ", ".join(ignore.selectors) - raise RulesError(f"rule ignore selectors resolve no resources with paths: {joined}") - resolved.append( - replace( - ignore, - paths=tuple(sorted({*ignore.paths, *selector_paths})), - selectors=(), - ) - ) - return replace(config, rule_ignores=tuple(resolved)) - - -def _resource_paths(*, project: CompiledProjectView) -> dict[CompiledObjectKey, Path]: - paths: dict[CompiledObjectKey, Path] = { - model.key: model.relative_path for model in project.models - } - paths.update({function.key: function.relative_path for function in project.functions}) - paths.update({seed.key: seed.seed_file.relative_path for seed in project.seeds}) - paths.update({source.key: source.source_file.relative_path for source in project.sources}) - return paths diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py index c017406120..ae211e78c6 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py @@ -8,16 +8,17 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import rule_selected, selected_rule_codes +from sqlbuild._native import ( + build_rules_request, + evaluate_rules_request, + finalize_rule_findings_rows, + rule_selected, + selected_rule_codes, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.run.native_rows import ( - build_rules_request, - evaluate_rules_request, - finalize_rules_rows, -) from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.models import ( CustomRulesOutcome, @@ -110,7 +111,7 @@ def _evaluate_native_rows( *custom.findings, ) if not defer_suppressions: - findings = finalize_rules_rows( + findings = finalize_rule_findings_rows( config=config, project_dir=project_dir, evaluated_codes=native.selected_codes, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 140c20c2cd..5acdaa97b9 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -15,6 +15,7 @@ custom_rule_identity, custom_rule_implementation_fingerprint, custom_rule_import_closure, + rules_bulk_cache_path, ) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView @@ -22,7 +23,6 @@ from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact from sqlbuild.rule_engine._helpers.host.custom_host_pool import run_custom_hosts -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index 2d039faef4..6717de9446 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -6,7 +6,7 @@ from pathlib import Path from tempfile import TemporaryDirectory -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs, load_rules_config from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -17,7 +17,6 @@ from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.exceptions import RuleAssertionError, RuleUsageError from sqlbuild.rule_engine.main._evaluate import evaluate -from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( Rule, RuleCase, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py deleted file mode 100644 index 1b1625c074..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Single rules catalogue and active-ruleset resolution.""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - custom_rule_implementation_fingerprint, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors -from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native -from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig, RulesResult - - -def evaluate_project( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - dialect: str = "generic", - verify_determinism: bool = False, -) -> RulesResult: - """Evaluate selected rules through one native project boundary.""" - - effective_config: RulesConfig = resolve_rule_ignore_selectors(config=config, project=project) - catalogue: tuple[Rule, ...] = build_catalogue( - config=effective_config, - project_dir=project_dir, - include_custom=_config_selects_custom(effective_config), - ) - if any(rule.custom for rule in catalogue): - selected: tuple[Rule, ...] = select_rules( - catalogue=catalogue, config=effective_config, project_dir=project_dir - ) - verify_custom_rules(rules=selected, project_dir=project_dir) - return evaluate_native( - project=project, - config=effective_config, - project_dir=project_dir, - catalogue=catalogue, - dialect=dialect, - verify_determinism=verify_determinism, - ) - - -def resolve_ruleset(*, config: RulesConfig, project_dir: Path) -> ResolvedRuleset: - """Resolve one immutable ruleset for evaluation, inspection, caching, and skills.""" - - catalogue: tuple[Rule, ...] = build_catalogue( - config=config, - project_dir=project_dir, - include_custom=_config_selects_custom(config), - ) - rules: tuple[Rule, ...] = select_rules( - catalogue=catalogue, config=config, project_dir=project_dir - ) - custom_selected: bool = any(rule.custom for rule in rules) - cacheable: bool = True - if custom_selected: - verify_custom_rules(rules=rules, project_dir=project_dir) - payload: list[dict[str, object]] = [] - for rule in rules: - source_hash: str | None = None - if rule.source is not None: - source_hash = hashlib.sha256(Path(rule.source).read_bytes()).hexdigest() - implementation_hash: str = custom_rule_implementation_fingerprint( - rule=rule, project_dir=project_dir - ) - payload.append( - { - "code": rule.code, - "family": rule.family, - "slug": rule.slug, - "message": rule.message, - "remediation": rule.remediation, - "options": config.rule_options.get(rule.code, {}), - "source_hash": source_hash, - "implementation_hash": implementation_hash, - } - ) - rules_payload: dict[str, object] = { - "engine": _engine_fingerprint(), - "rules": payload, - "thresholds": config.thresholds, - "threshold_overrides": config.threshold_overrides, - "select_star_allow": config.select_star_allow, - "domains": config.domains, - "approved_source_tokens": config.approved_source_tokens, - "retired_source_tokens": config.retired_source_tokens, - "sql_tests": config.sql_tests, - "layout": config.layout, - "exceptions": config.rule_exceptions, - "graph_edge_exceptions": config.graph_edge_exceptions, - "ignores": config.rule_ignores, - } - fingerprint: str = hashlib.sha256( - json.dumps(rules_payload, sort_keys=True, default=str).encode() - ).hexdigest() - return ResolvedRuleset( - catalogue=catalogue, - rules=rules, - fingerprint=fingerprint, - cacheable=cacheable, - ) - - -def _engine_fingerprint() -> str: - digest: Any = hashlib.sha256() - engine_root: Path = Path(__file__).parents[2] - for path in sorted(engine_root.rglob("*.py")): - digest.update(path.relative_to(engine_root).as_posix().encode()) - digest.update(path.read_bytes()) - return digest.hexdigest() - - -def _config_selects_custom(config: RulesConfig) -> bool: - ignored_selectors: list[str] = [] - for entry in config.rule_ignores: - ignored_selectors.extend(entry.rules) - selectors: tuple[str, ...] = ( - *config.select, - *config.ignore, - *config.rule_options, - *(entry.rule for entry in config.rule_exceptions), - *ignored_selectors, - ) - return any(selector.startswith("XSQBR") for selector in selectors) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py deleted file mode 100644 index 156d4b925f..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Deterministic rules result and rule rendering.""" - -from __future__ import annotations - -from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines -from sqlbuild.rule_engine.models import Rule, RulesConfig - -_MINIMUM_RULES_CODES: frozenset[str] = frozenset({"SQBRTEST201", "SQBRTEST202"}) - - -def format_rule_text(*, rule: Rule, config: RulesConfig | None = None) -> str: - lines: list[str] = [ - f"{rule.code}: {rule.slug}", - f"Family: {rule.family}", - f"Enabled by default: {'yes' if rule.enabled_by_default else 'no'}", - f"Kind: {'custom' if rule.custom else 'built-in'}", - "", - rule.message, - "", - f"Remediation: {rule.remediation}", - ] - if rule.source is not None: - lines.extend(("", f"Source: {rule.source}")) - if rule.options: - lines.extend(("", "Options:")) - for option in rule.options: - lines.append(f"- {option.name}: default={option.default!r} ({option.description})") - if config is not None and rule.code in _MINIMUM_RULES_CODES: - lines.extend(("", "Effective thresholds:", *format_threshold_lines(config=config))) - if config is not None and rule.code.startswith("SQBRTEST"): - lines.extend( - ( - "", - "Effective SQL test paths:", - "- Unit tests: tests/unit/", - "- Scenarios: tests/scenarios/", - f"- Cross-domain pipelines: tests/unit/{config.sql_tests.pipeline_directory}/", - ) - ) - return "\n".join(lines) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py b/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py deleted file mode 100644 index cbe6fa7817..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Atomic generated-skill installation phases.""" - -from __future__ import annotations - -from contextlib import ExitStack -from pathlib import Path -from stat import S_IMODE -from tempfile import NamedTemporaryFile - -from sqlbuild._native import skill_freshness -from sqlbuild.rule_engine.constants import REPLACEABLE_SKILL_STATES -from sqlbuild.rule_engine.exceptions import RulesError - - -def install_rendered_skills( - *, content: str, input_fingerprint: str, targets: tuple[Path, ...] -) -> None: - """Validate and atomically replace all generated skill targets.""" - - freshness: tuple[str, ...] = tuple( - skill_freshness( - content=path.read_text(encoding="utf-8") if path.is_file() else None, - input_fingerprint=input_fingerprint, - ) - for path in targets - ) - for path, state in zip(targets, freshness, strict=True): - if state not in REPLACEABLE_SKILL_STATES: - raise RulesError(f"refusing to overwrite {state} rules skill: {path}") - staged: tuple[tuple[Path, Path | None, Path], ...] = _stage_skills( - targets=targets, content=content - ) - _ = _apply_staged_skills(staged=staged, installed_bytes=content.encode()) - - -def _apply_staged_skills( - *, staged: tuple[tuple[Path, Path | None, Path], ...], installed_bytes: bytes -) -> None: - retained_backups: set[Path] = set() - replaced: list[tuple[Path | None, Path]] = [] - try: - for temporary, backup, target in staged: - current: bytes | None = target.read_bytes() if target.is_file() else None - expected: bytes | None = backup.read_bytes() if backup is not None else None - if current != expected: - raise RulesError( - f"refusing to overwrite rules skill changed during installation: {target}" - ) - temporary.replace(target) - replaced.append((backup, target)) - except Exception as error: - retained_backups = _rollback_replaced_skills( - replaced=tuple(replaced), - installed_bytes=installed_bytes, - error=error, - ) - raise - finally: - _remove_staged(staged=staged, retained_backups=retained_backups) - - -def _stage_skills( - *, targets: tuple[Path, ...], content: str -) -> tuple[tuple[Path, Path | None, Path], ...]: - staged: list[tuple[Path, Path | None, Path]] = [] - with ExitStack() as staging_cleanup: - for target in targets: - target.parent.mkdir(parents=True, exist_ok=True) - target_mode: int = S_IMODE(target.stat().st_mode) if target.is_file() else 0o644 - with NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=target.parent, - prefix=f".{target.name}.", - suffix=".tmp", - delete=False, - ) as temporary_file: - temporary: Path = Path(temporary_file.name) - staged.append((temporary, None, target)) - _ = staging_cleanup.callback(temporary.unlink, missing_ok=True) - _ = temporary_file.write(content) - temporary.chmod(target_mode) - if target.is_file(): - with NamedTemporaryFile( - mode="wb", - dir=target.parent, - prefix=f".{target.name}.", - suffix=".bak", - delete=False, - ) as backup_file: - backup: Path = Path(backup_file.name) - staged[-1] = (temporary, backup, target) - _ = staging_cleanup.callback(backup.unlink, missing_ok=True) - _ = backup_file.write(target.read_bytes()) - backup.chmod(target_mode) - _ = staging_cleanup.pop_all() - return tuple(staged) - - -def _rollback_replaced_skills( - *, - replaced: tuple[tuple[Path | None, Path], ...], - installed_bytes: bytes, - error: Exception, -) -> set[Path]: - retained_backups: set[Path] = set() - for backup, target in reversed(replaced): - try: - if not target.is_file() or target.read_bytes() != installed_bytes: - if backup is not None: - retained_backups.add(backup) - error.add_note( - f"rollback preserved concurrently changed target {target}; " - f"recovery backup: {backup}" - ) - elif backup is None: - target.unlink(missing_ok=True) - else: - backup.replace(target) - except Exception as rollback_error: - if backup is not None: - retained_backups.add(backup) - error.add_note( - f"rollback failed for {target}: {rollback_error}; recovery backup: {backup}" - ) - return retained_backups - - -def _remove_staged( - *, - staged: tuple[tuple[Path, Path | None, Path], ...], - retained_backups: set[Path], -) -> None: - for temporary, backup, _target in staged: - temporary.unlink(missing_ok=True) - if backup is not None and backup not in retained_backups: - backup.unlink(missing_ok=True) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py deleted file mode 100644 index 266b9ee7f3..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Rules agent-guidance rendering.""" - -from __future__ import annotations - -from collections.abc import Mapping -from pathlib import Path - -from sqlbuild._native import render_owned_skill -from sqlbuild.rule_engine._helpers.engine.ruleset import resolve_ruleset -from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines -from sqlbuild.rule_engine.constants import ( - RULES_LAYOUT_RULE_PREFIXES, - RULES_LAYOUT_THRESHOLD_RULE_CODES, - RULES_THRESHOLD_RULE_CODES, -) -from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig - - -def render_skills(*, config: RulesConfig, project_dir: Path) -> tuple[str, str]: - """Render stable guidance from ordinary rules catalogue resolution.""" - - ruleset: ResolvedRuleset = resolve_ruleset(config=config, project_dir=project_dir) - rules: tuple[Rule, ...] = ruleset.rules - body: list[str] = [ - "# SQLBuild Rules", - "", - "Rules enforce SQL model architecture and never modifies SQL files.", - "Run `sqb rules run SQBR` before handing off model changes.", - "", - "## Active Rules", - ] - for rule in rules: - body.extend( - ( - "", - f"### {rule.code}: {rule.slug}", - "", - rule.message, - "", - f"Correct shape: {_rule_example(rule=rule)}", - "", - f"Remediation: {rule.remediation}", - ) - ) - configured: Mapping[str, object] = config.rule_options.get(rule.code, {}) - effective: dict[str, object] = { - option.name: configured.get(option.name, option.default) for option in rule.options - } - if effective: - body.extend(("", f"Effective options: {effective}")) - if ( - config.rule_exceptions - or config.graph_edge_exceptions - or config.rule_ignores - or config.select_star_allow - ): - body.extend( - ( - "", - "## Scoped Deviations", - "", - ) - ) - for entry in config.rule_exceptions: - body.append(f"- Exception `{entry.rule}` at `{entry.path}`: {entry.reason}") - for entry in config.rule_ignores: - scopes: tuple[str, ...] = ( - *(f"path:{path}" for path in entry.paths), - *(f"selector:{selector}" for selector in entry.selectors), - ) - body.append( - f"- Ignore `{','.join(entry.rules)}` at `{','.join(scopes)}`: {entry.reason}" - ) - for entry in config.graph_edge_exceptions: - body.append( - f"- `SQBRGRAPH101` exact edge `{entry.consumer}` -> " - f"`{entry.dependency}`: {entry.reason}" - ) - for entry in config.select_star_allow: - body.append(f"- Lone-star allowance `{','.join(entry.paths)}`: {entry.reason}") - if any( - rule.code in RULES_THRESHOLD_RULE_CODES or rule.code in RULES_LAYOUT_THRESHOLD_RULE_CODES - for rule in rules - ): - body.extend(("", "## Effective Thresholds", "")) - body.extend(format_threshold_lines(config=config)) - if any(rule.code.startswith(RULES_LAYOUT_RULE_PREFIXES) for rule in rules): - body.extend(("", "## Owner Layout", "")) - body.append(f"- Configured levels: `{', '.join(config.layout.levels)}`") - body.append("- Every model owner is a leaf or a branch, never both.") - body.append("- Declaration-role buckets organize files without changing visibility.") - if any(rule.code.startswith("SQBRTEST") for rule in rules): - body.extend(("", "## SQL Test Paths", "")) - body.append("- Unit tests: `tests/unit/`") - body.append("- Scenarios: `tests/scenarios/`") - body.append( - f"- Cross-domain pipelines: `tests/unit/{config.sql_tests.pipeline_directory}/`" - ) - if config.domains or config.approved_source_tokens or config.retired_source_tokens: - body.extend(("", "## Naming Vocabulary", "")) - body.append(f"- Domains: `{', '.join(config.domains)}`") - body.append(f"- Approved source tokens: `{', '.join(config.approved_source_tokens)}`") - for retired, replacement in sorted(config.retired_source_tokens.items()): - body.append(f"- Replace retired `{retired}` with `{replacement}`") - content: str = "\n".join(body).rstrip() + "\n" - return ( - render_owned_skill(content=content, input_fingerprint=ruleset.fingerprint), - ruleset.fingerprint, - ) - - -def _rule_example(*, rule: Rule) -> str: - examples: dict[str, str] = { - "SQBRSQL040": ( - "Compute category_key in an input CTE, then JOIN categories AS c " - "ON o.category_key = c.category_key." - ), - "SQBRSQL041": ( - "WITH orders AS (...), final AS (SELECT order_id FROM orders) " - "SELECT order_id FROM final." - ), - "SQBRMODEL101": 'WITH upstream AS (SELECT * FROM __ref("domain__stg__entity")), ...', - "SQBRMODEL102": "SELECT id, status FROM final; use a lone SELECT * only when allowed.", - "SQBRMODEL103": "A view uses stg_v, int_v, or mart_v; a table uses a non-v layer.", - "SQBRMODEL104": "MODEL() omits schema; central path defaults provide it.", - "SQBRGRAPH101": ( - "Internal flow is stg -> int_clean -> int_enriched; marts are published interfaces." - ), - "SQBRPROJECT101": "domain__int_clean__entity or domain__mart_v__entity.", - "SQBRPROJECT105": "A domain__int_enriched__entity model lives under intermediate/enriched.", - "SQBRPROJECT106": "A domain__stg__entity model resolves to the staging logical schema.", - "SQBRCONTRACT101": ( - "MODEL (contract enforced, columns (...)); declare authoritative columns." - ), - "SQBRPROJECT202": "domain/level/owner contains models or child owners, never both.", - "SQBRPROJECT201": "models///model.sql resolves one domain root.", - "SQBRPROJECT203": "domain/level/subdomain/model.sql at the default depth of one.", - "SQBRPROJECT204": "order_status and order_status_history consolidate under order_status.", - "SQBRDECLARATION302": "_macros/ is flat, or every file uses one concern bucket.", - "SQBRDECLARATION303": "_macros/normalisation/names.py uses one default bucket level.", - "SQBRDECLARATION304": "Keep each flat role or concern bucket within its file cap.", - "SQBRDECLARATION305": "Use temporal/ or scoring/, not utils/ or common/.", - "SQBRDECLARATION306": "Group normalise_order.py and normalise_person.py under normalise/.", - "SQBRTEST101": "Keep unit tests in tests/unit/ and scenarios in tests/scenarios/.", - "SQBRTEST102": "test_stg_orders__excludes_cancelled.sql and daily_revenue__minimal.sql.", - "SQBRTEST103": "models/staging/stg_orders.sql maps to tests/unit/staging/.", - "SQBRTEST104": 'TEST (name "stg_orders__excludes_cancelled_orders");', - "SQBRTEST201": "Attach not_null/unique audits to the model key.", - } - if rule.guidance is not None: - return rule.guidance.good_example - return examples.get(rule.code, f"Apply `{rule.slug}` at the SQL node named by the finding.") diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py b/src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py deleted file mode 100644 index 522842924c..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Shared rendering for effective rule test thresholds.""" - -from sqlbuild.rule_engine.constants import RULES_THRESHOLD_DEFAULTS -from sqlbuild.rule_engine.models import RulesConfig - - -def format_threshold_lines(*, config: RulesConfig) -> tuple[str, ...]: - """Render global and path-scoped thresholds in deterministic authored order.""" - - thresholds: dict[str, int] = {**RULES_THRESHOLD_DEFAULTS, **config.thresholds} - lines: list[str] = [f"- `{name}` = `{value}`" for name, value in sorted(thresholds.items())] - for index, entry in enumerate(config.threshold_overrides, start=1): - lines.append( - f"- Override {index} at `{','.join(entry.paths)}`: " - f"`{dict(sorted(entry.thresholds.items()))}` ({entry.reason})" - ) - if config.threshold_overrides: - lines.append("- Matching overrides apply in authored order; the last match wins.") - return tuple(lines) diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 4923494533..76d0c2cbf9 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,9 +14,9 @@ from pathlib import Path from typing import Any, BinaryIO +from sqlbuild._native import build_rule_catalogue from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard @@ -80,7 +80,9 @@ def main() -> int: guard.guard_filesystem_metadata() messages: io.StringIO = io.StringIO() with contextlib.redirect_stdout(messages): - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=True + ) guard.raise_violation() by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} selected: tuple[Rule, ...] = tuple(by_code[code] for code in sorted(plan)) diff --git a/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py b/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py deleted file mode 100644 index ea496307ac..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Locations of the Rules caches, namespaced by compiler engine.""" - -from pathlib import Path - -from sqlbuild.compiler.frontier.constants import ( - NATIVE_CACHE_NAMESPACE_SUFFIX, - TARGET_DIRECTORY_NAME, -) -from sqlbuild.rule_engine.constants import ( - RULES_BULK_CACHE_DIRECTORY_NAME, - RULES_CACHE_DIRECTORY_NAME, -) - - -def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: - """Return one bulk Rules cache file owned by the active compiler engine.""" - - return ( - project_dir - / TARGET_DIRECTORY_NAME - / f"{RULES_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}" - / RULES_BULK_CACHE_DIRECTORY_NAME - / file_name - ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/findings.py b/src/sqlbuild/rule_engine/_helpers/run/findings.py deleted file mode 100644 index db350a1de3..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/findings.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public evaluation faults grouped after per-Rule suppression policy.""" - -from dataclasses import replace -from pathlib import Path - -from sqlbuild.rule_engine.models import Finding - - -def group_unevaluated_findings(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: - grouped: dict[tuple[Path, str], list[Finding]] = {} - result: list[Finding] = [] - for finding in findings: - if not finding.unevaluated: - result.append(finding) - continue - reason: str = finding.message.removeprefix( - f"Rule {finding.code} could not be evaluated" - ).lstrip(": ") - grouped.setdefault((finding.path, reason), []).append(finding) - for (_, reason), failures in grouped.items(): - affected_rules: tuple[str, ...] = tuple(sorted({failure.code for failure in failures})) - first: Finding = min(failures, key=lambda failure: (failure.line, failure.column)) - result.append( - replace( - first, - code="rules-unevaluated", - affected_rules=affected_rules, - message=( - f"{len(affected_rules)} selected Rules could not evaluate this resource: " - f"{reason}" - ), - ) - ) - return tuple(result) diff --git a/src/sqlbuild/rule_engine/_helpers/run/memory.py b/src/sqlbuild/rule_engine/_helpers/run/memory.py deleted file mode 100644 index ddc6c45869..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/memory.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Return memory the C allocator keeps after earlier compile phases freed it.""" - -from __future__ import annotations - -import ctypes -import sys -from collections.abc import Callable -from functools import cache -from typing import Any - - -def release_freed_memory() -> None: - """Hand freed glibc heap pages back to the OS; a no-op on other platforms and allocators.""" - - trim: Callable[[int], int] | None = _malloc_trim() - if trim is not None: - _ = trim(0) - - -@cache -def _malloc_trim() -> Callable[[int], int] | None: - if not sys.platform.startswith("linux"): - return None - try: - trim: Any = ctypes.CDLL(None).malloc_trim - except (OSError, AttributeError): - return None - trim.argtypes = (ctypes.c_size_t,) - trim.restype = ctypes.c_int - return trim diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py deleted file mode 100644 index 7af6ef3860..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Hand the native rules request builder rows read from the Python compiled project.""" - -from __future__ import annotations - -from dataclasses import asdict -from functools import partial -from pathlib import Path - -import orjson - -import sqlbuild._native as _native -from sqlbuild._native import compiled_project_facts, declaration_rows, scope_metadata_projection -from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine.constants import ( - NATIVE_RULES_CACHE_FILE, - NATIVE_RULES_MEMO_FILE, - RULES_NATIVE_API_VERSION, -) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Finding, NativeRulesEvaluation, RulesConfig - - -def build_rules_request( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - dialect: str, - initial_findings: tuple[Finding, ...], - custom_payloads: list[dict[str, object]], - include_type_proof: bool, -) -> _native.NativeRulesRequest: - """Build the built-in rules request natively from the compiled project's rows.""" - - resolved: Path = project_dir.resolve() - header: dict[str, object] = { - "version": RULES_NATIVE_API_VERSION, - "project_dir": str(resolved), - "dialect": dialect, - "config": asdict(config), - "scope_index": scope_metadata_projection(index=project.scope_index), - "defer_suppressions": True, - "custom_rules": custom_payloads, - "rules_cache_path": str( - rules_bulk_cache_path(project_dir=resolved, file_name=NATIVE_RULES_CACHE_FILE) - ), - } - public_enums, public_constants = declaration_rows( - enums=tuple(project.public_enums.values()), - constants=tuple(project.public_constants.values()), - ) - project_rows: dict[str, object] = { - "header_json": _encoded(header), - "memo": ( - ( - compiled_code_identity(), - str( - rules_bulk_cache_path(project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE) - ), - ) - if config.cache.enabled - else None - ), - "sql_analysis_enabled": project.settings.sql_analysis, - "include_type_proof": include_type_proof, - "attached_audit_targets": [ - audit.attached_target_name - for audit in project.audits - if audit.attached_target_name is not None - ], - "public_enums": public_enums, - "public_constants": public_constants, - "initial_findings": list(initial_findings), - "types_equal": partial(types_equal, dialect=dialect), - } - try: - request: _native.NativeRulesRequest - request = _native.build_rules_request( - project_rows, - (compiled_project_facts(project), [model.name for model in project.models]), - list(project.sql_tests), - list(project.sql_scenarios), - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - return request - - -def evaluate_rules_request(request: _native.NativeRulesRequest) -> NativeRulesEvaluation: - """Answer the request from the response memo or evaluate it, decoding findings natively.""" - - try: - faults, selected_codes, evaluated, hits, misses, built_in_ms, reused = ( - _native.evaluate_rules_request(request) - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - return NativeRulesEvaluation( - findings=tuple(faults), - selected_codes=tuple(selected_codes), - evaluated_models=evaluated, - cache_hits=hits, - cache_misses=misses, - built_in_ms=built_in_ms, - reused=reused, - ) - - -def finalize_rules_rows( - *, - config: RulesConfig, - project_dir: Path, - evaluated_codes: tuple[str, ...], - findings: tuple[Finding, ...], -) -> tuple[Finding, ...]: - """Apply the exception policy natively, keeping each original finding's fixable mark.""" - - try: - finalized: list[Finding] = _native.finalize_rule_findings_rows( - { - "project_dir": str(project_dir.resolve()), - "config_json": _encoded(asdict(config)), - "evaluated_codes": list(evaluated_codes), - "findings": list(findings), - } - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - return tuple(finalized) - - -def _encoded(value: object) -> bytes: - try: - return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=str) - except orjson.JSONEncodeError as error: - raise RulesError(str(error)) from error diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 962a16c4ad..32a8c500c5 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -7,7 +7,15 @@ from pathlib import Path from threading import Event -from sqlbuild._native import build_lint_expansion_context, run_lint +from sqlbuild._native import ( + build_lint_expansion_context, + early_lint_expansions, + finish_rule_run, + plan_rule_run, + run_lint, + run_planned_sql_rules, + sql_lint_preparation, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -20,13 +28,6 @@ from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules -from sqlbuild.rule_engine._helpers.run.rules import ( - early_lint_expansions, - finish_rule_run, - plan_rule_run, - run_planned_sql_rules, - sql_lint_preparation, -) from sqlbuild.rule_engine.models import ( CustomRulesOutcome, PreparedSqlLint, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py deleted file mode 100644 index ccbf7171c9..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ /dev/null @@ -1,699 +0,0 @@ -"""Unified compiler-rule orchestration implementation.""" - -from __future__ import annotations - -import json -import time -from dataclasses import asdict, dataclass, replace -from pathlib import Path - -from sqlbuild._native import ( - build_lint_expansion_context, - collect_lint_project_files, - compiled_relation_keys, - declared_relation_keys, - encode_sql_rule_findings, - load_sql_rule_cache, - lookup_sql_rule_cache, - project_with, - run_lint, - selected_sql_rule_codes, - sql_file_rule_key, - sql_model_rule_keys, - sql_rule_lint_key, - with_duplicate_literal_hints, - write_sql_rule_cache, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlExpansion, - DeclarationScopeBuild, - SqlExpansionContext, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompileInputsView, - ModelInput, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.constants import ( - DEFAULT_MAX_LITERAL_LENGTH, - DEFAULT_MAX_RANKING_ORDER_BY, - MAX_LITERAL_LENGTH_THRESHOLD, - MAX_RANKING_ORDER_BY_THRESHOLD, - SAFE_RULE_FIX_CODES, -) -from sqlbuild.lint.exceptions import NativeLintError -from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation -from sqlbuild.lint.types import RelationKeys -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors -from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads -from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings -from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory -from sqlbuild.rule_engine._helpers.run.native_rows import finalize_rules_rows -from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.models import ( - Finding, - PreparedSqlLint, - PreparedSqlLintResult, - Rule, - RuleRunPlan, - RulesConfig, - RulesResult, - RulesRunResult, - SqlExpansionReuse, - SqlRulesEvaluation, -) - -_SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" - - -@dataclass(frozen=True) -class _SqlRuleInputs: - identities: dict[str, str] - project_files: dict[Path, str] | None - - -def plan_rule_run( - *, - graph: ProjectGraph, - config: RulesConfig, - project_dir: Path, - selected_keys: frozenset[CompiledObjectKey] | None, - no_sql_analysis: bool, -) -> RuleRunPlan: - """Build the catalogue and select the rules, project slice and settings one run evaluates.""" - - release_freed_memory() - effective_config: RulesConfig = resolve_rule_ignore_selectors( - config=config, project=graph.project - ) - resolved_project_dir: Path = project_dir.resolve() - include_custom: bool = _config_references_custom_rules(effective_config) - catalogue: tuple[Rule, ...] = build_catalogue( - config=effective_config, - project_dir=resolved_project_dir, - include_custom=include_custom, - ) - custom_payloads: list[dict[str, object]] = custom_rule_payloads( - catalogue=catalogue, project_dir=resolved_project_dir - ) - selected: tuple[Rule, ...] = select_rules( - catalogue=catalogue, - config=effective_config, - project_dir=resolved_project_dir, - custom_payloads=custom_payloads, - ) - skipped_type_proof_rules: tuple[str, ...] = () - if no_sql_analysis or not graph.project.settings.sql_analysis: - skipped_type_proof_rules = tuple( - rule.code for rule in selected if rule.code in TYPE_PROOF_RULE_CODES - ) - selected = tuple(rule for rule in selected if rule.code not in TYPE_PROOF_RULE_CODES) - custom_rules: tuple[Rule, ...] = tuple(rule for rule in selected if rule.custom) - if custom_rules: - _ = verify_custom_rules(rules=custom_rules, project_dir=resolved_project_dir) - model_paths: frozenset[str] | None = _selected_model_paths( - graph=graph, selected_keys=selected_keys - ) - selected_project: CompiledProjectView = _selected_project(graph=graph, model_paths=model_paths) - selected_config: RulesConfig = replace( - _selection_rules_config( - config=effective_config, - model_paths=model_paths, - project=selected_project, - ), - select=tuple(rule.code for rule in selected), - ignore=(), - ) - return RuleRunPlan( - effective_config=effective_config, - project_dir=resolved_project_dir, - catalogue=catalogue, - custom_payloads=custom_payloads, - selected=selected, - skipped_type_proof_rules=skipped_type_proof_rules, - model_paths=model_paths, - project=selected_project, - config=selected_config, - ) - - -def run_planned_sql_rules( - *, - plan: RuleRunPlan, - graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, - dialect: str, - prepared_sql: PreparedSqlLint | None, - expansion_reuse: SqlExpansionReuse | None, -) -> SqlRulesEvaluation: - """Run the planned SQL rules on the caller while built-in rules evaluate natively.""" - - native_rules: tuple[Rule, ...] = tuple(rule for rule in plan.selected if not rule.custom) - lint_config: LintConfig = sql_lint_config( - dialect=dialect, - codes=_sql_rule_codes(native_rules), - thresholds=plan.effective_config.thresholds, - relation_keys=compiled_relation_keys(graph.project), - allow_suppressions=plan.effective_config.allow_exceptions, - ) - sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( - rules=native_rules, - project=plan.project, - dialect=sql_rule_lint_key(config=lint_config), - project_dir=plan.project_dir, - collect_files=prepared_sql is None and plan.model_paths is None, - discovered_inputs=discovered_inputs, - ) - sql_started: float = time.monotonic() - sql_result: SqlRulesEvaluation = _run_sql_rules( - rules=native_rules, - project_dir=plan.project_dir, - discovered_inputs=discovered_inputs, - project=plan.project, - lint_config=lint_config, - selected_model_paths=plan.model_paths, - cache_enabled=plan.effective_config.cache.enabled, - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - prepared_inputs=sql_rule_inputs, - ) - return replace(sql_result, sql_ms=round((time.monotonic() - sql_started) * 1000)) - - -def finish_rule_run( - *, plan: RuleRunPlan, sql_result: SqlRulesEvaluation, result: RulesResult -) -> RulesRunResult: - """Finalize the combined SQL, built-in and custom findings of one run.""" - - finalized: tuple[Finding, ...] = finalize_rules_rows( - config=plan.config, - project_dir=plan.project_dir, - evaluated_codes=tuple(rule.code for rule in plan.selected), - findings=(*sql_result.findings, *result.findings), - ) - findings: tuple[Finding, ...] = tuple( - sorted( - group_unevaluated_findings(finalized), - key=lambda item: (item.path.as_posix(), item.line, item.column, item.code), - ) - ) - return RulesRunResult( - findings=findings, - evaluated_models=max( - 0, - result.evaluated_models - - len( - {item.path for item in findings if item.unevaluated} - & {model.relative_path for model in plan.project.models} - ), - ), - unevaluated_resources=len({item.path for item in findings if item.unevaluated}), - built_in_ms=sql_result.sql_ms + result.built_in_ms, - custom_ms=result.custom_ms, - custom_cpu_ms=result.custom_cpu_ms, - cache_hits=result.cache_hits + sql_result.cache_hits, - cache_misses=result.cache_misses + sql_result.cache_misses, - skipped_type_proof_rules=plan.skipped_type_proof_rules, - ) - - -def _prepare_sql_rule_inputs( - *, - rules: tuple[Rule, ...], - project: CompiledProjectView, - dialect: str, - project_dir: Path, - collect_files: bool, - discovered_inputs: DiscoveredProjectInputs, -) -> _SqlRuleInputs: - """Hash SQL and read the project files on the caller while built-in rules evaluate natively.""" - - identities: dict[str, str] = _sql_model_rule_identities( - rules=rules, project=project, dialect=dialect - ) - return _SqlRuleInputs( - identities=identities, - project_files=( - collect_lint_project_files(project_dir=project_dir, selected_paths=None) - if identities and collect_files - else None - ), - ) - - -def _config_references_custom_rules(config: RulesConfig) -> bool: - ignore_selectors: list[str] = [] - for entry in config.rule_ignores: - ignore_selectors.extend(entry.rules) - selectors: tuple[str, ...] = ( - *config.select, - *config.ignore, - *config.rule_options, - *(entry.rule for entry in config.rule_exceptions), - *ignore_selectors, - ) - return any(selector.startswith("XSQBR") for selector in selectors) - - -def sql_lint_preparation( - *, inputs: CompileInputsView, dialect: str -) -> tuple[Path, tuple[str, ...], LintConfig] | None: - """The project, SQL rule codes and lint settings early SQL lint runs with, if it should run.""" - - project_dir: Path | None = inputs.discovered_inputs.project_dir - if project_dir is None or any(diagnostic.is_error for diagnostic in inputs.diagnostics): - return None - try: - config: RulesConfig = load_rules_config(project_dir=project_dir) - except RulesError: - return None - if not config.select or (config.cache.enabled and _read_sql_rule_cache(project_dir)): - return None - codes: tuple[str, ...] = _selected_sql_codes(config) - if not codes: - return None - return ( - project_dir, - codes, - sql_lint_config( - dialect=dialect, - codes=codes, - thresholds=config.thresholds, - allow_suppressions=config.allow_exceptions, - relation_keys=declared_relation_keys( - model_rows=(model.relation_key_row for model in inputs.model_inputs), - seeds=(seed.schema_entry for seed in inputs.seed_inputs), - sources=(source.source_entry for source in inputs.source_inputs), - ), - ), - ) - - -def early_lint_expansions( - model_inputs: tuple[ModelInput, ...], -) -> dict[Path, CompiledSqlExpansion]: - """Each model's compile expansion for early lint, whole when the compile handed one over.""" - - expansions: dict[Path, CompiledSqlExpansion] = {} - for model in model_inputs: - expansion: CompiledSqlExpansion | None = model.lint_expansion or model.sql_expansion - if expansion is not None: - expansions[model.model_file.file_path] = expansion - return expansions - - -def _selection_rules_config( - *, - config: RulesConfig, - model_paths: frozenset[str] | None, - project: CompiledProjectView, -) -> RulesConfig: - if model_paths is None: - return config - selected_model_names: frozenset[str] = frozenset(model.name for model in project.models) - return replace( - config, - rule_exceptions=tuple( - entry for entry in config.rule_exceptions if Path(entry.path).as_posix() in model_paths - ), - graph_edge_exceptions=tuple( - entry - for entry in config.graph_edge_exceptions - if entry.consumer in selected_model_names - ), - ) - - -def _selected_sql_codes(config: RulesConfig) -> tuple[str, ...]: - return tuple(selected_sql_rule_codes(select=list(config.select), ignore=list(config.ignore))) - - -def _selected_project( - *, graph: ProjectGraph, model_paths: frozenset[str] | None -) -> CompiledProjectView: - if model_paths is None: - return graph.project - return project_with( - project=graph.project, - models=tuple( - model for model in graph.project.models if model.relative_path.as_posix() in model_paths - ), - ) - - -def _prepared_project_files( - *, prepared_sql: PreparedSqlLint | None, project_dir: Path, codes: tuple[str, ...] -) -> dict[Path, str] | None: - """Reuse the full-project read of a matching early SQL lint for this invocation.""" - - if ( - prepared_sql is None - or prepared_sql.project_dir != project_dir - or set(prepared_sql.codes) != set(codes) - or prepared_sql.future.exception() is not None - ): - return None - return dict(prepared_sql.future.result().result.source_texts) - - -def _run_prepared_lint( - *, - project_dir: Path, - config: LintConfig, - selected_paths: frozenset[Path], - discovered_inputs: DiscoveredProjectInputs, - compiled_expansions: dict[Path, CompiledSqlExpansion], - dynamic_output_paths: frozenset[Path], - prepared_sql: PreparedSqlLint | None, - expansion_reuse: SqlExpansionReuse | None, - source_files: dict[Path, str] | None, -) -> LintRunResult: - if prepared_sql is None or set(prepared_sql.codes) != set(config.enabled_native_rules or ()): - return run_lint( - project_dir=project_dir, - config=config, - selected_paths=selected_paths, - discovered_inputs=discovered_inputs, - compiled_expansions=compiled_expansions, - dynamic_output_paths=dynamic_output_paths, - expansion_context=( - _lint_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - expansion_reuse=expansion_reuse, - ) - if config.native_enabled - else None - ), - source_files=source_files, - ) - completed: PreparedSqlLintResult = prepared_sql.future.result() - prepared: LintRunResult = completed.result - proof_paths: frozenset[Path] = selected_paths & dynamic_output_paths - violations: tuple[LintViolation, ...] = tuple( - violation - for violation in prepared.violations - if violation.file_path in selected_paths and violation.file_path not in proof_paths - ) - if proof_paths: - proven: LintRunResult = run_lint( - project_dir=project_dir, - config=config, - selected_paths=proof_paths, - discovered_inputs=discovered_inputs, - compiled_expansions=compiled_expansions, - dynamic_output_paths=proof_paths, - expansion_context=completed.context, - source_files=dict(prepared.source_texts), - ) - violations = (*violations, *proven.violations) - return replace(prepared, violations=violations, files_checked=len(selected_paths)) - - -def _run_sql_rules( - *, - rules: tuple[Rule, ...], - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - project: CompiledProjectView, - lint_config: LintConfig, - selected_model_paths: frozenset[str] | None, - cache_enabled: bool, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, - prepared_inputs: _SqlRuleInputs | None = None, -) -> SqlRulesEvaluation: - codes: tuple[str, ...] = _sql_rule_codes(rules) - if not codes: - return SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) - models_by_path: dict[str, CompiledModelView] = _sql_rule_models(project) - bucket: dict[str, tuple[str, str]] = _read_sql_rule_cache(project_dir) if cache_enabled else {} - identities: dict[str, str] = ( - prepared_inputs.identities - if prepared_inputs is not None - else _sql_model_rule_identities( - rules=rules, project=project, dialect=sql_rule_lint_key(config=lint_config) - ) - ) - findings: list[Finding] = [] - model_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( - bucket=bucket, keys=identities - ) - misses: set[str] = set(identities) - set(model_hits) - for path in identities: - findings.extend(model_hits.get(path, ())) - selected_paths: set[Path] = {project_dir / path for path in misses} - file_identities: dict[str, str] = {} - project_files: dict[Path, str] | None = None - if selected_model_paths is None: - model_paths: frozenset[Path] = frozenset( - project_dir / model.relative_path for model in project.models - ) - project_files = _prepared_project_files( - prepared_sql=prepared_sql, project_dir=project_dir, codes=codes - ) - if project_files is None: - project_files = ( - prepared_inputs.project_files - if prepared_inputs is not None and prepared_inputs.project_files is not None - else collect_lint_project_files(project_dir=project_dir, selected_paths=None) - ) - lint_key: str = sql_rule_lint_key(config=lint_config) - file_keys: dict[str, str | None] = { - file_path.relative_to(project_dir).as_posix(): sql_file_rule_key( - lint_key=lint_key, - codes=list(codes), - relative_path=file_path.relative_to(project_dir).as_posix(), - contents=contents, - expanded=file_path in project.sql_expansions, - ) - for file_path, contents in project_files.items() - if file_path not in model_paths - } - file_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( - bucket=bucket, - keys={path: key for path, key in file_keys.items() if key is not None}, - ) - file_path: Path - for file_path in project_files: - if file_path in model_paths: - continue - relative_path: str = file_path.relative_to(project_dir).as_posix() - file_identity: str | None = file_keys[relative_path] - cached_file: tuple[Finding, ...] | None = file_hits.get(relative_path) - if cached_file is None: - selected_paths.add(file_path) - if file_identity is not None: - file_identities[relative_path] = file_identity - else: - findings.extend(cached_file) - result: LintRunResult | None = ( - _run_prepared_lint( - project_dir=project_dir, - config=lint_config, - selected_paths=frozenset(selected_paths), - discovered_inputs=discovered_inputs, - compiled_expansions=project.sql_expansions, - dynamic_output_paths=frozenset( - (project_dir / model.relative_path).resolve() - for model in models_by_path.values() - if model.dynamic_column_contract is not None - and model.dynamic_column_contract.output_proven - ), - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - source_files=project_files, - ) - if selected_paths - else None - ) - selected_codes: frozenset[str] = frozenset((*codes, _SQL_RULE_SUPPRESSION_CODE)) - evaluated: list[Finding] = [] - for violation in () if result is None else result.violations: - if violation.code in selected_codes: - evaluated.append( - _lint_finding( - violation=violation, - project_dir=project_dir, - fixable_paths=frozenset(models_by_path), - ) - ) - elif violation.code == NativeLintError.code: - finding: Finding = _lint_finding( - violation=violation, project_dir=project_dir, fixable_paths=frozenset() - ) - evaluated.extend( - replace( - finding, - code=code, - message=f"Rule {code} could not be evaluated: {violation.message}", - unevaluated=True, - ) - for code in codes - ) - findings.extend(evaluated) - written_identities: dict[str, str] = { - **{path: identities[path] for path in misses}, - **file_identities, - } - if cache_enabled and written_identities: - by_path: dict[str, list[Finding]] = {path: [] for path in written_identities} - for finding in evaluated: - path: str = finding.path.as_posix() - if path in by_path: - by_path[path].append(finding) - for path, identity in written_identities.items(): - bucket[path] = (identity, encode_sql_rule_findings(findings=by_path[path])) - write_sql_rule_cache(path=_sql_rule_cache_path(project_dir), entries=bucket) - return SqlRulesEvaluation( - findings=with_duplicate_literal_hints(findings=tuple(findings), project_dir=project_dir), - cache_hits=len(identities) - len(misses), - cache_misses=len(misses), - ) - - -def _lint_expansion_context( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - expansion_reuse: SqlExpansionReuse | None, -) -> SqlExpansionContext: - """Build lint expansion, reusing the compiler's scope only for the same discovery.""" - - declaration_scope: DeclarationScopeBuild | None = ( - expansion_reuse.declaration_scope - if expansion_reuse is not None and expansion_reuse.discovered_inputs is discovered_inputs - else None - ) - return build_lint_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - declaration_scope=declaration_scope, - ) - - -def _lint_finding( - *, violation: LintViolation, project_dir: Path, fixable_paths: frozenset[str] -) -> Finding: - """Convert one lint violation; fixable when `format --fix` would attempt its edit.""" - - path: Path = violation.file_path - if path.is_absolute() and path.is_relative_to(project_dir.resolve()): - path = path.relative_to(project_dir.resolve()) - return Finding( - code=violation.code, - path=path, - line=violation.line, - column=violation.column, - message=violation.message, - remediation=violation.remediation or "Update the authored SQL to satisfy this rule.", - fixable=violation.fix is not None - and violation.code in SAFE_RULE_FIX_CODES - and path.as_posix() in fixable_paths, - ) - - -def sql_lint_config( - *, - dialect: str, - codes: tuple[str, ...], - thresholds: dict[str, int], - relation_keys: RelationKeys, - allow_suppressions: bool, -) -> LintConfig: - """Build the native SQL lint configuration that SQL Rules run with.""" - - return LintConfig( - dialect=dialect, - enabled_native_rules=codes, - header_rules_enabled=False, - max_literal_length=thresholds.get(MAX_LITERAL_LENGTH_THRESHOLD, DEFAULT_MAX_LITERAL_LENGTH), - max_ranking_order_by=thresholds.get( - MAX_RANKING_ORDER_BY_THRESHOLD, DEFAULT_MAX_RANKING_ORDER_BY - ), - relation_keys=relation_keys, - allow_suppressions=allow_suppressions, - ) - - -def _sql_rule_codes(rules: tuple[Rule, ...]) -> tuple[str, ...]: - return tuple(rule.code for rule in rules if rule.code.startswith("SQBRSQL")) - - -def _sql_rule_models(project: CompiledProjectView) -> dict[str, CompiledModelView]: - return { - model.relative_path.as_posix(): model - for model in project.models - if project.settings.sql_analysis and model.config.values.get("sql_analysis") is not False - } - - -def _sql_model_rule_identities( - *, rules: tuple[Rule, ...], project: CompiledProjectView, dialect: str -) -> dict[str, str]: - codes: tuple[str, ...] = _sql_rule_codes(rules) - if not codes: - return {} - models: list[tuple[str, str, str, str | None]] = [ - (path, model.authored_sql, model.query_sql, _contract_text(model)) - for path, model in _sql_rule_models(project).items() - ] - keys: dict[str, str] = sql_model_rule_keys(lint_key=dialect, codes=list(codes), models=models) - return {path: keys[path] for path, *_ in models} - - -def _contract_text(model: CompiledModelView) -> str | None: - """Canonical text of the dynamic column contract that SQL rule findings depend on.""" - - if model.dynamic_column_contract is None: - return None - return json.dumps( - asdict(model.dynamic_column_contract), - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - -def _sql_rule_cache_path(project_dir: Path) -> Path: - return rules_bulk_cache_path(project_dir=project_dir, file_name=SQL_RULES_CACHE_FILE) - - -def _read_sql_rule_cache(project_dir: Path) -> dict[str, tuple[str, str]]: - """Return usable entries so incompatible builds cannot suppress early lint.""" - - return load_sql_rule_cache(path=_sql_rule_cache_path(project_dir)) - - -def _cached_sql_findings( - *, bucket: dict[str, tuple[str, str]], keys: dict[str, str] -) -> dict[str, tuple[Finding, ...]]: - """Decoded stored findings of each path whose key still matches; undecodable ones miss.""" - - hits: dict[str, list[Finding]] = lookup_sql_rule_cache(entries=bucket, keys=list(keys.items())) - return {path: tuple(findings) for path, findings in hits.items()} - - -def _selected_model_paths( - *, graph: ProjectGraph, selected_keys: frozenset[CompiledObjectKey] | None -) -> frozenset[str] | None: - if selected_keys is None: - return None - selected_names: frozenset[str] = frozenset( - key.name for key in selected_keys if key.resource_type is CompiledResourceType.MODEL - ) - return frozenset( - model.relative_path.as_posix() - for model in graph.project.models - if model.name in selected_names - ) diff --git a/src/sqlbuild/rule_engine/classes/early_sql_lint.py b/src/sqlbuild/rule_engine/classes/early_sql_lint.py deleted file mode 100644 index 1e0cd3d9c7..0000000000 --- a/src/sqlbuild/rule_engine/classes/early_sql_lint.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Own speculative SQL checks for one compilation invocation.""" - -from __future__ import annotations - -import time -from concurrent.futures import ThreadPoolExecutor -from threading import Event -from types import TracebackType - -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine.main._prepare_sql_rules import prepare_sql_rules -from sqlbuild.rule_engine.models import PreparedSqlLint, SqlExpansionReuse - - -class EarlySqlLint: - def __init__(self, *, enabled: bool) -> None: - self.enabled: bool = enabled - self.preparation: PreparedSqlLint | None = None - self.expansion_reuse: SqlExpansionReuse | None = None - self.stop_requested: Event = Event() - self.executor: ThreadPoolExecutor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="sqlbuild-lint" - ) - - def __enter__(self) -> EarlySqlLint: - return self - - def __exit__( - self, - exception_type: type[BaseException] | None, - exception: BaseException | None, - traceback: TracebackType | None, - ) -> None: - _ = self.stop() - - def start(self, *, inputs: CompileInputsView, dialect: str) -> None: - if self.enabled: - self.preparation = prepare_sql_rules( - inputs=inputs, - executor=self.executor, - dialect=dialect, - stop=self.stop_requested, - ) - self.expansion_reuse = ( - None - if inputs.declaration_scope is None - else SqlExpansionReuse( - discovered_inputs=inputs.discovered_inputs, - declaration_scope=inputs.declaration_scope, - ) - ) - - def stop(self) -> int: - """Abandon unconsumed speculative work and return the milliseconds spent joining it.""" - - stop_start: float = time.monotonic() - self.stop_requested.set() - self.executor.shutdown(wait=True, cancel_futures=True) - return int((time.monotonic() - stop_start) * 1000) diff --git a/src/sqlbuild/rule_engine/main/_evaluate.py b/src/sqlbuild/rule_engine/main/_evaluate.py index b239779aa9..0677a2dd9d 100644 --- a/src/sqlbuild/rule_engine/main/_evaluate.py +++ b/src/sqlbuild/rule_engine/main/_evaluate.py @@ -2,9 +2,15 @@ from pathlib import Path +from sqlbuild._native import ( + build_rule_catalogue, + resolve_rule_ignore_selectors, + select_catalogue_rules, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.ruleset import evaluate_project -from sqlbuild.rule_engine.models import RulesConfig, RulesResult +from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native +from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules +from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult def evaluate( @@ -17,10 +23,22 @@ def evaluate( ) -> RulesResult: """Evaluate the selected compiler rules over compiled models.""" - return evaluate_project( + effective_config: RulesConfig = resolve_rule_ignore_selectors(config=config, project=project) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=effective_config, project_dir=project_dir + ) + if any(rule.custom for rule in catalogue): + _ = verify_custom_rules( + rules=select_catalogue_rules( + catalogue=catalogue, config=effective_config, project_dir=project_dir + ), + project_dir=project_dir, + ) + return evaluate_native( project=project, - config=config, + config=effective_config, project_dir=project_dir, + catalogue=catalogue, dialect=dialect, verify_determinism=verify_determinism, ) diff --git a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py deleted file mode 100644 index c9be51978e..0000000000 --- a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Start reusable SQL lint work after all compiler inputs have been expanded.""" - -from __future__ import annotations - -from concurrent.futures import Executor -from threading import Event - -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine._helpers.run.rule_threads import prepare_sql_rules as prepare -from sqlbuild.rule_engine.models import PreparedSqlLint - - -def prepare_sql_rules( - *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event -) -> PreparedSqlLint | None: - """Prepare configured SQL checks without publishing findings before compilation finishes.""" - - return prepare(inputs=inputs, executor=executor, dialect=dialect, stop=stop) diff --git a/src/sqlbuild/rule_engine/main/build_catalogue.py b/src/sqlbuild/rule_engine/main/build_catalogue.py deleted file mode 100644 index 3cf786006a..0000000000 --- a/src/sqlbuild/rule_engine/main/build_catalogue.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Rules catalogue construction entrypoint.""" - -from pathlib import Path - -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue as _build_catalogue -from sqlbuild.rule_engine.models import Rule, RulesConfig - - -def build_catalogue(*, config: RulesConfig, project_dir: Path) -> tuple[Rule, ...]: - """Build the validated built-in and custom rules catalogue.""" - - return _build_catalogue(config=config, project_dir=project_dir) diff --git a/src/sqlbuild/rule_engine/main/load_config.py b/src/sqlbuild/rule_engine/main/load_config.py deleted file mode 100644 index eb3a354929..0000000000 --- a/src/sqlbuild/rule_engine/main/load_config.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Rules configuration loading entrypoint.""" - -from pathlib import Path - -from sqlbuild._native import load_rules_config as _load_rules_config -from sqlbuild.rule_engine.models import RulesConfig - - -def load_rules_config(*, project_dir: Path) -> RulesConfig: - """Load strict rules configuration from a SQLBuild project.""" - - return _load_rules_config(project_dir=project_dir.resolve()) diff --git a/src/sqlbuild/rule_engine/main/render_result.py b/src/sqlbuild/rule_engine/main/render_result.py deleted file mode 100644 index bb6afd1c1c..0000000000 --- a/src/sqlbuild/rule_engine/main/render_result.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Rules result formatting entrypoint.""" - -from sqlbuild._native import format_rules_result -from sqlbuild.rule_engine.models import RulesResult - - -def format_result(*, result: RulesResult, json_output: bool) -> str: - """Format one rules result for the requested CLI surface.""" - - return format_rules_result(result=result, json_output=json_output) diff --git a/src/sqlbuild/rule_engine/main/render_rule.py b/src/sqlbuild/rule_engine/main/render_rule.py deleted file mode 100644 index c428ba36f3..0000000000 --- a/src/sqlbuild/rule_engine/main/render_rule.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Rule formatting entrypoint.""" - -from sqlbuild.rule_engine._helpers.guidance.presentation import format_rule_text -from sqlbuild.rule_engine.models import Rule, RulesConfig - - -def format_rule(*, rule: Rule, config: RulesConfig | None = None) -> str: - """Format one rule for inspection.""" - - return format_rule_text(rule=rule, config=config) diff --git a/src/sqlbuild/rule_engine/main/render_rules_progress.py b/src/sqlbuild/rule_engine/main/render_rules_progress.py deleted file mode 100644 index 11f4bc5531..0000000000 --- a/src/sqlbuild/rule_engine/main/render_rules_progress.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Rules phase completion line entrypoint.""" - -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -from sqlbuild.rule_engine.models import RulesRunResult - - -def format_rules_progress( - *, elapsed_seconds: float, result: RulesRunResult, note_skipped_rules: bool = False -) -> str: - """Report the rules phase wall time first, then built-in and custom rule time.""" - - message: str = ( - f"Evaluated rules. ({elapsed_seconds:.2f}s; built-in {result.built_in_ms / 1000:.2f}s, " - f"custom {result.custom_ms / 1000:.2f}s)" - ) - skipped_rules_note: str | None = ( - format_skipped_type_proof_rules(codes=result.skipped_type_proof_rules) - if note_skipped_rules - else None - ) - return message if skipped_rules_note is None else f"{message}; {skipped_rules_note}" diff --git a/src/sqlbuild/rule_engine/main/render_skipped_rules.py b/src/sqlbuild/rule_engine/main/render_skipped_rules.py deleted file mode 100644 index f883250905..0000000000 --- a/src/sqlbuild/rule_engine/main/render_skipped_rules.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Skipped type-proof rules note entrypoint.""" - - -def format_skipped_type_proof_rules(*, codes: tuple[str, ...]) -> str | None: - """Describe type-proof rules skipped because SQL analysis is disabled, if any.""" - - if not codes: - return None - return f"skipped type-proof rules ({', '.join(codes)}) because SQL analysis is disabled" diff --git a/src/sqlbuild/rule_engine/main/run_rules.py b/src/sqlbuild/rule_engine/main/run_rules.py deleted file mode 100644 index ea50bdb95a..0000000000 --- a/src/sqlbuild/rule_engine/main/run_rules.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Unified compiler-rule orchestration entrypoint.""" - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine._helpers.run.rule_threads import evaluate_rules -from sqlbuild.rule_engine.models import ( - PreparedSqlLint, - RulesConfig, - RulesRunResult, - SqlExpansionReuse, -) - - -def run_rules( - *, - graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, - config: RulesConfig, - project_dir: Path, - dialect: str, - selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, - no_sql_analysis: bool = False, -) -> RulesRunResult: - """Run selected native built-ins before selected custom Python rules.""" - - return evaluate_rules( - graph=graph, - discovered_inputs=discovered_inputs, - config=config, - project_dir=project_dir, - dialect=dialect, - selected_keys=selected_keys, - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - no_sql_analysis=no_sql_analysis, - ) diff --git a/src/sqlbuild/rule_engine/main/skills.py b/src/sqlbuild/rule_engine/main/skills.py deleted file mode 100644 index 68e6c5846e..0000000000 --- a/src/sqlbuild/rule_engine/main/skills.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Generate deterministic agent guidance from the resolved compiler rules.""" - -from __future__ import annotations - -import hashlib -import os -from pathlib import Path -from tempfile import gettempdir - -from filelock import FileLock - -from sqlbuild._native import skill_freshness -from sqlbuild.rule_engine._helpers.guidance.skill_installation import install_rendered_skills -from sqlbuild.rule_engine._helpers.guidance.skills import render_skills -from sqlbuild.rule_engine.constants import ( - RULES_SKILL_PATHS, - SKILL_FRESH, -) -from sqlbuild.rule_engine.models import RulesConfig - - -def install_skills(*, config: RulesConfig, project_dir: Path, check: bool) -> bool: - """Install all owned skill targets or report whether they are fresh.""" - - content: str - input_fingerprint: str - content, input_fingerprint = render_skills(config=config, project_dir=project_dir) - targets: tuple[Path, ...] = tuple(project_dir / value for value in RULES_SKILL_PATHS) - if check: - return all( - skill_freshness( - content=path.read_text(encoding="utf-8") if path.is_file() else None, - input_fingerprint=input_fingerprint, - ) - == SKILL_FRESH - for path in targets - ) - canonical_project: str = os.path.normcase(str(project_dir.resolve())) - lock_name: str = hashlib.sha256(canonical_project.encode()).hexdigest() - lock_dir: Path = Path(gettempdir()) / "sqlbuild-rules-skill-locks" - lock_dir.mkdir(parents=True, exist_ok=True) - with FileLock(lock_dir / f"{lock_name}.lock"): - _ = install_rendered_skills( - content=content, - input_fingerprint=input_fingerprint, - targets=targets, - ) - return True diff --git a/src/sqlbuild/runtime/event_exporting/main/event_exporter_command_scope.py b/src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py similarity index 100% rename from src/sqlbuild/runtime/event_exporting/main/event_exporter_command_scope.py rename to src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 70f7e9f404..ad3bd2d6d9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -31,8 +31,8 @@ import sqlbuild._native as native_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization -import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings +import sqlbuild.cli.compile_reuse.constants as reuse_constants import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class @@ -3397,7 +3397,7 @@ def record_digested_files(*, monkeypatch: pytest.MonkeyPatch) -> list[int]: def settle_racy_window(*, monkeypatch: pytest.MonkeyPatch) -> None: """Treat files written before a compile starts as settled instead of waiting two seconds.""" - monkeypatch.setattr(native_reuse, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(reuse_constants, "RACY_WINDOW_NS", 0) def write_large_file(*, project_dir: Path, relative_path: str, size_bytes: int) -> Path: @@ -4221,7 +4221,7 @@ def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: """Make compile reuse stop watching the models folder, so a model edit goes unseen.""" monkeypatch.setattr( - native_reuse, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} + reuse_constants, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 2cd5640d92..5e0749c357 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -14,8 +14,7 @@ from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( assert_required_cgroup_memory_limit, ) -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine.main.load_config import load_rules_config +from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, @@ -77,8 +76,10 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) config: RulesConfig = load_rules_config(project_dir=project_dir) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) - selected: tuple[Rule, ...] = select_rules( + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=True + ) + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=config, project_dir=project_dir ) builtin_codes: set[str] = {rule.code for rule in filterfalse(attrgetter("custom"), catalogue)} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 1d9438aea9..4f1330ff82 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -11,7 +11,7 @@ from typing import Any from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path +from sqlbuild._native import rules_bulk_cache_path from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import replace_project_text __all__ = ("write_custom_rules",) diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py deleted file mode 100644 index e0de81c887..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Integration tests for shared CLI command runtime helpers.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py deleted file mode 100644 index 8d5e37e5b1..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ /dev/null @@ -1,20 +0,0 @@ -from collections.abc import Callable -from dataclasses import dataclass - -from sqlbuild.cli.commands.models import ( - AuditInvocation, - CheckInvocation, - SeedInvocation, - TestInvocation, -) - - -@dataclass(frozen=True) -class CommandInvocationTestCase: - description: str - resolve: Callable[..., AuditInvocation | CheckInvocation | SeedInvocation | TestInvocation] - request_type: Callable[..., object] - json_output: bool - expected_invocation_type_name: str - expected_stream_name: str - expected_has_progress_reporters: bool diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py deleted file mode 100644 index f2845910f7..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Integration tests for resolving CLI command invocations against a real DuckDB project.""" - -from __future__ import annotations - -import sys -from functools import partial -from pathlib import Path - -import pytest - -from sqlbuild.cli.commands import models as command_models -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_command_invocation, - resolve_reported_command_invocation, -) -from tests.integration.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( - CommandInvocationTestCase, -) - -_PROJECT_TOML: str = ( - 'name = "orders_shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' -) - - -@pytest.mark.parametrize( - "test_case", - ( - CommandInvocationTestCase( - description="audit json output reports progress on stderr", - resolve=partial( - resolve_reported_command_invocation, - invocation_type=command_models.AuditInvocation, - ), - request_type=command_models.AuditCommandRequest, - json_output=True, - expected_invocation_type_name="AuditInvocation", - expected_stream_name="stderr", - expected_has_progress_reporters=True, - ), - CommandInvocationTestCase( - description="check text output reports progress on stdout", - resolve=partial( - resolve_reported_command_invocation, - invocation_type=command_models.CheckInvocation, - ), - request_type=command_models.CheckCommandRequest, - json_output=False, - expected_invocation_type_name="CheckInvocation", - expected_stream_name="stdout", - expected_has_progress_reporters=True, - ), - CommandInvocationTestCase( - description="test json output reports progress on stderr", - resolve=partial( - resolve_reported_command_invocation, - invocation_type=command_models.TestInvocation, - require_connection=True, - ), - request_type=command_models.TestCommandRequest, - json_output=True, - expected_invocation_type_name="TestInvocation", - expected_stream_name="stderr", - expected_has_progress_reporters=True, - ), - CommandInvocationTestCase( - description="seed text output omits shared progress reporters", - resolve=partial( - resolve_command_invocation, - invocation_type=command_models.SeedInvocation, - ), - request_type=command_models.SeedCommandRequest, - json_output=False, - expected_invocation_type_name="SeedInvocation", - expected_stream_name="stdout", - expected_has_progress_reporters=False, - ), - ), - ids=lambda case: case.description, -) -def test_given_duckdb_project_when_resolving_command_invocation_then_returns_command_context( - tmp_path: Path, - test_case: CommandInvocationTestCase, -) -> None: - _ = (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") - request: object = test_case.request_type( - project_dir=tmp_path, no_color=True, json_output=test_case.json_output - ) - - invocation: ( - command_models.AuditInvocation - | command_models.CheckInvocation - | command_models.SeedInvocation - | command_models.TestInvocation - ) = test_case.resolve(request=request) - - assert type(invocation).__name__ == test_case.expected_invocation_type_name - assert invocation.effective_project_dir == tmp_path - assert invocation.adapter_name == "duckdb" - assert invocation.discovered_inputs.project_config.name == "orders_shop" - assert invocation.use_color is False - assert invocation.progress_stream is getattr(sys, test_case.expected_stream_name) - assert hasattr(invocation, "connection_progress") is test_case.expected_has_progress_reporters - assert hasattr(invocation, "planning_progress") is test_case.expected_has_progress_reporters diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index 037911a191..e24bb68e25 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -13,8 +13,9 @@ import sqlbuild._native as native_module from sqlbuild._native import PreparedCompileArtifacts from sqlbuild._native import run_cli as main -from sqlbuild.cli.compile_reuse._helpers import native_reuse as native_reuse_module +from sqlbuild.cli.compile_reuse import constants as reuse_constants_module from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module +from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity from sqlbuild.cli.compile_reuse.constants import ( NATIVE_REUSE_DIRECTORY_NAME, @@ -109,7 +110,7 @@ def prepare_output_compiles(*, monkeypatch: pytest.MonkeyPatch, reuse_disabled: monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, reuse_disabled) monkeypatch.setattr(native_module, "PreparedCompileArtifacts", _StagingEveryProject) - monkeypatch.setattr(native_reuse_module, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(reuse_constants_module, "RACY_WINDOW_NS", 0) def main_compile_json(*, project_dir: Path) -> int: @@ -210,7 +211,7 @@ def change_runtime(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: def changed() -> dict[str, str]: return {**runtime_identity(), "python": "3.0.0 (simulated)"} - monkeypatch.setattr(native_reuse_module, "runtime_identity", changed) + monkeypatch.setattr(runtime_identity_module, "runtime_identity", changed) def change_native_build(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 4a59de8ac9..db6c1fc58e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -19,6 +19,7 @@ run_audit_command, run_build_command, run_load_command, + run_plan_command, run_scenario_test_command, run_test_command, select_load_entries, @@ -26,7 +27,6 @@ from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.main.project._plan import run_plan from sqlbuild.cli.commands.models import ( AuditCommandRequest, BuildCommandRequest, @@ -1065,8 +1065,8 @@ def test_given_plan_source_deferral_override_when_planning_then_succeeds( ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_plan( - PlanCommandRequest( + exit_code: int = run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -1280,8 +1280,8 @@ def test_given_invalid_managed_source_target_config_when_planning_then_errors( write_repo_files(tmp_path, test_case.project_files) with pytest.raises(PlannerInputError) as exc_info: - run_plan( - PlanCommandRequest( + run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -1722,8 +1722,8 @@ def test_given_plan_load_controls_when_running_plan_then_formats_source_load_sec ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_plan( - PlanCommandRequest( + exit_code: int = run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, select=("stg_orders",), @@ -1795,8 +1795,8 @@ def test_given_plan_json_output_when_source_auto_loads_then_includes_source_load ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_plan( - PlanCommandRequest( + exit_code: int = run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, json_output=True, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 08b31e3398..289af32bd6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -15,14 +15,13 @@ import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module -import sqlbuild.rule_engine._helpers.run.rules as rules_module +from sqlbuild._native import rules_bulk_cache_path from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintRunResult -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, CustomRuleCacheSequenceTestCase, @@ -713,7 +712,7 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule cold_exit: int = main(command) cold: dict[str, object] = json.loads(capsys.readouterr().out) - cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = f"{stored['build_identity']}-rebuilt" cache_path.write_text(json.dumps(stored), encoding="utf-8") @@ -931,7 +930,7 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_ b"SELECT 1\r\n" ) collections: list[frozenset[Path] | None] = [] - original_collect: Callable[..., dict[Path, str]] = rules_module.collect_lint_project_files + original_collect: Callable[..., dict[Path, str]] = native_module.collect_lint_project_files def counting_collect( *, project_dir: Path, selected_paths: frozenset[Path] | None @@ -939,7 +938,7 @@ def counting_collect( collections.append(selected_paths) return original_collect(project_dir=project_dir, selected_paths=selected_paths) - monkeypatch.setattr(rules_module, "collect_lint_project_files", counting_collect) + monkeypatch.setattr(native_module, "collect_lint_project_files", counting_collect) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) cold_collections: int = len(collections) @@ -994,7 +993,7 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( project_dir=tmp_path, capsys=capsys ) - cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = "previous-build" cache_path.write_text(json.dumps(stored), encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 8a61ffa85e..9fc9989506 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -42,7 +42,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.rule_engine._helpers.run import native_rows as native_rows_module from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -67,10 +66,7 @@ } -_FACTS_CONSUMERS: tuple[ModuleType, ...] = ( - native_rows_module, - _native, -) +_FACTS_CONSUMERS: tuple[ModuleType, ...] = (_native,) def forbid_hand_built_projects(*, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 79c0b1d7c9..11294e2004 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -57,7 +57,7 @@ class BuildExecutionPlanTestCase: class PlannerChangeDetectionWorkTestCase: description: str selection_diagnostics: bool - expected_model_calls: dict[str, int] + expected_stale_detection_calls: dict[str, int] expected_warning_count: int query_change_tracking: bool = True expected_identity_builds: int = 1 diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 432303fc0f..bc10eaabad 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -13,6 +13,7 @@ from typing import Any, cast from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, compiled_model_record, compute_query_hash, discover_project_inputs, @@ -51,9 +52,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index fe34b95e93..8212321405 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -15,7 +15,6 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.identity import model_metadata from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -64,20 +63,20 @@ PlannerChangeDetectionWorkTestCase( description="selection diagnostics disabled", selection_diagnostics=False, - expected_model_calls={"orders": 1}, + expected_stale_detection_calls={}, expected_warning_count=0, ), PlannerChangeDetectionWorkTestCase( description="selection diagnostics enabled", selection_diagnostics=True, - expected_model_calls={"stg_orders": 1, "orders": 1}, + expected_stale_detection_calls={"stg_orders": 1}, expected_warning_count=0, expected_identity_builds=2, ), PlannerChangeDetectionWorkTestCase( description="selection diagnostics preserve independent query tracking", selection_diagnostics=True, - expected_model_calls={"stg_orders": 1, "orders": 2}, + expected_stale_detection_calls={"stg_orders": 1, "orders": 1}, expected_warning_count=0, query_change_tracking=False, expected_identity_builds=2, @@ -119,8 +118,8 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d query_change_tracking=test_case.query_change_tracking, ), ) - model_detector: Mock = Mock(wraps=model_metadata.declared_columns_hash) - monkeypatch.setattr(model_metadata, "declared_columns_hash", model_detector) + stale_detector: Mock = Mock(wraps=identity_planning.detect_model_changes_in_scope) + monkeypatch.setattr(identity_planning, "detect_model_changes_in_scope", stale_detector) identity_builder: Mock = Mock(wraps=identity_planning.build_direct_model_version_identities) monkeypatch.setattr( identity_planning, @@ -136,10 +135,10 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d selection_diagnostics=test_case.selection_diagnostics, ) - model_calls: Counter[str] = Counter( - call.kwargs["model"].name for call in model_detector.call_args_list - ) - assert dict(model_calls) == test_case.expected_model_calls + stale_calls: Counter[str] = Counter() + for call in stale_detector.call_args_list: + stale_calls.update(key.name for key in call.kwargs["scope"].selected_keys) + assert dict(stale_calls) == test_case.expected_stale_detection_calls assert identity_builder.call_count == test_case.expected_identity_builds assert len(plan.warnings) == test_case.expected_warning_count diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py index 3388c46221..9be0333b4d 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py @@ -1,6 +1,15 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class SqlRulePublicationTestCase: + """Increasing path counts and the linear conversion budget.""" + + description: str + sizes: tuple[int, ...] + expected_counts: list[tuple[int, int]] + + @dataclass(frozen=True) class NativeRulesRequestTestCase: """A compile of the rules fixture: findings and cache use.""" diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py index 00059e1f38..6fb27cb42a 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py @@ -1,7 +1,7 @@ import json import shutil from pathlib import Path -from typing import Any, cast +from typing import Any, ClassVar, cast import pytest @@ -14,6 +14,16 @@ type FindingKey = tuple[str, str, int | None, int | None, str] +class CountingRulePath(Path): + """Count finding path conversions across the real native cache publication.""" + + conversions: ClassVar[int] = 0 + + def as_posix(self) -> str: + CountingRulePath.conversions += 1 + return super().as_posix() + + def write_rules_fixture(*, project_dir: Path) -> None: """Copy the neutral fixture and select every built-in rule.""" diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py new file mode 100644 index 0000000000..78fda746e9 --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -0,0 +1,104 @@ +"""Bound native SQL rule cache publication work while preserving persisted findings.""" + +import json +from concurrent.futures import Future +from pathlib import Path +from time import monotonic +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +import sqlbuild._native as native +from sqlbuild.rule_engine.models import Finding, SqlRulesEvaluation +from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( + SqlRulePublicationTestCase, +) +from tests.integration.src.sqlbuild.rule_engine.native_rows.helpers import CountingRulePath + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRulePublicationTestCase( + "doubling paths and findings", (4, 8, 16), [(4, 12), (8, 24), (16, 48)] + ) + ], + ids=lambda case: case.description, +) +def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and_order_preserved( + test_case: SqlRulePublicationTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exercise planning, finding conversion, encoding and disk publication through the binding.""" + monkeypatch.setattr(native, "compiled_relation_keys", lambda project: {}) + counts: list[tuple[int, int]] = [] + started: float = monotonic() + for size in test_case.sizes: + project_dir: Path = tmp_path / str(size) + paths: list[Path] = [Path(f"models/orders_{index}.sql") for index in range(size)] + models: list[SimpleNamespace] = [ + SimpleNamespace( + relative_path=path, + authored_sql="SELECT 1 AS order_id", + query_sql="SELECT 1 AS order_id", + config=SimpleNamespace(values={}), + dynamic_column_contract=None, + ) + for path in paths + ] + project: SimpleNamespace = SimpleNamespace( + settings=SimpleNamespace(sql_analysis=True), models=models, sql_expansions={} + ) + plan: SimpleNamespace = SimpleNamespace( + project_dir=project_dir, + project=project, + model_paths=frozenset(paths), + selected=(SimpleNamespace(code="SQBRSQL004", custom=False),), + effective_config=SimpleNamespace( + thresholds={}, allow_exceptions=True, cache=SimpleNamespace(enabled=True) + ), + ) + violations: list[SimpleNamespace] = [ + SimpleNamespace( + file_path=CountingRulePath(project_dir / paths[index % (size - 1)]), + code="SQBRSQL004", + line=index + 1, + column=1, + message=f"Finding {index}", + remediation="Use an explicit projection.", + fix=None, + ) + for index in range(size) + ] + future: Future[Any] = Future() + future.set_result(SimpleNamespace(result=SimpleNamespace(violations=violations))) + CountingRulePath.conversions = 0 + result: SqlRulesEvaluation = native.run_planned_sql_rules( + plan=cast(Any, plan), + graph=cast(Any, SimpleNamespace(project=project)), + dialect="duckdb", + discovered_inputs=cast(Any, None), + prepared_sql=cast(Any, SimpleNamespace(codes=("SQBRSQL004",), future=future)), + expansion_reuse=None, + ) + conversions: int = CountingRulePath.conversions + assert result.cache_misses == size + assert len(result.findings) == size + stored: dict[str, tuple[str, str]] = native.load_sql_rule_cache( + path=native.rules_bulk_cache_path(project_dir=project_dir, file_name="sql.json") + ) + assert set(stored) == {path.as_posix() for path in paths} + expected: list[list[Finding]] = [[] for _ in paths] + for position, finding in enumerate(result.findings): + expected[position % (size - 1)].append(finding) + for path, own in zip(paths, expected, strict=True): + assert stored[path.as_posix()][1] == native.encode_sql_rule_findings(findings=own) + assert json.loads(stored[paths[-1].as_posix()][1]) == [] + # One conversion each for grouping, encoding, and duplicate-literal hints. + counts.append((size, conversions)) + assert counts == test_case.expected_counts + assert monotonic() - started < 30 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py index 2176a57cb3..dd1ac19762 100644 --- a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py +++ b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py @@ -12,7 +12,7 @@ import pytest -from sqlbuild.cli.commands._helpers.entry.observability import cli_observability_scope +from sqlbuild._native import cli_observability_scope from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.execution_history import ( CanonicalLifecycleEvent, diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index b4605751e6..491d20a309 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -19,17 +19,17 @@ "test_case", [ CustomRuleTestCase( - description="raw color capability import faults", + description="raw terminal capability import faults", path="src/sqlbuild/example/main/build.py", source=( - "from sqlbuild.presentation._helpers.terminal_capabilities import supports_color\n" + "from sqlbuild.presentation._helpers.terminal_capabilities import terminal_columns\n" ), expected_fault_count=1, ), CustomRuleTestCase( - description="public color capability import passes", + description="public terminal capability import passes", path="src/sqlbuild/example/main/build.py", - source="from sqlbuild.presentation.main.supports_color import supports_color\n", + source="from sqlbuild.presentation.main.terminal_columns import terminal_columns\n", expected_fault_count=0, ), ], @@ -153,8 +153,8 @@ def test_given_dbt_code_when_checking_graph_projection_then_matches_contract( expected_fault_count=2, ), CustomRuleTestCase( - description="central selector parsing passes", - path="src/sqlbuild/compiler/planner/main/selection/selector_expansion.py", + description="selector parsing outside the planner and dbt passes", + path="src/sqlbuild/cli/commands/_helpers/selection.py", source="result = raw.startswith('+'), raw.lstrip('+')\n", expected_fault_count=0, ), diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 36e7280746..57fb768f0a 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -112,6 +112,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli", "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", + "sqlbuild.cli.observability", "sqlbuild.compile", "sqlbuild.executor.runtime", "sqlbuild.fingerprints", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index f310432175..b7c2fc1f1d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -91,7 +91,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", lambda **kwargs: _RefreshingCostAdapter(), ) monkeypatch.setattr( @@ -195,7 +195,7 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", lambda **kwargs: _StaticCostAdapter(candidate), ) monkeypatch.setattr( @@ -278,7 +278,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", lambda **kwargs: _StaticCostAdapter(candidate), ) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py index 42fbaaa224..3fe992d79a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import run_debug_command -from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope +from sqlbuild._native import command_warehouse_scope, run_debug_command from tests.unit.src.sqlbuild.cli.commands._helpers.debug._test_types import ( DebugWarehouseVarsTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py index 65c5d922dc..e5edd3e8ac 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py @@ -37,14 +37,6 @@ class QueryDiffParsingTestCase: expected_unkeyed: bool -@dataclass(frozen=True) -class EventExportWarningTestCase: - description: str - exporter_counts: tuple[tuple[str, int, int, int, int], ...] - command_succeeded: bool - expected_message: str | None - - @dataclass(frozen=True) class GlobalFlagPlacementTestCase: """One placement of the flags every command accepts.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py deleted file mode 100644 index 422af92b1a..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Local helpers for CLI entry tests.""" - -from __future__ import annotations - -from sqlbuild.runtime.event_exporting.models import ( - EventExporterAccounting, - EventExporterCounts, - EventExportSummary, -) - - -def build_export_summary(counts: tuple[tuple[str, int, int, int, int], ...]) -> EventExportSummary: - per_exporter: tuple[EventExporterAccounting, ...] = tuple( - EventExporterAccounting( - exporter_name=name, - counts=EventExporterCounts( - accepted=accepted, delivered=delivered, failed=failed, dropped=dropped - ), - ) - for name, accepted, delivered, failed, dropped in counts - ) - return EventExportSummary( - aggregate=EventExporterCounts( - accepted=sum(item.counts.accepted for item in per_exporter), - delivered=sum(item.counts.delivered for item in per_exporter), - failed=sum(item.counts.failed for item in per_exporter), - dropped=sum(item.counts.dropped for item in per_exporter), - ), - per_exporter=per_exporter, - queue_depth=0, - queue_capacity=1024, - flush_complete=all(item.counts.dropped == 0 for item in per_exporter), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py deleted file mode 100644 index 2a445718e5..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Incomplete lifecycle-event export warning formatting.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.entry.observability import ( - format_event_export_warning, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( - EventExportWarningTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.entry.helpers import build_export_summary - - -@pytest.mark.parametrize( - "test_case", - ( - EventExportWarningTestCase( - "all delivered", - (("orders_events", 248, 248, 0, 0),), - True, - None, - ), - EventExportWarningTestCase( - "successful command with drops", - (("orders_events", 248, 108, 0, 140),), - True, - "Warning: command completed successfully, but lifecycle event export was " - "incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). " - "Increase sinks.lifecycle.shutdown_timeout or check sink health.", - ), - EventExportWarningTestCase( - "failed command with failures", - (("orders_events", 10, 9, 1, 0),), - False, - "Warning: lifecycle event export was incomplete: 0 of 10 events dropped, " - "1 failed (sink 'orders_events'). " - "Increase sinks.lifecycle.shutdown_timeout or check sink health.", - ), - EventExportWarningTestCase( - "multiple sinks names only affected sinks", - ( - ("customer_events", 20, 20, 0, 0), - ("orders_events", 20, 15, 1, 4), - ), - True, - "Warning: command completed successfully, but lifecycle event export was " - "incomplete: 4 of 40 event deliveries dropped, 1 failed (sink 'orders_events'). " - "Increase sinks.lifecycle.shutdown_timeout or check sink health.", - ), - ), - ids=lambda case: case.description, -) -def test_given_final_export_summary_when_formatting_warning_then_reports_loss_only( - test_case: EventExportWarningTestCase, -) -> None: - message: str | None = format_event_export_warning( - summary=build_export_summary(test_case.exporter_counts), - command_succeeded=test_case.command_succeeded, - ) - - assert message == test_case.expected_message - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 2d8f5984f9..bb6f34880b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation +from sqlbuild._native import parse_cli_invocation from sqlbuild.cli.commands.types import CompileLineageMode, PlaygroundTemplate from sqlbuild.cli.entry.models import ( ParsedCliInvocation, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index efe524a245..ef9873a880 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -6,9 +6,12 @@ import pytest -from sqlbuild._native import describe_connection_warehouse, resolve_project_connection_config -from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation -from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope +from sqlbuild._native import ( + command_warehouse_scope, + describe_connection_warehouse, + parse_cli_invocation, + resolve_project_connection_config, +) from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index 34f8c5f0c9..a1ca166e58 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import resolve_effective_adapter_name +from sqlbuild._native import resolve_adapter, resolve_effective_adapter_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 4b6351bb72..394f533461 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -11,7 +11,7 @@ import pytest import sqlbuild._native as discovery_entry -from sqlbuild.cli.commands._helpers.runtime import adapters as runtime_adapters +from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 6ee8bb077c..77823c58a4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -6,7 +6,7 @@ import pytest from sqlbuild._native import run_dag_command -from sqlbuild.cli.commands._helpers.runtime import adapters as adapter_resolution +from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_resolution from tests.unit.src.sqlbuild.cli.commands.main.dag._test_types import ( DagCommandTestCase, PythonDagCommandTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py index c67d0fc59f..8c7f13ab9c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -7,7 +7,6 @@ import sqlbuild._native as dbt_init_module import sqlbuild.integrations.dbt.main.profile.profile_init as dbt_profile_init -import sqlbuild.presentation.main.supports_color as color_support from sqlbuild.cli.commands.models import DbtInitCommandRequest from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult @@ -16,6 +15,7 @@ DbtInitProjectDirDefaultTestCase, DbtInitValidationOrderTestCase, ) +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -173,7 +173,7 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: dry_run=False, expected_fragments=(), expected_color_fragments=( - "\033[2mInspecting dbt project and profile...\033[0m", + "\033[32m⠋\033[0m Inspecting dbt project and profile...", "\033[34m\033[1mSQLBuild project created\033[0m", "\033[1mSetup summary\033[0m:", "\033[34mProject", @@ -209,7 +209,7 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: toml='name = "analytics"\n', ) - monkeypatch.setattr(color_support, "supports_color", lambda: True) + use_color_terminal(monkeypatch) monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) monkeypatch.setattr( dbt_profile_init, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index f5a4a72bee..e96da9a59c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any, Protocol, cast -from sqlbuild._native import NativeProgressProjector, loader_destination_parts +from sqlbuild._native import loader_destination_parts from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, ) @@ -88,16 +88,3 @@ def build_handlers(**overrides: Any) -> CliEntrypointHandlers: run_rules=overrides.get("run_rules", noop_handler), run_scope=overrides.get("run_scope", noop_handler), ) - - -class CloseFailingProjector: - """Delegate to a native progress projector whose close always fails.""" - - def __init__(self, **options: Any) -> None: - self._projector: NativeProgressProjector = NativeProgressProjector(**options) - - def __getattr__(self, name: str) -> object: - return getattr(self._projector, name) - - def close(self) -> None: - raise OSError("controlled projector close failure") diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 13806edd53..ae59421a72 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -52,6 +52,7 @@ build_json_recording_handler, invalid_loader_destination_error, ) +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -2167,7 +2168,7 @@ def test_given_parser_error_and_color_support_when_running_main_then_it_colorize capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("sqlbuild.presentation.main.supports_color.supports_color", lambda: True) + use_color_terminal(monkeypatch) exit_code: int = main(test_case.argv) rendered_stderr: str = capsys.readouterr().err @@ -2193,7 +2194,7 @@ def test_given_parser_error_and_no_color_when_running_main_then_it_renders_plain capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("sqlbuild.presentation.main.supports_color.supports_color", lambda: True) + use_color_terminal(monkeypatch) exit_code: int = main(test_case.argv) rendered_stderr: str = capsys.readouterr().err @@ -2467,7 +2468,7 @@ def test_given_expected_cli_error_and_color_support_when_running_main_then_it_co capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("sqlbuild.presentation.main.supports_color.supports_color", lambda: True) + use_color_terminal(monkeypatch) def run_query( project_dir: Path | None, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 30e64d5637..10a3861cee 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -5,16 +5,15 @@ from contextvars import Token from io import StringIO from pathlib import Path -from unittest.mock import patch import pytest from _pytest.capture import CaptureResult from sqlbuild._native import ( NativeProgressProjector, + cli_observability_scope, current_native_progress_projector, ) -from sqlbuild.cli.commands._helpers.entry.observability import cli_observability_scope from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.observability import OperationLifecycle @@ -23,24 +22,19 @@ NoExporterFastPathTestCase, ObservabilityCleanupCase, ) -from tests.unit.src.sqlbuild.cli.commands.main.entry.helpers import CloseFailingProjector - -_PROJECTOR_FACTORY: str = ( - "sqlbuild.cli.commands._helpers.entry.observability.NativeProgressProjector" -) @pytest.mark.parametrize( "test_case", ( ObservabilityCleanupCase( - description="projector close failure preserves original exception and cleanup", + description="command failure propagates and restores the outer projector", expected_original_error="original command failure", ), ), ids=lambda case: case.description, ) -def test_given_projector_close_failure_when_command_fails_then_original_error_and_cleanup_survive( +def test_given_command_failure_when_scope_exits_then_original_error_and_projector_survive( tmp_path: Path, test_case: ObservabilityCleanupCase, ) -> None: @@ -48,10 +42,7 @@ def test_given_projector_close_failure_when_command_fails_then_original_error_an outer: NativeProgressProjector = NativeProgressProjector(stream=StringIO(), use_color=False) outer_token: Token[NativeProgressProjector | None] = outer.install() - with ( - patch(_PROJECTOR_FACTORY, CloseFailingProjector), - pytest.raises(ValueError, match=test_case.expected_original_error), - ): + with pytest.raises(ValueError, match=test_case.expected_original_error): with cli_observability_scope(args=args, project_dir=tmp_path): assert current_native_progress_projector() is not outer raise ValueError(test_case.expected_original_error) @@ -217,27 +208,3 @@ def test_given_integration_result_file_when_operation_runs_then_human_progress_u captured: CaptureResult[str] = capsys.readouterr() assert test_case.expected_stdout_fragment in captured.out assert captured.err == test_case.expected_stderr_fragment - - -@pytest.mark.parametrize( - "test_case", - ( - ObservabilityCleanupCase( - description="projector close failure preserves successful scope result", - expected_original_error="", - ), - ), - ids=lambda case: case.description, -) -def test_given_projector_close_failure_when_scope_succeeds_then_cleanup_does_not_raise( - tmp_path: Path, - test_case: ObservabilityCleanupCase, -) -> None: - args: CliNamespace = CliNamespace() - - with patch(_PROJECTOR_FACTORY, CloseFailingProjector): - with cli_observability_scope(args=args, project_dir=tmp_path): - result: str = "command result" - - assert result == "command result" - assert test_case.expected_original_error == "" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 6da7fb9048..0d1820b0af 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -10,15 +10,15 @@ import pytest from sqlbuild import _native as native +from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module from sqlbuild.cli.commands._helpers.planning import external_refs as external_refs_module -from sqlbuild.cli.commands._helpers.runtime import adapters as adapters_module from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module -from sqlbuild.presentation.main import supports_color as supports_color_module from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal def fail_build_discovery(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> Mock: @@ -57,7 +57,7 @@ def configure_native_scenario_command( local_config=LocalConfig(), ) monkeypatch.setattr(sys, "stdout", stream) - monkeypatch.setattr(supports_color_module, "supports_color", lambda: False) + use_plain_terminal(monkeypatch) monkeypatch.setattr(native, "discover_project_inputs", lambda **_: discovered_inputs) monkeypatch.setattr(native, "resolve_project_connection_config", lambda **_: {}) monkeypatch.setattr(native, "run_compile_only_pipeline", compile_project) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py index f2b1e72ffe..fb3e5b2524 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py @@ -6,9 +6,9 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.presentation.main.supports_color as color_module from sqlbuild._native import run_init_command from tests.unit.src.sqlbuild.cli.commands.main.init._test_types import InitScaffoldTestCase +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal, use_plain_terminal @pytest.mark.parametrize( @@ -87,7 +87,7 @@ def test_given_empty_directory_when_running_init_then_scaffolds_project_director project_dir: Path = tmp_path / test_case.project_dir_name project_dir.mkdir() monkeypatch.setattr(native_module, "update_sqlbuild_skills", lambda *, project_dir: None) - monkeypatch.setattr(color_module, "supports_color", lambda: False) + use_plain_terminal(monkeypatch) result: int = run_init_command(project_dir) @@ -131,7 +131,7 @@ def test_given_color_terminal_when_running_init_then_it_styles_heading( project_dir: Path = tmp_path / test_case.project_dir_name project_dir.mkdir() monkeypatch.setattr(native_module, "update_sqlbuild_skills", lambda *, project_dir: None) - monkeypatch.setattr(color_module, "supports_color", lambda: True) + use_color_terminal(monkeypatch) result: int = run_init_command(project_dir) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 026803509f..d4843a3838 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -9,12 +9,12 @@ from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.cli.output.models import SkillMaintenanceResult -from sqlbuild.presentation.main import supports_color as playground_completion_output from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, RunPlaygroundTestCase, WrapperPlaygroundSkillsTestCase, ) +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -399,7 +399,7 @@ def test_given_color_terminal_when_running_playground_command_then_it_styles_key capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(playground_completion_output, "supports_color", lambda: True) + use_color_terminal(monkeypatch) exit_code: int = run_playground_command( PlaygroundCommandRequest( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index f19b8b079b..913badbca7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -7,13 +7,14 @@ import pytest +from sqlbuild import _native as native from sqlbuild._native import run_scope_command -from sqlbuild.cli.commands._helpers.runtime import adapter_context +from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_registry from sqlbuild.cli.commands.models import ScopeCommandRequest from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex -from sqlbuild.presentation.main import supports_color as color_support from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -25,7 +26,8 @@ def test_given_offline_index_loader_when_running_then_connection_runtime_is_not_ def fail(**_kwargs: object) -> None: pytest.fail("scope must not resolve adapters or warehouse credentials") - monkeypatch.setattr(adapter_context, "resolve_adapter_connection_context", fail) + monkeypatch.setattr(adapter_registry, "resolve_adapter", fail) + monkeypatch.setattr(native, "resolve_project_connection_config", fail) exit_code: int = run_scope_command( request=ScopeCommandRequest(target="model:orders", json_output=True), load_scope_index=lambda **_kwargs: report_scope_lookup().index, @@ -99,7 +101,7 @@ def test_given_no_color_flag_when_rendering_text_to_color_terminal_then_output_i capsys: pytest.CaptureFixture[str], test_case: ScopeCommandCase, ) -> None: - monkeypatch.setattr(color_support, "supports_color", lambda: True) + use_color_terminal(monkeypatch) index: ScopeIndex = report_scope_lookup().index outputs: list[str] = [] for no_color in (False, True): diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 0be484741e..77e77250e7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -18,7 +18,7 @@ ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.progress.models import NestedProgressChildRow +from sqlbuild.cli.commands.models import NestedProgressChildRow from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode diff --git a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py index 2986b6d893..806d07e06d 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py @@ -17,9 +17,8 @@ class VerboseInspectionFormatTestCase: @dataclass(frozen=True) class VerboseInspectionOutputTestCase: - """Inspection reads recorded inside and after a verbose output context.""" + """Inspection reads recorded inside and after a verbose writer sink.""" description: str - enabled: bool records: tuple[tuple[str, float, int], ...] expected_stderr_lines: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py index 775b1d4844..58f8dd94c1 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py @@ -6,9 +6,9 @@ from _pytest.capture import CaptureResult from sqlbuild._native import VerboseInspectionWriter +from sqlbuild.adapter.relations.main.inspection_query_sink import inspection_query_sink from sqlbuild.adapter.relations.main.record_inspection_query import record_inspection_query from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output from tests.unit.src.sqlbuild.cli.progress.main.verbose_inspection_output._test_types import ( VerboseInspectionFormatTestCase, VerboseInspectionOutputTestCase, @@ -74,8 +74,7 @@ def test_given_inspection_record_when_formatting_then_renders_one_bounded_line( "test_case", [ VerboseInspectionOutputTestCase( - description="enabled output writes each read and a total to stderr", - enabled=True, + description="sink writes each read and the total to stderr", records=(("SELECT 1", 0.5, 3), ("SELECT 2", 0.25, 0)), expected_stderr_lines=( " inspect 0.50s 3 rows SELECT 1", @@ -84,18 +83,18 @@ def test_given_inspection_record_when_formatting_then_renders_one_bounded_line( ), ), VerboseInspectionOutputTestCase( - description="disabled output writes nothing", - enabled=False, - records=(("SELECT 1", 0.5, 3),), + description="no reads write no total", + records=(), expected_stderr_lines=(), ), ], ids=lambda case: case.description, ) -def test_given_verbose_setting_when_recording_reads_then_writes_only_inside_context_to_stderr( +def test_given_writer_sink_when_recording_reads_then_writes_only_inside_sink_to_stderr( test_case: VerboseInspectionOutputTestCase, capsys: pytest.CaptureFixture[str] ) -> None: - with verbose_inspection_output(enabled=test_case.enabled): + writer: VerboseInspectionWriter = VerboseInspectionWriter() + with inspection_query_sink(writer.write): for sql, elapsed, rows in test_case.records: record_inspection_query( record=InspectionQueryRecord(sql=sql, elapsed_seconds=elapsed, row_count=rows) @@ -103,6 +102,7 @@ def test_given_verbose_setting_when_recording_reads_then_writes_only_inside_cont record_inspection_query( record=InspectionQueryRecord(sql="SELECT 3", elapsed_seconds=0.1, row_count=0) ) + writer.write_total() captured: CaptureResult[str] = capsys.readouterr() assert captured.out == "" diff --git a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py index c65d50fd21..e4d9c27550 100644 --- a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target +from sqlbuild._native import write_runtime_target from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 2a730a73fc..f67198973a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_execution_upstream_deps, build_manifest, build_native_declaration_contexts, build_scope_lookup, @@ -62,7 +63,6 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index fd96381c3d..761be7aae8 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_model_version_identity_metadata_json, compute_query_hash, discover_project_inputs, ) @@ -22,9 +23,6 @@ ) from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CompileDeclarationsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 0cba374c8f..f167adfc1f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_model_version_identity_metadata_json, discover_project_inputs, evaluate_model_contracts, ) @@ -24,9 +25,6 @@ CompileInputsView, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index c20ea521fa..deee9d90ef 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from pathlib import Path from unittest.mock import Mock import pytest @@ -15,8 +16,8 @@ from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings -from sqlbuild.rule_engine.main import load_config, run_rules -from sqlbuild.rule_engine.models import RulesConfig, RulesRunResult +from sqlbuild.rule_engine._helpers.run import rule_threads +from sqlbuild.rule_engine.models import RulesRunResult from sqlbuild.runtime.contracts.main import open_connection from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase @@ -50,6 +51,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_disjoint( test_case: PipelinePhaseTimingTestCase, monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ) -> None: monotonic: Mock = Mock(side_effect=test_case.clock_values) project: CompiledProjectView = CompiledProject( @@ -66,11 +68,11 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) - monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) + (tmp_path / "sqlbuild_project.toml").write_text('name = "timing"\n', encoding="utf-8") monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) monkeypatch.setattr( - run_rules, - "run_rules", + rule_threads, + "evaluate_rules", Mock( return_value=RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) ), @@ -84,7 +86,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di with tracker.scope(): result: CompilePipelineResult = _native.run_compile_pipeline( - discovered_inputs=Mock(), + discovered_inputs=Mock(project_dir=tmp_path), adapter=adapter, options=CompilePipelineOptions(connection_config={}), hooks=progress_hooks[test_case.report_progress], diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 9348e7321a..345612fbe9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -16,11 +16,10 @@ from sqlbuild.compiler.planner.models import ( CursorBounds, EmptyCursorInputDecision, - GraphIdentityNode, - GraphNodeKey, MissingUpstream, ParsedSelector, PathSelector, + PlanWarning, ScenarioArtifactIdentity, ScenarioGraphPlan, ScenarioRelationMap, @@ -66,7 +65,7 @@ class CompleteFixturePlanningTestCase: @dataclass(frozen=True) class CompleteModelChangesWorkTestCase: description: str - expected_detector_calls: int + expected_warnings: tuple[PlanWarning, ...] @dataclass(frozen=True) @@ -193,18 +192,6 @@ class SelectionStalenessWarningTestCase: expected_warning_fragments: tuple[str, ...] -@dataclass(frozen=True) -class StaleWarningMessageTestCase: - description: str - model_label: str - model_name: str - trigger_label: str - trigger_names: tuple[str, ...] - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] - expected_bullet_count: int - - @dataclass(frozen=True) class SelectionStalenessGraphWarningTestCase: description: str @@ -218,32 +205,6 @@ class SelectionStalenessGraphWarningTestCase: unexpected_warning_fragments: tuple[str, ...] = () -@dataclass(frozen=True) -class GraphIdentityExpectedHashesTestCase: - description: str - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - expected_hashes: dict[GraphNodeKey, str | None] - - -@dataclass(frozen=True) -class GraphIdentityWriteHashesTestCase: - description: str - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - selected_keys: frozenset[GraphNodeKey] - base_identity_hashes: dict[GraphNodeKey, str] - expected_hashes: dict[GraphNodeKey, str] - - -@dataclass(frozen=True) -class GraphIdentityWritePerfTestCase: - description: str - layer_count: int - expected_max_seconds: float - layer_width: int = 2 - - @dataclass(frozen=True) class ModelClosureTestCase: description: str @@ -911,3 +872,11 @@ class EmptyCursorInputsSnapshotTestCase: expected_empty_input_names: tuple[str, ...] expected_unreadable_input_names: tuple[str, ...] expected_decision: EmptyCursorInputDecision + + +@dataclass(frozen=True) +class LoaderDependencyNamesTestCase: + """Expected first-seen loader names after resolving Python function dependencies.""" + + description: str + expected_names: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index dcfe977822..cece72c0bf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import cast -from sqlbuild._native import compute_query_hash +from sqlbuild._native import build_version_identity_metadata_json, compute_query_hash from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.compiler.compile.models import ( CompiledFunction, @@ -26,9 +26,6 @@ FunctionLanguage, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import PlannerScope, WarehouseFingerprints, WarehouseSnapshot from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index c3c95b77f6..c4099e9a61 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( build_model_warnings, + build_version_identity_metadata_json, compute_query_hash, detect_changes, detect_model_changes, @@ -18,9 +19,6 @@ from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py index fa2680d0f7..c4efc9f391 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py @@ -2,15 +2,15 @@ import pytest -from sqlbuild._native import project_lineage_views +from sqlbuild._native import ( + build_downstream_deps, + build_execution_upstream_deps, + project_lineage_views, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_test_project, model_key, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 4f60c4a0a5..18f2365be1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib from collections.abc import Callable, Iterator from dataclasses import replace from datetime import UTC, datetime @@ -10,11 +9,14 @@ from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Any, ClassVar +from unittest.mock import Mock import sqlbuild._native as _native from sqlbuild._native import ( + build_downstream_deps, compiled_model_record, compute_query_hash, + detect_changes, project_with, scan_sql_reference_calls, ) @@ -64,13 +66,11 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.scenario.artifacts import build_scenario_relation_map from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, - GraphIdentityNode, - GraphNodeKey, + PlannerChangeResults, PlanOutput, ScenarioArtifactIdentity, ScenarioRelationMap, @@ -78,7 +78,7 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - GraphResourceKind, + PlanReason, ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -159,77 +159,6 @@ def close(self, connection: object) -> None: del connection -def compose_readable_identity( - *, local_hash: str, upstream_hashes: tuple[tuple[GraphNodeKey, str], ...] -) -> str: - rendered: str = ",".join( - f"{key.node_type}:{key.node_name}={upstream_hash}" for key, upstream_hash in upstream_hashes - ) - return (f"{local_hash}|{rendered}", local_hash)[not upstream_hashes] - - -def compose_hashed_identity( - *, local_hash: str, upstream_hashes: tuple[tuple[GraphNodeKey, str], ...] -) -> str: - payload: str = ( - local_hash + "|" + ",".join(upstream_hash for _, upstream_hash in upstream_hashes) - ) - return (hashlib.sha256(payload.encode()).hexdigest(), local_hash)[not upstream_hashes] - - -def build_diamond_ladder_identity_nodes( - *, layer_count: int -) -> tuple[dict[GraphNodeKey, GraphIdentityNode], tuple[GraphNodeKey, ...]]: - """Build a dense ladder where each layer depends on both nodes of the prior layer.""" - - def node_key(name: str) -> GraphNodeKey: - return GraphNodeKey(node_type="model", node_name=name) - - nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - order: list[GraphNodeKey] = [] - previous: tuple[GraphNodeKey, ...] = () - layer_index: int - for layer_index in range(layer_count): - current: list[GraphNodeKey] = [] - suffix: str - for suffix in ("a", "b"): - key: GraphNodeKey = node_key(f"L{layer_index}_{suffix}") - nodes[key] = GraphIdentityNode( - key=key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=previous, - local_hash=f"local_{key.node_name}", - ) - order.append(key) - current.append(key) - previous = tuple(current) - return nodes, tuple(order) - - -def build_wide_layered_identity_nodes( - *, layer_count: int, layer_width: int -) -> tuple[dict[GraphNodeKey, GraphIdentityNode], tuple[GraphNodeKey, ...]]: - """Build a wide project graph where each node reads two nodes of the prior layer.""" - - nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - previous: tuple[GraphNodeKey, ...] = () - layer: int - for layer in range(layer_count): - current: list[GraphNodeKey] = [] - column: int - for column in range(layer_width): - key: GraphNodeKey = GraphNodeKey(node_type="model", node_name=f"L{layer}_{column}") - nodes[key] = GraphIdentityNode( - key=key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=previous[column : column + 2], - local_hash=f"local_{key.node_name}", - ) - current.append(key) - previous = tuple(current) - return nodes, tuple(nodes) - - def _append_relation_info(listed: list[RelationInfo], relation: RelationInfo) -> None: listed.append(relation) @@ -1509,3 +1438,32 @@ def _assertion_ref_targets(assertion_sql: tuple[str, ...]) -> tuple[str, ...]: lambda reference: reference.ref_kind == SqlReferenceKind.REF, references ) return tuple(dict.fromkeys(reference.ref_name for reference in refs)) + + +def build_unreadable_model() -> CompiledModelView: + """Build a model view whose every attribute read fails, so change detection cannot run.""" + + model: CompiledModelView = Mock(spec=()) + return model + + +def detect_function_reason( + *, function: CompiledFunctionView, fingerprints: dict[str, Fingerprint] +) -> PlanReason: + """Detect one selected function's change reason through planner change detection.""" + + project: Mock = Mock( + functions=(function,), + settings=SimpleNamespace(query_change_tracking=True), + sql_analysis_dialect=None, + hook_functions=(), + ) + scope: Mock = Mock( + selected_keys=frozenset({function.key}), execution_order=(), models_by_name={} + ) + snapshot: Mock = Mock(fingerprints=SimpleNamespace(functions=fingerprints), column_dialect=None) + changes: PlannerChangeResults = detect_changes( + project=project, scope=scope, snapshot=snapshot, full_refresh=False + ) + reason: PlanReason = changes.functions[function.name].reason + return reason diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py index 5264136d02..8eba85f9f6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py @@ -2,13 +2,13 @@ from __future__ import annotations +from dataclasses import replace + import pytest -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - function_definition_hash, - model_definition_hash, -) +from sqlbuild._native import build_function_local_hashes, model_definition_hash from sqlbuild.compiler.planner.exceptions import PlannerInputError +from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( DefinitionHashFailureTestCase, ) @@ -45,10 +45,10 @@ def test_given_untokenizable_sql_function_when_hashing_then_planning_error_names test_case: DefinitionHashFailureTestCase, ) -> None: with pytest.raises(PlannerInputError, match=test_case.expected_message) as raised: - _ = function_definition_hash( - function_name="is_open", - fingerprint_sql=_UNTOKENIZABLE_SQL, - language="sql", + _ = build_function_local_hashes( + functions=( + replace(build_compiled_function(body_sql=_UNTOKENIZABLE_SQL), name="is_open"), + ), dialect="duckdb", ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index 208b1a2173..0e9b1029fe 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.identity.honest import merge_recomputed_model_changes +from sqlbuild._native import merge_recomputed_model_changes from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py new file mode 100644 index 0000000000..1858c9a743 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py @@ -0,0 +1,33 @@ +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + + +def build_selector_discovered_inputs( + *, tests: frozenset[str], resources: frozenset[str] +) -> DiscoveredProjectInputs: + """Discovered inputs naming these unit tests and, as model files, these resources.""" + + inputs: SimpleNamespace = SimpleNamespace( + test_files=( + SimpleNamespace( + relative_path=Path("tests/unit_tests.sql"), + blocks=tuple(SimpleNamespace(name=name) for name in sorted(tests)), + ), + ), + model_files=tuple( + SimpleNamespace(file_path=Path(f"models/{name}.sql")) for name in sorted(resources) + ), + schema_files=(), + source_files=(), + seed_files=(), + sql_function_files=(), + python_function_files=(), + loader_functions=(), + task_functions=(), + asset_functions=(), + check_functions=(), + ) + return cast(DiscoveredProjectInputs, inputs) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index 3c4352512d..0dfa091810 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -4,21 +4,19 @@ import pytest -from sqlbuild.compiler.planner._helpers.selection.unit_test_selectors import ( - split_unit_test_selectors, -) +from sqlbuild._native import split_project_unit_test_selectors +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - SqlTestSelection, - UnitTestSelectorNames, - UnitTestSelectorSplit, -) +from sqlbuild.compiler.planner.models import SqlTestSelection, UnitTestSelectorSplit from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner._helpers.selection.helpers import ( + build_selector_discovered_inputs, +) -_NAMES: UnitTestSelectorNames = UnitTestSelectorNames( +_DISCOVERED: DiscoveredProjectInputs = build_selector_discovered_inputs( tests=frozenset({"order_status", "order_totals_check", "customer_status"}), resources=frozenset({"orders", "order_totals", "customers", "raw_orders"}), ) @@ -93,10 +91,10 @@ def test_given_selectors_when_splitting_unit_tests_then_tests_and_resources_separate( test_case: UnitTestSelectorSplitTestCase, ) -> None: - split: UnitTestSelectorSplit = split_unit_test_selectors( + split: UnitTestSelectorSplit = split_project_unit_test_selectors( select=test_case.select, exclude=test_case.exclude, - names=_NAMES, + discovered_inputs=_DISCOVERED, accepts_unit_tests=True, ) @@ -183,10 +181,10 @@ def test_given_invalid_unit_test_selectors_when_splitting_then_it_raises_clear_e test_case: UnitTestSelectorErrorTestCase, ) -> None: with pytest.raises(PlannerInputError) as raised: - split_unit_test_selectors( + split_project_unit_test_selectors( select=test_case.select, exclude=(), - names=_NAMES, + discovered_inputs=_DISCOVERED, accepts_unit_tests=test_case.accepts_unit_tests, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py index 88caa414e8..fc42a40697 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py @@ -2,11 +2,8 @@ import pytest +from sqlbuild._native import check_buildability, missing_upstream_message from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.planner._helpers.graph.buildability import ( - check_buildability, - missing_upstream_message, -) from sqlbuild.compiler.planner.models import MissingUpstream, WarehouseSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CheckBuildabilityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index 0a1d58915f..e46a99578a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -5,10 +5,6 @@ import pytest from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, - detect_function_change, -) from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( DetectFunctionChangeTestCase, @@ -16,6 +12,7 @@ from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_compiled_function, build_fingerprint, + detect_function_reason, ) @@ -69,13 +66,8 @@ def test_given_function_fingerprint_when_detecting_change_then_returns_reason( body_sql=test_case.body_sql, target_schema=test_case.target_schema, ) - reason: PlanReason = detect_function_change( - function=function, - fingerprint_sql=build_compiled_function_fingerprint_sql(function), - fingerprint=test_case.existing_function_fingerprints.get(function.name), - query_change_tracking=True, - full_refresh=False, - dialect=None, + reason: PlanReason = detect_function_reason( + function=function, fingerprints=test_case.existing_function_fingerprints ) assert reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py index f68ec6b046..85a57d11bb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py @@ -2,15 +2,14 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.core import ( +from sqlbuild._native import ( build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, - expand_downstream, - expand_upstream, topologically_order_keys, + transitive_closure, ) +from sqlbuild.compiler.compile.models import CompiledObjectKey from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildDownstreamDepsTestCase, BuildUpstreamDepsTestCase, @@ -296,8 +295,8 @@ def test_given_audit_scope_deps_when_building_edge_origins_then_names_the_audit( def test_given_key_when_expanding_upstream_then_returns_expected_keys( test_case: ExpandUpstreamTestCase, ) -> None: - result: frozenset[CompiledObjectKey] = expand_upstream( - key=test_case.key, upstream=test_case.upstream + result: frozenset[CompiledObjectKey] = transitive_closure( + start=test_case.key, edges=test_case.upstream ) assert result == test_case.expected_keys @@ -331,8 +330,8 @@ def test_given_key_when_expanding_upstream_then_returns_expected_keys( def test_given_key_when_expanding_downstream_then_returns_expected_keys( test_case: ExpandDownstreamTestCase, ) -> None: - result: frozenset[CompiledObjectKey] = expand_downstream( - key=test_case.key, downstream=test_case.downstream + result: frozenset[CompiledObjectKey] = transitive_closure( + start=test_case.key, edges=test_case.downstream ) assert result == test_case.expected_keys diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py deleted file mode 100644 index e1d04af0ef..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py +++ /dev/null @@ -1,259 +0,0 @@ -from __future__ import annotations - -import time - -import pytest - -from sqlbuild.compiler.planner.main.identity._graph_identity import ( - build_expected_graph_identity_hashes, -) -from sqlbuild.compiler.planner.main.identity._graph_write_identity import ( - build_graph_write_identity_hashes, -) -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphResourceKind -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - GraphIdentityExpectedHashesTestCase, - GraphIdentityWriteHashesTestCase, - GraphIdentityWritePerfTestCase, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( - build_diamond_ladder_identity_nodes, - build_wide_layered_identity_nodes, - compose_hashed_identity, - compose_readable_identity, -) - -MODEL_A: GraphNodeKey = GraphNodeKey(node_type="model", node_name="a") -MODEL_B: GraphNodeKey = GraphNodeKey(node_type="model", node_name="b") -MODEL_C: GraphNodeKey = GraphNodeKey(node_type="model", node_name="c") -MODEL_D: GraphNodeKey = GraphNodeKey(node_type="model", node_name="d") - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityExpectedHashesTestCase( - description=( - "uses dependency order and named upstream keys while preserving missing nodes" - ), - nodes={ - MODEL_A: GraphIdentityNode( - key=MODEL_A, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_B, MODEL_C), - local_hash="local_a", - ), - MODEL_B: GraphIdentityNode( - key=MODEL_B, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash="local_b", - ), - MODEL_C: GraphIdentityNode( - key=MODEL_C, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash=None, - ), - }, - execution_order=(MODEL_B, MODEL_C, MODEL_A), - expected_hashes={ - MODEL_A: "local_a|model:b=local_b", - MODEL_B: "local_b", - MODEL_C: None, - }, - ) - ], - ids=lambda case: case.description, -) -def test_given_identity_graph_when_building_expected_hashes_then_resolves_from_upstreams( - test_case: GraphIdentityExpectedHashesTestCase, -) -> None: - result: dict[GraphNodeKey, str | None] = build_expected_graph_identity_hashes( - nodes=test_case.nodes, - execution_order=test_case.execution_order, - compose_identity=compose_readable_identity, - ) - - assert result == test_case.expected_hashes - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWriteHashesTestCase( - description="recomputes selected child from caller supplied parent write hash", - nodes={ - MODEL_A: GraphIdentityNode( - key=MODEL_A, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_B,), - local_hash="local_a", - ), - MODEL_B: GraphIdentityNode( - key=MODEL_B, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash="local_b", - ), - }, - execution_order=(MODEL_B, MODEL_A), - selected_keys=frozenset({MODEL_A}), - base_identity_hashes={MODEL_B: "honest_b"}, - expected_hashes={ - MODEL_A: "local_a|model:b=honest_b", - MODEL_B: "honest_b", - }, - ), - GraphIdentityWriteHashesTestCase( - description="composes a fully selected diamond from the shared root once", - nodes={ - MODEL_A: GraphIdentityNode( - key=MODEL_A, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash="local_a", - ), - MODEL_B: GraphIdentityNode( - key=MODEL_B, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_A,), - local_hash="local_b", - ), - MODEL_C: GraphIdentityNode( - key=MODEL_C, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_A,), - local_hash="local_c", - ), - MODEL_D: GraphIdentityNode( - key=MODEL_D, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_B, MODEL_C), - local_hash="local_d", - ), - }, - execution_order=(MODEL_A, MODEL_B, MODEL_C, MODEL_D), - selected_keys=frozenset({MODEL_A, MODEL_B, MODEL_C, MODEL_D}), - base_identity_hashes={}, - expected_hashes={ - MODEL_A: "local_a", - MODEL_B: "local_b|model:a=local_a", - MODEL_C: "local_c|model:a=local_a", - MODEL_D: "local_d|model:b=local_b|model:a=local_a,model:c=local_c|model:a=local_a", - }, - ), - ], - ids=lambda case: case.description, -) -def test_given_available_hashes_when_building_write_hashes_then_uses_caller_supplied_upstreams( - test_case: GraphIdentityWriteHashesTestCase, -) -> None: - result: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=test_case.nodes, - execution_order=test_case.execution_order, - selected_keys=test_case.selected_keys, - base_identity_hashes=test_case.base_identity_hashes, - compose_identity=compose_readable_identity, - ) - - assert result == test_case.expected_hashes - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWritePerfTestCase( - description="fully selected densely connected graph resolves without path blowup", - layer_count=400, - expected_max_seconds=5.0, - ) - ], - ids=lambda case: case.description, -) -def test_given_fully_selected_dense_graph_when_building_write_hashes_then_stays_linear( - test_case: GraphIdentityWritePerfTestCase, -) -> None: - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - nodes, execution_order = build_diamond_ladder_identity_nodes(layer_count=test_case.layer_count) - - start: float = time.monotonic() - result: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=nodes, - execution_order=execution_order, - selected_keys=frozenset(nodes.keys()), - base_identity_hashes={}, - compose_identity=compose_hashed_identity, - ) - elapsed: float = time.monotonic() - start - - assert len(result) == len(nodes) - assert elapsed < test_case.expected_max_seconds - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWritePerfTestCase( - description="twenty thousand node project graph resolves in linear time", - layer_count=40, - layer_width=500, - expected_max_seconds=1.5, - ) - ], - ids=lambda case: case.description, -) -def test_given_wide_project_graph_when_building_expected_hashes_then_stays_linear( - test_case: GraphIdentityWritePerfTestCase, -) -> None: - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - nodes, execution_order = build_wide_layered_identity_nodes( - layer_count=test_case.layer_count, layer_width=test_case.layer_width - ) - - start: float = time.monotonic() - result: dict[GraphNodeKey, str | None] = build_expected_graph_identity_hashes( - nodes=nodes, execution_order=execution_order, compose_identity=compose_hashed_identity - ) - elapsed: float = time.monotonic() - start - - assert len(result) == len(nodes) - assert elapsed < test_case.expected_max_seconds - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWritePerfTestCase( - description="twenty thousand node project graph resolves in linear time", - layer_count=40, - layer_width=500, - expected_max_seconds=1.5, - ) - ], - ids=lambda case: case.description, -) -def test_given_wide_selected_project_graph_when_building_write_hashes_then_stays_linear( - test_case: GraphIdentityWritePerfTestCase, -) -> None: - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - nodes, execution_order = build_wide_layered_identity_nodes( - layer_count=test_case.layer_count, layer_width=test_case.layer_width - ) - - start: float = time.monotonic() - result: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=nodes, - execution_order=execution_order, - selected_keys=frozenset(nodes.keys()), - base_identity_hashes={}, - compose_identity=compose_hashed_identity, - ) - elapsed: float = time.monotonic() - start - - assert len(result) == len(nodes) - assert elapsed < test_case.expected_max_seconds diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index cb71d34ebc..923a00170c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -2,11 +2,9 @@ import pytest +from sqlbuild._native import build_downstream_model_name_closure from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.model_closure import ( - build_downstream_model_name_closure, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase SOURCE_ORDERS: CompiledObjectKey = CompiledObjectKey( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py index 0be0633d36..3f34e9ed29 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity +from sqlbuild._native import build_seed_identity from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( SeedIdentityCsvConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index a6dbdd897b..b2b494bb98 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -2,11 +2,11 @@ from dataclasses import replace from datetime import UTC, datetime -from unittest.mock import Mock import pytest from sqlbuild._native import ( + build_stale_out_of_selection_warnings, compiled_model_record, ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject @@ -18,13 +18,6 @@ from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes -from sqlbuild.compiler.planner._helpers.pruning import selection_staleness -from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( - format_stale_upstream_warning_message, -) -from sqlbuild.compiler.planner._helpers.pruning.selection_staleness import ( - build_stale_out_of_selection_warnings, -) from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, @@ -47,10 +40,10 @@ PlannerStaleWarningScopeTestCase, SelectionStalenessGraphWarningTestCase, SelectionStalenessWarningTestCase, - StaleWarningMessageTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_selection_model, + build_unreadable_model, ) MODEL_KEY: CompiledObjectKey = CompiledObjectKey( @@ -64,20 +57,15 @@ [ CompleteModelChangesWorkTestCase( description="complete model changes", - expected_detector_calls=0, + expected_warnings=(), ) ], ids=lambda case: case.description, ) def test_given_complete_change_results_when_building_stale_warnings_then_models_are_not_redetected( test_case: CompleteModelChangesWorkTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: - model: CompiledModelView = build_selection_model( - key=MODEL_KEY, - name=MODEL_KEY.name, - materialized="table", - ) + model: CompiledModelView = build_unreadable_model() scope: PlannerScope = PlannerScope( upstream_deps={MODEL_KEY: ()}, downstream_deps={}, @@ -86,11 +74,6 @@ def test_given_complete_change_results_when_building_stale_warnings_then_models_ selected_keys=frozenset({MODEL_KEY}), execution_order=(MODEL_KEY,), ) - model_detector: Mock = Mock( - side_effect=AssertionError("complete change results must not trigger model detection") - ) - monkeypatch.setattr(selection_staleness, "detect_model_changes", model_detector) - warnings: tuple[PlanWarning, ...] = build_stale_out_of_selection_warnings( original_scope=scope, execution_scope=scope, @@ -116,8 +99,7 @@ def test_given_complete_change_results_when_building_stale_warnings_then_models_ model_changes_complete=True, ) - assert warnings == () - assert model_detector.call_count == test_case.expected_detector_calls + assert warnings == test_case.expected_warnings SEED_KEY: CompiledObjectKey = CompiledObjectKey( @@ -561,60 +543,3 @@ def test_given_stale_upstream_graph_when_classifying_then_reports_expected_trigg unexpected_fragment: str for unexpected_fragment in test_case.unexpected_warning_fragments: assert unexpected_fragment not in warning_text - - -@pytest.mark.parametrize( - "test_case", - [ - StaleWarningMessageTestCase( - description="lists every trigger as a bullet when under the cap", - model_label="selected model", - model_name="leaf", - trigger_label="upstream(s)", - trigger_names=("a", "b", "c"), - expected_fragments=( - "selected model 'leaf' will build on 3 stale upstream(s) not selected for rebuild:", - " - a", - " - b", - " - c", - " rebuild the closure to refresh them: --select +leaf", - ), - unexpected_fragments=("more",), - expected_bullet_count=3, - ), - StaleWarningMessageTestCase( - description="caps the bullet list at five and summarizes the remainder", - model_label="selected dbt model", - model_name="leaf", - trigger_label="upstream(s)", - trigger_names=("a", "b", "c", "d", "e", "f", "g"), - expected_fragments=( - "selected dbt model 'leaf' will build on 7 stale upstream(s) not selected for rebuild:", - " - a", - " - e", - " +2 more", - " rebuild the closure to refresh them: --select +leaf", - ), - unexpected_fragments=(" - f", " - g"), - expected_bullet_count=5, - ), - ], - ids=lambda case: case.description, -) -def test_given_trigger_names_when_formatting_stale_warning_then_caps_and_structures_message( - test_case: StaleWarningMessageTestCase, -) -> None: - message: str = format_stale_upstream_warning_message( - model_label=test_case.model_label, - model_name=test_case.model_name, - trigger_label=test_case.trigger_label, - trigger_names=test_case.trigger_names, - ) - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in message - for fragment in test_case.unexpected_fragments: - assert fragment not in message - bullet_count: int = sum(line.strip().startswith("- ") for line in message.splitlines()) - assert bullet_count == test_case.expected_bullet_count diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index eb51f0cd49..63ff653331 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -2,16 +2,16 @@ import pytest -from sqlbuild._native import project_lineage_views +from sqlbuild._native import ( + build_downstream_deps, + parse_project_selector, + project_lineage_views, + resolve_project_selectors, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps -from sqlbuild.compiler.planner._helpers.graph.selectors import ( - parse_selector, - resolve_selectors, -) from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.planner.types import SelectorKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -136,7 +136,7 @@ def test_given_raw_selector_when_parsing_then_returns_expected_result( test_case: ParseSelectorTestCase, ) -> None: - result: ParsedSelector | PathSelector = parse_selector(test_case.raw) + result: ParsedSelector | PathSelector = parse_project_selector(test_case.raw) assert result == test_case.expected_result @@ -181,7 +181,7 @@ def test_given_invalid_selector_when_parsing_then_raises( test_case: ParseSelectorErrorTestCase, ) -> None: with pytest.raises(test_case.expected_error_type): - parse_selector(test_case.raw) + parse_project_selector(test_case.raw) @pytest.mark.parametrize( @@ -332,12 +332,14 @@ def test_given_selectors_when_resolving_then_returns_expected_names( upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[1] downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[2] - result: frozenset[CompiledObjectKey] = resolve_selectors( + result: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, + path_index={}, ) result_names: frozenset[str] = frozenset(key.name for key in result) @@ -390,12 +392,14 @@ def test_given_model_table_function_dependency_when_selecting_then_graph_expands key.name: key for key in (base_key, function_key, consumer_key) } - selected: frozenset[CompiledObjectKey] = resolve_selectors( + selected: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, + path_index={}, ) assert frozenset(key.name for key in selected) == test_case.expected_names @@ -432,12 +436,14 @@ def test_given_unknown_name_when_resolving_then_raises( downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[2] with pytest.raises(test_case.expected_error_type): - resolve_selectors( + resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, + path_index={}, ) @@ -484,13 +490,14 @@ def test_given_tag_selectors_when_resolving_then_returns_expected_names( upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[1] downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[2] - result: frozenset[CompiledObjectKey] = resolve_selectors( + result: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, tag_index=diamond_tag_index, + path_index={}, ) result_names: frozenset[str] = frozenset(key.name for key in result) @@ -604,12 +611,13 @@ def test_given_path_selectors_when_resolving_then_returns_expected_names( downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = path_graph[2] path_idx: dict[CompiledObjectKey, str] = path_graph[3] - result: frozenset[CompiledObjectKey] = resolve_selectors( + result: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, path_index=path_idx, ) result_names: frozenset[str] = frozenset(key.name for key in result) @@ -659,12 +667,13 @@ def test_given_invalid_path_selector_when_resolving_then_raises_with_message( path_idx: dict[CompiledObjectKey, str] = path_graph[3] with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_selectors( + resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, path_index=path_idx, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index b8f8830034..06ad20a51d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -8,7 +8,12 @@ import pytest from sqlbuild._native import ( + build_intermediate_source_map, + build_source_load_entries, + build_source_load_map, + expand_selected_loader_dependencies, project_with, + upstream_loader_dependency_names, ) from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -22,14 +27,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_intermediate_source_map, - expand_selected_loader_dependencies, -) -from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import ( - build_source_load_entries, - build_source_load_map, -) +from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.planner._helpers.warehouse.snapshot import _build_metadata_name_filter from sqlbuild.compiler.planner.models import SourceLoadPlanEntry from sqlbuild.runtime.contracts.types import ExecutionResourceKind @@ -37,6 +35,7 @@ from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( LoaderDagExpansionTestCase, + LoaderDependencyNamesTestCase, LoaderDestinationPlanningTestCase, MetadataNameFilterTestCase, SourceLoadNodesTestCase, @@ -326,3 +325,31 @@ def test_given_qualified_intermediate_destination_when_building_map_then_parses_ entry: SourceEntry = entries["fetch_orders"] assert (entry.database, entry.schema, entry.table) == test_case.expected_parts + + +@pytest.mark.parametrize( + "test_case", + [ + LoaderDependencyNamesTestCase( + description="drops non-loader function dependencies and deduplicates repeated loaders", + expected_names=("fetch_orders",), + ) + ], + ids=lambda case: case.description, +) +def test_given_mixed_function_dependencies_when_resolving_then_returns_only_known_loaders( + test_case: LoaderDependencyNamesTestCase, +) -> None: + project: CompiledProjectView = build_source_load_nodes_project() + dependency: DiscoveredLoaderFunction = project.loader_functions[0] + root: DiscoveredLoaderFunction = replace( + project.loader_functions[1], + depends_on=(dependency.function, build_source_load_nodes_project, dependency.function), + ) + assert ( + upstream_loader_dependency_names( + loader_function=root, + loader_functions=project.loader_functions, + ) + == test_case.expected_names + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index 621c272656..a67263cbbc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -181,3 +181,10 @@ class ProtectedRebuildBlockedTestCase: description: str external_blocked_model_names: tuple[str, ...] expected_action: str + + +@dataclass(frozen=True) +class DeepChainStalenessTestCase: + description: str + depth: int + expected_trigger_count: int diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index d1fb0ad54b..6fd2cae9c2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -5,7 +5,11 @@ from pathlib import Path from typing import Any -from sqlbuild._native import compute_query_hash, write_fingerprint +from sqlbuild._native import ( + build_version_identity_metadata_json, + compute_query_hash, + write_fingerprint, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -21,15 +25,14 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, PlannerPolicies, PlannerSelection, PlanOutput, + SelectionStalenessGraph, + SelectionStalenessNodeKey, ) from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry @@ -176,3 +179,22 @@ def create_protected_model_state(*, adapter: DuckDbAdapter, connection: Any) -> render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, ) + + +def build_chain_staleness_graph(*, depth: int) -> SelectionStalenessGraph: + """Build a model chain whose most upstream model changed and only the head runs.""" + + keys: list[SelectionStalenessNodeKey] = [ + SelectionStalenessNodeKey(resource_type="model", name=f"m{index}") for index in range(depth) + ] + head: str = keys[-1].name + return SelectionStalenessGraph( + upstream_deps={key: (upstream,) for key, upstream in zip(keys[1:], keys[:-1], strict=True)}, + selected_model_names=frozenset({head}), + run_model_names=frozenset({head}), + run_seed_names=frozenset(), + run_source_names=frozenset(), + changed_model_names=frozenset({"m0"}), + changed_seed_names=frozenset(), + changed_source_names=frozenset(), + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py index 53a0099164..b573b77eb9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py @@ -1,4 +1,6 @@ +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path @dataclass(frozen=True) @@ -46,3 +48,17 @@ class HookIdentityPayloadTestCase: hooks: dict[str, object] expected_fragments: tuple[str, ...] forbidden_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PersistedIdentityTestCase: + description: str + build: Callable[[Path], object] + expected_identity: object + + +@dataclass(frozen=True) +class DeepChainIdentityTestCase: + description: str + depth: int + expected_hash_count: int diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index cd00e48e5d..8f7c4ad4ae 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -1,7 +1,13 @@ +import json from dataclasses import replace from pathlib import Path +from types import SimpleNamespace from sqlbuild._native import ( + build_direct_model_version_identities, + build_downstream_deps, + build_model_version_identity_metadata_json, + build_version_identity_metadata_json, compiled_model_record, ) from sqlbuild.compiler.compile.models import ( @@ -10,20 +16,29 @@ CompiledObjectKey, CompiledRelationLocation, CompileModelConfig, + FunctionArgument, FunctionReturnColumn, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps -from sqlbuild.compiler.planner._helpers.identity.direct import ( - build_direct_model_version_identities, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledResourceType, + FunctionLanguage, ) from sqlbuild.compiler.planner.models import DirectModelVersionIdentities, PlannerScope +from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function +def build_version_identity_config(config_values: dict[str, object]) -> dict[str, object]: + """Return the config entries recorded in a model's version identity metadata.""" + + metadata: dict[str, object] = json.loads( + build_version_identity_metadata_json(model_name="orders", config_values=config_values) + ) + config: dict[str, object] = metadata["config"] + return config + + def build_table_function_graph_identities(*, base_query: str) -> DirectModelVersionIdentities: base_key: CompiledObjectKey = CompiledObjectKey( resource_type=CompiledResourceType.MODEL, @@ -111,3 +126,72 @@ def build_hooked_model_identity_json(*, hook_config: dict[str, object]) -> str: model=model, hook_version_hashes={"notify_orders": "notify-hash-v1"}, ) + + +def build_contract_model() -> SimpleNamespace: + """Build a model view with one enforced declared column.""" + + return SimpleNamespace( + config=SimpleNamespace(values={"contract": "enforced"}), + schema_entry=SimpleNamespace( + columns=(SchemaColumn(name="id", type="int", nullable=False),), + dynamic_columns=(), + type_enforcement=True, + ), + ) + + +def build_identity_function() -> SimpleNamespace: + """Build a SQL scalar function view with one argument.""" + + return SimpleNamespace( + name="add_one", + arguments=(FunctionArgument(name="x", type="int"),), + returns="int", + return_columns=(), + body_sql="x + 1", + language=FunctionLanguage.SQL, + runtime_version=None, + entry_point=None, + packages=(), + ) + + +def build_identity_seed(directory: Path) -> SimpleNamespace: + """Build a seed view over a two-row CSV file with Windows line endings.""" + + file_path: Path = directory / "regions.csv" + file_path.write_text("id,name\r\n1,café\r\n", encoding="utf-8") + return SimpleNamespace( + name="regions", + external=False, + seed_file=SimpleNamespace(file_path=file_path), + schema_entry=SimpleNamespace( + csv_settings=SeedCsvSettings(), + columns=(SchemaColumn(name="id", type="int"),), + ), + ) + + +def build_chain_graph_identities(*, depth: int) -> DirectModelVersionIdentities: + """Build identities of a model chain resolved from its most downstream model.""" + + keys: list[CompiledObjectKey] = [ + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=f"m{index}") + for index in range(depth) + ] + upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { + key: tuple(keys[index - 1 : index]) for index, key in enumerate(keys) + } + scope: PlannerScope = PlannerScope( + upstream_deps=upstream, + downstream_deps={}, + all_keys={key.name: key for key in keys}, + models_by_name={ + key.name: _build_model(key=key, query_sql="SELECT 1 AS id", deps=upstream[key]) + for key in keys + }, + selected_keys=frozenset(keys), + execution_order=tuple(reversed(keys)), + ) + return build_direct_model_version_identities(functions=(), scope=scope, dialect=None) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py new file mode 100644 index 0000000000..95faa636e2 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner.models import DirectModelVersionIdentities +from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( + DeepChainIdentityTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner.main.identity.helpers import ( + build_chain_graph_identities, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeepChainIdentityTestCase( + description="one hundred thousand chained models", + depth=100_000, + expected_hash_count=100_000, + ), + ], + ids=lambda case: case.description, +) +def test_given_deep_model_chain_when_building_identities_then_resolves_every_model( + test_case: DeepChainIdentityTestCase, +) -> None: + identities: DirectModelVersionIdentities = build_chain_graph_identities(depth=test_case.depth) + + assert len(identities.model_version_hashes) == test_case.expected_hash_count diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py new file mode 100644 index 0000000000..dbc48e6433 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py @@ -0,0 +1,83 @@ +"""Version identities are persisted in warehouse state; these values must never change.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import ( + build_function_local_hashes, + build_seed_identity, + build_version_identity_metadata_json, + declared_columns_hash, +) +from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( + PersistedIdentityTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner.main.identity.helpers import ( + build_contract_model, + build_identity_function, + build_identity_seed, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PersistedIdentityTestCase( + description="version identity metadata", + build=lambda _: build_version_identity_metadata_json( + model_name="orders", + config_values={ + "materialized": "table", + "batch_size": 10, + "lookback": 1.5, + "unique_key": ("id", "café"), + "cursor_inputs": {"b": None, "a": True}, + "ignored": 1, + }, + local_function_hashes={"f": "h"}, + ), + expected_identity=( + '{"config":{"batch_size":10,"cursor_inputs":{"a":true,"b":null},' + '"lookback":1.5,"materialized":"table","unique_key":["id","caf\\u00e9"]},' + '"execution_signature":{},"local_function_hashes":{"f":"h"},"model_name":"orders"}' + ), + ), + PersistedIdentityTestCase( + description="declared columns hash", + build=lambda _: declared_columns_hash(model=build_contract_model()), + expected_identity="fd43639fd4c7a585e25ad4bc134d59e890cb81f34f87c7083bfbb83a7c5a45c9", + ), + PersistedIdentityTestCase( + description="function local hash", + build=lambda _: build_function_local_hashes( + functions=(build_identity_function(),), dialect="duckdb" + ), + expected_identity={ + "add_one": "f0b98ef7f39ec80c918937418d5d1b13c01407d83846e9817ce4402748d74d97" + }, + ), + PersistedIdentityTestCase( + description="seed identity", + build=lambda directory: build_seed_identity(build_identity_seed(directory)), + expected_identity=( + "37b6c31dd600a9c5b2f811338382d211faa6b7278e6e552cec636fdabf243323", + '{"columns":[{"name":"id","nullable":null,"type":"int"}],"csv_settings":' + '{"delimiter":null,"doublequote":null,"encoding":null,"escapechar":null,' + '"keep_default_na":null,"lineterminator":null,"na_values":null,"quotechar":null,' + '"skipinitialspace":null},"name":"regions","rows":[["id","name"],["1","caf\\u00e9"]]}', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_fixed_inputs_when_building_identities_then_matches_pinned_values( + test_case: PersistedIdentityTestCase, tmp_path: Path +) -> None: + assert test_case.build(tmp_path) == test_case.expected_identity + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py index 6446e8d359..580b489c60 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild.compiler.planner.main.identity._version_identity_config import ( - build_version_identity_config, -) from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( CursorRoleIdentityTestCase, VersionIdentityConfigTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner.main.identity.helpers import ( + build_version_identity_config, +) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index f578be893f..1062a23f69 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -4,15 +4,13 @@ import pytest +from sqlbuild._native import build_function_local_hashes from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledObjectKey, FunctionReturnColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) from sqlbuild.compiler.planner.models import DirectModelVersionIdentities from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py index f0b8a9edb4..e58e05e028 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py @@ -2,17 +2,17 @@ import pytest -from sqlbuild.compiler.planner.main.changes._selection_staleness import ( - classify_selection_staleness_warnings, -) +from sqlbuild._native import classify_selection_staleness_warnings from sqlbuild.compiler.planner.models import ( SelectionStalenessGraph, SelectionStalenessNodeKey, SelectionStalenessWarning, ) from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( + DeepChainStalenessTestCase, SelectionStalenessClassifierTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner.main.helpers import build_chain_staleness_graph MODEL_A: SelectionStalenessNodeKey = SelectionStalenessNodeKey(resource_type="model", name="a") MODEL_B: SelectionStalenessNodeKey = SelectionStalenessNodeKey(resource_type="model", name="b") @@ -483,3 +483,26 @@ def test_given_graph_when_classifying_then_reports_expected_warnings( ) assert warnings == test_case.expected_warnings + + +@pytest.mark.parametrize( + "test_case", + [ + DeepChainStalenessTestCase( + description="one hundred thousand chained models with a changed root", + depth=100_000, + expected_trigger_count=99_999, + ), + ], + ids=lambda case: case.description, +) +def test_given_deep_chain_when_classifying_then_walks_without_exhausting_the_stack( + test_case: DeepChainStalenessTestCase, +) -> None: + warnings: tuple[SelectionStalenessWarning, ...] = classify_selection_staleness_warnings( + build_chain_staleness_graph(depth=test_case.depth) + ) + + assert sum(len(warning.trigger_names) for warning in warnings) == ( + test_case.expected_trigger_count + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py index 52b4ca3ca9..54984ac676 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion +from sqlbuild._native import split_selector_expansion from sqlbuild.compiler.planner.models import SelectorExpansion from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( SelectorExpansionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py index ae9d266cf7..e46555e333 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( - resolve_sqlbuild_model_selector_names, -) +from sqlbuild._native import resolve_sqlbuild_model_selector_names from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( SqlbuildModelSelectorNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 2bd507852b..86b9b54830 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import resolve_python_node_selection -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector +from sqlbuild._native import parse_project_selector, resolve_python_node_selection from sqlbuild.compiler.planner.models import ParsedSelector from sqlbuild.compiler.planner.types import SelectorKind from sqlbuild.compiler.python_nodes.models import PythonNodeGraph diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 888a3350c6..47297041ae 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -4,9 +4,9 @@ import pytest +from sqlbuild._native import resolve_project_selectors from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py index 89c32a2216..068f788a87 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py @@ -58,4 +58,5 @@ class StatusReporterOutputTestCase: @dataclass(frozen=True) class StatusReporterExitTestCase: description: str + script: str expected_returncode: int diff --git a/tests/unit/src/sqlbuild/presentation/classes/helpers.py b/tests/unit/src/sqlbuild/presentation/classes/helpers.py index ebb758f669..829f388973 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/helpers.py +++ b/tests/unit/src/sqlbuild/presentation/classes/helpers.py @@ -80,6 +80,14 @@ def persistent_lines(text: str) -> tuple[str, ...]: return tuple(line.rsplit("\x1b[K", 1)[-1] for line in without_cursor.split("\n")) +SLOW_FINALIZATION_SCRIPT: str = ( + "class SlowFinalizer:\n" + " def __init__(self):\n" + " self.cycle = self\n" + " def __del__(self):\n" + " time.sleep(0.5)\n" + "SlowFinalizer()\n" +) OPEN_SPINNER_SCRIPT: str = ( "import io, time\n" "from sqlbuild._native import TransientStatusReporter\n" @@ -89,7 +97,18 @@ def persistent_lines(text: str) -> tuple[str, ...]: "reporter = TransientStatusReporter(stream=Tty())\n" "reporter.start('Working...')\n" "time.sleep(0.35)\n" -) +) + SLOW_FINALIZATION_SCRIPT +OPEN_NESTED_SPINNER_SCRIPT: str = ( + "import io, time\n" + "from sqlbuild._native import NestedCommandProgressCallbacks\n" + "class Tty(io.StringIO):\n" + " def isatty(self):\n" + " return True\n" + "reporter = NestedCommandProgressCallbacks(\n" + " total=1, label='audits', stream=Tty(), use_color=False\n" + ")\n" + "reporter.on_item_start(group_name='orders', item_name='not_null')\n" +) + SLOW_FINALIZATION_SCRIPT def without_no_progress_env() -> dict[str, str]: diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index 9f0c1e5db5..d69ab52cc5 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -7,13 +7,13 @@ import pytest from sqlbuild._native import TransientStatusReporter -from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from tests.unit.src.sqlbuild.presentation.classes._test_types import ( ProgressSpinnersDisabledTestCase, StatusReporterExitTestCase, StatusReporterOutputTestCase, ) from tests.unit.src.sqlbuild.presentation.classes.helpers import ( + OPEN_NESTED_SPINNER_SCRIPT, OPEN_SPINNER_SCRIPT, RecordingStream, apply_no_progress_env, @@ -28,56 +28,6 @@ def isatty(self) -> bool: return True -@pytest.mark.parametrize( - "test_case", - ( - ProgressSpinnersDisabledTestCase( - description="unset env keeps spinners enabled", - env_value=None, - expected_disabled=False, - ), - ProgressSpinnersDisabledTestCase( - description="truthy '1' disables spinners", - env_value="1", - expected_disabled=True, - ), - ProgressSpinnersDisabledTestCase( - description="truthy 'true' is case-insensitive and trimmed", - env_value=" TRUE ", - expected_disabled=True, - ), - ProgressSpinnersDisabledTestCase( - description="truthy 'on' disables spinners", - env_value="on", - expected_disabled=True, - ), - ProgressSpinnersDisabledTestCase( - description="falsey '0' keeps spinners enabled", - env_value="0", - expected_disabled=False, - ), - ProgressSpinnersDisabledTestCase( - description="arbitrary value keeps spinners enabled", - env_value="maybe", - expected_disabled=False, - ), - ProgressSpinnersDisabledTestCase( - description="empty value keeps spinners enabled", - env_value="", - expected_disabled=False, - ), - ), - ids=lambda case: case.description, -) -def test_given_no_progress_env_when_checking_then_reports_expected_disabled_state( - test_case: ProgressSpinnersDisabledTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - apply_no_progress_env(monkeypatch=monkeypatch, env_value=test_case.env_value) - - assert progress_spinners_disabled() == test_case.expected_disabled - - @pytest.mark.parametrize( "test_case", ( @@ -191,6 +141,12 @@ def test_given_status_sequence_when_reporting_then_writes_expected_persistent_li ( StatusReporterExitTestCase( description="an open spinner is stopped before the interpreter exits", + script=OPEN_SPINNER_SCRIPT, + expected_returncode=0, + ), + StatusReporterExitTestCase( + description="an unclosed nested progress spinner is stopped before finalization", + script=OPEN_NESTED_SPINNER_SCRIPT, expected_returncode=0, ), ), @@ -200,7 +156,7 @@ def test_given_open_spinner_when_process_exits_then_exits_cleanly( test_case: StatusReporterExitTestCase, ) -> None: completed: subprocess.CompletedProcess[str] = subprocess.run( - (sys.executable, "-c", OPEN_SPINNER_SCRIPT), + (sys.executable, "-c", test_case.script), capture_output=True, text=True, check=False, diff --git a/tests/unit/src/sqlbuild/presentation/helpers.py b/tests/unit/src/sqlbuild/presentation/helpers.py new file mode 100644 index 0000000000..7c2cf34b10 --- /dev/null +++ b/tests/unit/src/sqlbuild/presentation/helpers.py @@ -0,0 +1,40 @@ +"""Terminal colour capability fixtures for CLI output tests.""" + +from __future__ import annotations + +import sys +from typing import TextIO + +import pytest + + +class ColorTerminalStream: + """A text stream that reports a terminal and writes through to the captured stream.""" + + def __init__(self, stream: TextIO) -> None: + self._stream: TextIO = stream + + def isatty(self) -> bool: + return True + + def write(self, text: str) -> int: + return self._stream.write(text) + + def flush(self) -> None: + self._stream.flush() + + def __getattr__(self, name: str) -> object: + return getattr(self._stream, name) + + +def use_color_terminal(monkeypatch: pytest.MonkeyPatch) -> None: + """Make `sys.stdout` a colour terminal that still writes to the captured output.""" + + monkeypatch.delenv("NO_COLOR", raising=False) + monkeypatch.setattr(sys, "stdout", ColorTerminalStream(sys.stdout)) + + +def use_plain_terminal(monkeypatch: pytest.MonkeyPatch) -> None: + """Disable terminal colour through `NO_COLOR`.""" + + monkeypatch.setenv("NO_COLOR", "1") diff --git a/tests/unit/src/sqlbuild/presentation/main/_test_types.py b/tests/unit/src/sqlbuild/presentation/main/_test_types.py index 635b17ba07..1851522168 100644 --- a/tests/unit/src/sqlbuild/presentation/main/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/main/_test_types.py @@ -1,17 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class FormatCodedErrorTestCase: - description: str - code: str - message: str - help: str | None - use_color: bool - expected_rendered: str - include_error_label: bool = True - - @dataclass(frozen=True) class SummaryFooterTestCase: description: str @@ -21,13 +10,6 @@ class SummaryFooterTestCase: expected_color_fragments: tuple[str, ...] -@dataclass(frozen=True) -class InlineErrorLinesTestCase: - description: str - content_width: int - expected_lines: list[str] - - @dataclass(frozen=True) class CountHeaderTestCase: description: str diff --git a/tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py b/tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py deleted file mode 100644 index 56cc1aea2d..0000000000 --- a/tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Tests for coded error rendering.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.presentation.main.coded_error_text import format_coded_error -from tests.unit.src.sqlbuild.presentation.main._test_types import FormatCodedErrorTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - FormatCodedErrorTestCase( - description="renders plain coded error", - code="R002", - message="materialization failed", - help=None, - use_color=False, - expected_rendered="error[R002]: materialization failed", - ), - FormatCodedErrorTestCase( - description="renders coded detail beneath an existing error label", - code="T002", - message="test execution failed", - help=None, - use_color=False, - expected_rendered="[T002] test execution failed", - include_error_label=False, - ), - FormatCodedErrorTestCase( - description="renders plain coded error with help", - code="K011", - message="contract requires staged promotion", - help="set table_promotion_mode to staged", - use_color=False, - expected_rendered=( - "error[K011]: contract requires staged promotion\n" - " = help: set table_promotion_mode to staged" - ), - ), - FormatCodedErrorTestCase( - description="renders colorized coded error with help", - code="R006", - message="audit failed", - help="fix failing audit rows", - use_color=True, - expected_rendered=( - "\033[38;5;167m\033[1merror[R006]:\033[0m audit failed\n" - " \033[2m= help:\033[0m fix failing audit rows" - ), - ), - FormatCodedErrorTestCase( - description="renders multiline coded error without color unchanged", - code="D012", - message=( - "Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "1 validation error for SlackProvider\n" - "SLACK_TOKEN" - ), - help=None, - use_color=False, - expected_rendered=( - "error[D012]: Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "1 validation error for SlackProvider\n" - "SLACK_TOKEN" - ), - ), - FormatCodedErrorTestCase( - description="renders multiline coded error with muted continuation lines", - code="D012", - message=( - "Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "1 validation error for SlackProvider\n" - "SLACK_TOKEN" - ), - help=None, - use_color=True, - expected_rendered=( - "\033[38;5;167m\033[1merror[D012]:\033[0m " - "Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "\033[2m1 validation error for SlackProvider\033[0m\n" - "\033[2mSLACK_TOKEN\033[0m" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_coded_error_when_formatting_then_renders_consistently( - test_case: FormatCodedErrorTestCase, -) -> None: - rendered: str = format_coded_error( - code=test_case.code, - message=test_case.message, - help=test_case.help, - use_color=test_case.use_color, - include_error_label=test_case.include_error_label, - ) - - assert rendered == test_case.expected_rendered diff --git a/tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py b/tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py deleted file mode 100644 index ebe3473032..0000000000 --- a/tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Tests for bounded inline error rendering.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from tests.unit.src.sqlbuild.presentation.main._test_types import InlineErrorLinesTestCase - - -@pytest.mark.parametrize( - "test_case", - ( - InlineErrorLinesTestCase( - description="one-column content keeps truncation marker within width", - content_width=1, - expected_lines=["a", "."], - ), - InlineErrorLinesTestCase( - description="wide content wraps twice before using an ellipsis", - content_width=8, - expected_lines=["abcdefgh", "ijklm..."], - ), - ), - ids=lambda case: case.description, -) -def test_given_content_width_when_formatting_inline_error_then_bounds_each_line( - test_case: InlineErrorLinesTestCase, -) -> None: - lines: list[str] = format_inline_error_lines( - error_code=None, - error_message="abcdefghijklmnopqrstuvwxyz", - error_help=None, - content_width=test_case.content_width, - style=CliStyle(use_color=False), - ) - - assert lines == test_case.expected_lines diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 6a5bc8a547..39123d7c0d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -9,8 +9,11 @@ import pytest +from sqlbuild import _native from sqlbuild._native import ( NativeRulesRequest, + build_rule_catalogue, + install_rules_skills, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -25,10 +28,8 @@ from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.run import native_rows -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES +from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES, RULES_SKILL_PATHS from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( CustomRulesOutcome, @@ -47,7 +48,8 @@ _MODEL_SQL: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final" -_BUILD_RULES_REQUEST: Callable[..., NativeRulesRequest] = native_rows._native.build_rules_request +_BUILD_RULES_REQUEST: Callable[..., NativeRulesRequest] = _native.build_rules_request +_SCOPE_PROJECTION: Callable[..., dict[str, Any]] = _native.scope_metadata_projection def project_with_scope(*, index: ScopeIndex) -> CompiledProjectView: @@ -69,17 +71,14 @@ def captured_native_request( captured: dict[str, Any] = {} - def build_rules_request( - project_rows: dict[str, Any], - models: tuple[object, list[str]], - sql_tests: list[object], - sql_scenarios: list[object], - ) -> object: - request: NativeRulesRequest = _BUILD_RULES_REQUEST( - project_rows, models, sql_tests, sql_scenarios - ) - captured.update(cast(dict[str, Any], json.loads(project_rows["header_json"]))) - captured.update(models=models, **json.loads(request.sql_facts_json())) + def scope_metadata_projection(**kwargs: Any) -> dict[str, Any]: + projection: dict[str, Any] = _SCOPE_PROJECTION(**kwargs) + captured.update(scope_index=projection) + return projection + + def build_rules_request(**kwargs: Any) -> object: + request: NativeRulesRequest = _BUILD_RULES_REQUEST(**kwargs) + captured.update(cast(dict[str, Any], json.loads(request.sql_facts_json()))) return request def evaluate_rules_request(request: object) -> NativeRulesEvaluation: @@ -93,7 +92,8 @@ def evaluate_rules_request(request: object) -> NativeRulesEvaluation: reused=False, ) - monkeypatch.setattr(native_rows._native, "build_rules_request", build_rules_request) + monkeypatch.setattr(_native, "scope_metadata_projection", scope_metadata_projection) + monkeypatch.setattr(native, "build_rules_request", build_rules_request) monkeypatch.setattr(native, "evaluate_rules_request", evaluate_rules_request) native.evaluate_native( project=project, @@ -226,7 +226,9 @@ def write_project_file_rule( def load_custom_rule(*, root: Path, configured_path: Path) -> Rule: del configured_path config: RulesConfig = RulesConfig() - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=root) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=root, include_custom=True + ) custom_rules: tuple[Rule, ...] = tuple(filter(attrgetter("custom"), catalogue)) return custom_rules[0] @@ -267,8 +269,8 @@ def custom_rules_with_imports( path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(source, encoding="utf-8") - catalogue: tuple[Rule, ...] = build_catalogue( - config=RulesConfig(select=("XSQBRT101",)), project_dir=project_dir + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=RulesConfig(select=("XSQBRT101",)), project_dir=project_dir, include_custom=True ) return tuple(filter(lambda rule: rule.custom, catalogue)) @@ -302,7 +304,10 @@ def evaluate_cached_custom_rules( select=("XSQBRT101",), cache=RulesCacheConfig(enabled=cache_enabled) ) rules: tuple[Rule, ...] = tuple( - filter(attrgetter("custom"), build_catalogue(config=config, project_dir=project_dir)) + filter( + attrgetter("custom"), + build_rule_catalogue(config=config, project_dir=project_dir, include_custom=True), + ) ) return evaluate_custom_rules_cached( project=project, config=config, project_dir=project_dir, rules=rules, dialect="duckdb" @@ -376,7 +381,9 @@ def evaluate_incremental_step( project=_incremental_project(step.models), config=config, project_dir=project_dir, - catalogue=build_catalogue(config=config, project_dir=project_dir, include_custom=False), + catalogue=build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=False + ), dialect=step.dialect, initial_findings=tuple( Finding( @@ -449,3 +456,10 @@ def scenario_summary(fact: dict[str, Any]) -> tuple[object, ...]: fact["assertion_target_model_names"], fact["target_model_names"], ) + + +def installed_skill(*, config: RulesConfig, project_dir: Path) -> str: + """Install the generated rules skill and return the first target's content.""" + + _ = install_rules_skills(config=config, project_dir=project_dir, check=False) + return (project_dir / RULES_SKILL_PATHS[0]).read_text(encoding="utf-8") diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 5593ed2d7e..2bce72eb50 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -4,11 +4,13 @@ import pytest -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine._helpers.engine.ruleset import resolve_ruleset -from sqlbuild.rule_engine._helpers.guidance.skills import render_skills +from sqlbuild._native import ( + build_rule_catalogue, + format_rule, + install_rules_skills, + select_catalogue_rules, +) from sqlbuild.rule_engine.main._evaluate import evaluate -from sqlbuild.rule_engine.main.render_rule import format_rule from sqlbuild.rule_engine.models import ( Rule, RulesCacheConfig, @@ -22,6 +24,7 @@ PolicySelectionTestCase, SqlTestPolicyGuidanceTestCase, ) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import installed_skill from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project RULES_BUILT_IN_CODES: tuple[str, ...] = ( @@ -167,9 +170,11 @@ def test_given_rule_selectors_when_resolving_then_returns_expected_codes( test_case: PolicySelectionTestCase, ) -> None: config: RulesConfig = RulesConfig(select=test_case.select, ignore=test_case.ignore) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=tmp_path) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=tmp_path, include_custom=True + ) - selected: tuple[Rule, ...] = select_rules( + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=config, project_dir=tmp_path ) @@ -195,12 +200,14 @@ def test_given_sql_test_policy_when_inspecting_and_rendering_skills_then_effecti select=("SQBRTEST",), sql_tests=SqlTestRulesConfig(pipeline_directory=test_case.pipeline_directory), ) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=tmp_path) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=tmp_path, include_custom=True + ) rules_by_code: dict[str, Rule] = {item.code: item for item in catalogue} rule: Rule = rules_by_code["SQBRTEST103"] inspection: str = format_rule(rule=rule, config=config) - skill, _ = render_skills(config=config, project_dir=tmp_path) + skill: str = installed_skill(config=config, project_dir=tmp_path) assert test_case.expected_path in inspection assert test_case.expected_path in skill @@ -236,7 +243,9 @@ def test_given_minimum_test_rule_when_rendering_guidance_then_all_surfaces_have_ threshold_overrides=(override,), cache=RulesCacheConfig(enabled=False), ) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=tmp_path) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=tmp_path, include_custom=True + ) rules_by_code: dict[str, Rule] = {item.code: item for item in catalogue} rule: Rule = rules_by_code["SQBRTEST202"] result: RulesResult = evaluate( @@ -250,17 +259,16 @@ def test_given_minimum_test_rule_when_rendering_guidance_then_all_surfaces_have_ project_dir=tmp_path, ) inspection: str = format_rule(rule=rule, config=config) - skill, _ = render_skills(config=config, project_dir=tmp_path) - without_override: str = resolve_ruleset( - config=RulesConfig(select=("SQBRTEST202",)), project_dir=tmp_path - ).fingerprint - with_override: str = resolve_ruleset(config=config, project_dir=tmp_path).fingerprint + skill: str = installed_skill(config=config, project_dir=tmp_path) + fresh_without_override: bool = install_rules_skills( + config=RulesConfig(select=("SQBRTEST202",)), project_dir=tmp_path, check=True + ) assert result.findings[0].remediation == rule.remediation assert f"Remediation: {rule.remediation}" in inspection assert f"Remediation: {rule.remediation}" in skill assert "marts require two focused tests" in inspection assert "marts require two focused tests" in skill - assert without_override != with_override + assert not fresh_without_override for snippet in test_case.expected_snippets: assert snippet in result.findings[0].remediation diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 095e19d3d9..a584d47f1c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import load_rules_config from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError -from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig, SqlTestRulesConfig from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( AllowExceptionsConfigTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index 65d1375bf6..89517c73b8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -5,11 +5,11 @@ import pytest from sqlbuild._native import ( + build_rule_catalogue, custom_rule_implementation_fingerprint, custom_rule_test_evidence, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult @@ -401,7 +401,7 @@ def test_given_duplicate_source_codes_when_building_catalogue_then_rejects_dupli config: RulesConfig = RulesConfig() with pytest.raises(RulesError, match=test_case.test_source) as raised: - build_catalogue(config=config, project_dir=tmp_path) + build_rule_catalogue(config=config, project_dir=tmp_path, include_custom=True) assert str(raised.value).count("XSQBRT101") == test_case.expected_count diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index daa5f077eb..9c581503f4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -14,10 +14,10 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + build_rule_catalogue, project_with, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.host import custom_host_pool from sqlbuild.rule_engine._helpers.host.module_state import module_state_token, rule_namespaces @@ -102,7 +102,10 @@ def evaluate_order_rules( select=(_MODEL_RULE, _PROJECT_RULE), cache=RulesCacheConfig(enabled=cache_enabled) ) rules: tuple[Rule, ...] = tuple( - filter(attrgetter("custom"), build_catalogue(config=config, project_dir=root)) + filter( + attrgetter("custom"), + build_rule_catalogue(config=config, project_dir=root, include_custom=True), + ) ) return evaluate_custom_rules_cached( project=project, config=config, project_dir=root, rules=rules, dialect="duckdb" @@ -365,7 +368,10 @@ def load_state_rules(*, root: Path, files: tuple[tuple[str, str], ...]) -> tuple path.parent.mkdir(parents=True, exist_ok=True) path.write_text(contents, encoding="utf-8") return tuple( - filter(attrgetter("custom"), build_catalogue(config=RulesConfig(), project_dir=root)) + filter( + attrgetter("custom"), + build_rule_catalogue(config=RulesConfig(), project_dir=root, include_custom=True), + ) ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 213a40c2aa..d14fe28676 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import rules_bulk_cache_path from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host, custom_host_pool -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from sqlbuild.rule_engine.models import CustomRulesOutcome diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py index 141ef13ceb..4a74d73192 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py @@ -2,9 +2,10 @@ import pytest +from sqlbuild import _native from sqlbuild._native import NativeRulesRequest +from sqlbuild.compiler.frontier.main import compiled_code_identity as code_identity from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native -from sqlbuild.rule_engine._helpers.run import native_rows from sqlbuild.rule_engine.models import NativeRulesEvaluation @@ -24,9 +25,7 @@ def record_reuse(*, monkeypatch: pytest.MonkeyPatch) -> list[bool]: """Record whether each natively evaluated request was answered from the response memo.""" reused: list[bool] = [] - evaluate: Callable[[NativeRulesRequest], NativeRulesEvaluation] = ( - native_rows.evaluate_rules_request - ) + evaluate: Callable[[NativeRulesRequest], NativeRulesEvaluation] = _native.evaluate_rules_request def recording(request: NativeRulesRequest) -> NativeRulesEvaluation: evaluation: NativeRulesEvaluation = evaluate(request) @@ -40,5 +39,5 @@ def recording(request: NativeRulesRequest) -> NativeRulesEvaluation: def rebuild_identity(*, monkeypatch: pytest.MonkeyPatch) -> None: """Pretend the compiled code changed, as after installing another build.""" - identity: str = native_rows.compiled_code_identity() - monkeypatch.setattr(native_rows, "compiled_code_identity", lambda: f"{identity}-rebuilt") + identity: str = code_identity.compiled_code_identity() + monkeypatch.setattr(code_identity, "compiled_code_identity", lambda: f"{identity}-rebuilt") diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py index a9dad4358d..e2ac06ade3 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.rule_engine.main.render_result import format_result +from sqlbuild._native import format_rules_result as format_result from sqlbuild.rule_engine.models import Finding, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.render_result._test_types import ( RenderResultParityTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py deleted file mode 100644 index 928e4a1882..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class RulesProgressTestCase: - description: str - elapsed_seconds: float - built_in_ms: int - custom_ms: int - expected_message: str - skipped_type_proof_rules: tuple[str, ...] = () - note_skipped_rules: bool = False diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py b/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py deleted file mode 100644 index 0e28eb9de1..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.rule_engine.main.render_rules_progress import format_rules_progress -from sqlbuild.rule_engine.models import RulesRunResult -from tests.unit.src.sqlbuild.rule_engine.main.render_rules_progress._test_types import ( - RulesProgressTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - RulesProgressTestCase( - description="wall time leads concurrent built-in and custom rule time", - elapsed_seconds=14.5, - built_in_ms=5_600, - custom_ms=13_400, - expected_message="Evaluated rules. (14.50s; built-in 5.60s, custom 13.40s)", - ), - RulesProgressTestCase( - description="skipped type-proof rules follow the timings when requested", - elapsed_seconds=0.25, - built_in_ms=100, - custom_ms=0, - skipped_type_proof_rules=("SQBRCONTRACT105",), - note_skipped_rules=True, - expected_message=( - "Evaluated rules. (0.25s; built-in 0.10s, custom 0.00s); skipped type-proof " - "rules (SQBRCONTRACT105) because SQL analysis is disabled" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_rules_run_when_formatting_progress_then_reports_phase_wall_time_first( - test_case: RulesProgressTestCase, -) -> None: - result: RulesRunResult = RulesRunResult( - findings=(), - evaluated_models=0, - built_in_ms=test_case.built_in_ms, - custom_ms=test_case.custom_ms, - skipped_type_proof_rules=test_case.skipped_type_proof_rules, - ) - - message: str = format_rules_progress( - elapsed_seconds=test_case.elapsed_seconds, - result=result, - note_skipped_rules=test_case.note_skipped_rules, - ) - - assert message == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py deleted file mode 100644 index cb6b90fdfe..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class SkippedTypeProofRulesNoteTestCase: - description: str - codes: tuple[str, ...] - expected_note: str | None diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py deleted file mode 100644 index 812a893850..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -from tests.unit.src.sqlbuild.rule_engine.main.render_skipped_rules._test_types import ( - SkippedTypeProofRulesNoteTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - SkippedTypeProofRulesNoteTestCase( - description="skipped rules produce one note", - codes=("SQBRCONTRACT105",), - expected_note=( - "skipped type-proof rules (SQBRCONTRACT105) because SQL analysis is disabled" - ), - ), - SkippedTypeProofRulesNoteTestCase( - description="no skipped rules produce no note", - codes=(), - expected_note=None, - ), - ), - ids=lambda case: case.description, -) -def test_given_skipped_type_proof_rules_when_formatting_note_then_names_skipped_codes( - test_case: SkippedTypeProofRulesNoteTestCase, -) -> None: - assert format_skipped_type_proof_rules(codes=test_case.codes) == test_case.expected_note diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 145414cc0e..0a2994db2d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -4,13 +4,16 @@ import pytest -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + discover_project_inputs, + load_rules_config, + resolve_adapter, + run_rules, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.run_rules import run_rules from sqlbuild.rule_engine.models import RulesRunResult from tests.unit.src.sqlbuild.rule_engine.main.run_rules._test_types import ( TypeProofRuleSkipTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py deleted file mode 100644 index f98746790c..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Process and failure orchestration for policy skill installation tests.""" - -from collections.abc import Callable -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event -from pathlib import Path -from typing import Any -from unittest.mock import patch - -from sqlbuild.rule_engine._helpers.guidance import skill_installation -from sqlbuild.rule_engine.main import skills as skills_module -from sqlbuild.rule_engine.models import RulesConfig - - -class FailingReplacement: - def __init__(self, *, original_replace: Callable[..., Path]) -> None: - self._original_replace = original_replace - self._temporary_replacements = 0 - - def __call__(self, path: Path, target: Path) -> Path: - if path.suffix == ".tmp": - self._temporary_replacements += 1 - if self._temporary_replacements == 2: - raise OSError("replacement failed") - if path.suffix == ".bak": - raise OSError("rollback failed") - return self._original_replace(path, target) - - -class ReplacementFailure: - def __init__(self, *, original_replace: Callable[..., Path]) -> None: - self._original_replace = original_replace - self._temporary_replacements = 0 - - def __call__(self, path: Path, target: Path) -> Path: - if path.suffix == ".tmp": - self._temporary_replacements += 1 - if self._temporary_replacements == 2: - raise OSError("replacement failed") - return self._original_replace(path, target) - - -class CoordinatedReplacementFailure: - def __init__( - self, - *, - original_replace: Callable[..., Path], - failure_reached: Event, - other_writer_attempting: Event, - ) -> None: - self._original_replace = original_replace - self._failure_reached = failure_reached - self._other_writer_attempting = other_writer_attempting - self._temporary_replacements = 0 - - def __call__(self, path: Path, target: Path) -> Path: - if path.suffix == ".tmp": - self._temporary_replacements += 1 - if self._temporary_replacements == 2: - self._failure_reached.set() - if not self._other_writer_attempting.wait(timeout=20): - raise TimeoutError("timed out waiting for the other skill writer") - raise OSError("replacement failed") - return self._original_replace(path, target) - - -class EditAfterBackupRead: - def __init__(self, *, original_read: Callable[..., bytes], target: Path) -> None: - self._original_read = original_read - self._target = target - self._edited = False - - def __call__(self, path: Path) -> bytes: - content: bytes = self._original_read(path) - if path == self._target and not self._edited: - self._edited = True - path.write_text("concurrent local edit\n", encoding="utf-8") - return content - - -def install_skills_in_process( - *, - root: str, - selected_rule: str, - result_queue: Queue[Any], - attempting: Event, - staged: Event | None = None, - release: Event | None = None, -) -> None: - original_stage: Callable[..., tuple[tuple[Path, Path | None, Path], ...]] = ( - skill_installation._stage_skills - ) - - def wait_and_stage( - *, targets: tuple[Path, ...], content: str - ) -> tuple[tuple[Path, Path | None, Path], ...]: - if staged is not None: - staged.set() - if release is not None and not release.wait(timeout=20): - raise TimeoutError("timed out waiting to release staged skill installation") - return original_stage(targets=targets, content=content) - - attempting.set() - try: - with patch.object(skill_installation, "_stage_skills", side_effect=wait_and_stage): - _ = skills_module.install_skills( - config=RulesConfig(select=(selected_rule,)), - project_dir=Path(root), - check=False, - ) - except Exception as error: - result_queue.put((selected_rule, type(error).__name__, str(error))) - else: - result_queue.put((selected_rule, "ok", "")) - - -def install_failing_skills_in_process( - *, - root: str, - selected_rule: str, - result_queue: Queue[Any], - failure_reached: Event, - other_writer_attempting: Event, -) -> None: - replacement: CoordinatedReplacementFailure = CoordinatedReplacementFailure( - original_replace=Path.replace, - failure_reached=failure_reached, - other_writer_attempting=other_writer_attempting, - ) - try: - with patch.object(Path, "replace", autospec=True, side_effect=replacement): - _ = skills_module.install_skills( - config=RulesConfig(select=(selected_rule,)), - project_dir=Path(root), - check=False, - ) - except Exception as error: - result_queue.put((selected_rule, type(error).__name__, str(error))) - else: - result_queue.put((selected_rule, "ok", "")) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index 209616fbbe..c6df6b64a3 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -1,19 +1,13 @@ """Generated rules skill freshness behavior tests.""" -from multiprocessing import get_context -from multiprocessing.context import SpawnContext -from multiprocessing.process import BaseProcess -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event from pathlib import Path from stat import S_IMODE -from unittest.mock import patch import pytest +from sqlbuild._native import install_rules_skills as install_skills from sqlbuild.rule_engine.constants import RULES_SKILL_PATHS from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.main.skills import install_skills from sqlbuild.rule_engine.models import RuleExemption, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( SkillDivergenceTestCase, @@ -21,13 +15,6 @@ SkillGuidanceTestCase, SkillInstallationTestCase, ) -from tests.unit.src.sqlbuild.rule_engine.main.skills.conftest import ( - EditAfterBackupRead, - FailingReplacement, - ReplacementFailure, - install_failing_skills_in_process, - install_skills_in_process, -) from tests.unit.src.sqlbuild.rule_engine.main.skills.helpers import write_custom_rule @@ -130,49 +117,6 @@ def test_given_later_target_failure_when_installing_then_all_targets_remain_unch assert unrelated.read_bytes() == b"keep me\n" -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="partial staging failure leaves every target unchanged", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="staging failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_staging_failure_when_installing_then_no_target_is_replaced( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - paths: tuple[Path, ...] = tuple(tmp_path / value for value in RULES_SKILL_PATHS) - expected_contents: tuple[bytes, ...] = tuple(path.read_bytes() for path in paths) - - with ( - patch.object( - Path, - "read_bytes", - side_effect=(expected_contents[0], OSError(test_case.expected_error)), - ), - pytest.raises(OSError, match=test_case.expected_error), - ): - install_skills( - config=RulesConfig(select=(test_case.changed_rule,)), - project_dir=tmp_path, - check=False, - ) - - assert tuple(path.read_bytes() for path in paths) == expected_contents - assert tuple(tmp_path.rglob("*.tmp")) == () - assert tuple(tmp_path.rglob("*.bak")) == () - - @pytest.mark.parametrize( "test_case", [ @@ -324,260 +268,3 @@ def test_given_same_policy_when_reinstalling_then_all_targets_are_byte_identical assert test_case.expected_fragment.encode() in second[0] assert test_case.absent_fragment.encode() not in second[0] assert tuple(S_IMODE(path.stat().st_mode) for path in paths) == (0o640, 0o640, 0o640) - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="different concurrent policies serialize complete installations", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="", - ) - ], - ids=lambda case: case.description, -) -def test_given_different_concurrent_policies_when_installing_then_last_writer_is_atomic( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - context: SpawnContext = get_context("spawn") - result_queue: Queue[object] = context.Queue() - first_attempting: Event = context.Event() - first_staged: Event = context.Event() - release_first: Event = context.Event() - second_attempting: Event = context.Event() - first: BaseProcess = context.Process( - target=install_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.selected_rule, - "result_queue": result_queue, - "attempting": first_attempting, - "staged": first_staged, - "release": release_first, - }, - ) - second: BaseProcess = context.Process( - target=install_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.changed_rule, - "result_queue": result_queue, - "attempting": second_attempting, - }, - ) - - first.start() - assert first_staged.wait(timeout=20) - second.start() - assert second_attempting.wait(timeout=20) - release_first.set() - first.join(timeout=20) - second.join(timeout=20) - - assert first.exitcode == 0 - assert second.exitcode == 0 - assert sorted((result_queue.get(timeout=5), result_queue.get(timeout=5))) == [ - (test_case.selected_rule, "ok", test_case.expected_error), - (test_case.changed_rule, "ok", test_case.expected_error), - ] - contents: tuple[str, ...] = tuple( - (tmp_path / value).read_text(encoding="utf-8") for value in RULES_SKILL_PATHS - ) - assert len(set(contents)) == 1 - assert test_case.changed_rule in contents[0] - assert test_case.selected_rule not in contents[0] - - -@pytest.mark.parametrize( - "test_case", - [ - SkillDivergenceTestCase( - description="target edited after staging", - selected_rule="SQBRMODEL101", - local_edit="concurrent local edit\n", - expected_error="changed during installation", - ) - ], - ids=lambda case: case.description, -) -def test_given_target_edit_between_validation_and_replacement_when_installing_then_edit_is_preserved( - tmp_path: Path, - test_case: SkillDivergenceTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - target: Path = tmp_path / RULES_SKILL_PATHS[0] - edit_after_backup: EditAfterBackupRead = EditAfterBackupRead( - original_read=Path.read_bytes, - target=target, - ) - - with ( - patch.object(Path, "read_bytes", autospec=True, side_effect=edit_after_backup), - pytest.raises(RulesError, match=test_case.expected_error), - ): - install_skills( - config=RulesConfig(select=("SQBRMODEL102",)), - project_dir=tmp_path, - check=False, - ) - - paths: tuple[Path, ...] = tuple(tmp_path / value for value in RULES_SKILL_PATHS) - assert paths[0].read_text(encoding="utf-8") == test_case.local_edit - assert test_case.selected_rule in paths[1].read_text(encoding="utf-8") - assert test_case.selected_rule in paths[2].read_text(encoding="utf-8") - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="rollback failure retains original error and recovery backup", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="replacement failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_rollback_failure_when_replacement_fails_then_original_error_and_backup_are_preserved( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - original_contents: bytes = (tmp_path / RULES_SKILL_PATHS[0]).read_bytes() - replacement: FailingReplacement = FailingReplacement(original_replace=Path.replace) - - with ( - patch.object(Path, "replace", autospec=True, side_effect=replacement), - pytest.raises(OSError, match=test_case.expected_error) as raised, - ): - install_skills( - config=RulesConfig(select=(test_case.changed_rule,)), - project_dir=tmp_path, - check=False, - ) - - backups: tuple[Path, ...] = tuple(tmp_path.rglob("*.bak")) - assert raised.value.__notes__ is not None - assert "rollback failed" in raised.value.__notes__[0] - assert len(backups) == 1 - assert backups[0].read_bytes() == original_contents - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="successful rollback restores original modes", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="replacement failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_replacement_failure_when_rolling_back_then_original_modes_are_restored( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - paths: tuple[Path, ...] = tuple(tmp_path / value for value in RULES_SKILL_PATHS) - for path in paths: - path.chmod(0o640) - replacement: ReplacementFailure = ReplacementFailure(original_replace=Path.replace) - - with ( - patch.object(Path, "replace", autospec=True, side_effect=replacement), - pytest.raises(OSError, match=test_case.expected_error), - ): - install_skills( - config=RulesConfig(select=(test_case.changed_rule,)), - project_dir=tmp_path, - check=False, - ) - - assert tuple(S_IMODE(path.stat().st_mode) for path in paths) == (0o640, 0o640, 0o640) - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="failed writer rolls back before successful writer installs", - selected_rule="SQBRMODEL102", - changed_rule="SQBRGRAPH101", - expected_error="replacement failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_failed_and_successful_concurrent_writers_when_installing_then_rollback_cannot_stale_overwrite( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=("SQBRMODEL101",)), - project_dir=tmp_path, - check=False, - ) - context: SpawnContext = get_context("spawn") - result_queue: Queue[object] = context.Queue() - failure_reached: Event = context.Event() - successful_writer_attempting: Event = context.Event() - failed: BaseProcess = context.Process( - target=install_failing_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.selected_rule, - "result_queue": result_queue, - "failure_reached": failure_reached, - "other_writer_attempting": successful_writer_attempting, - }, - ) - successful: BaseProcess = context.Process( - target=install_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.changed_rule, - "result_queue": result_queue, - "attempting": successful_writer_attempting, - }, - ) - - failed.start() - assert failure_reached.wait(timeout=20) - successful.start() - failed.join(timeout=20) - successful.join(timeout=20) - - assert failed.exitcode == 0 - assert successful.exitcode == 0 - results: set[tuple[str, str, str]] = { - result_queue.get(timeout=5), - result_queue.get(timeout=5), - } - assert results == { - (test_case.selected_rule, "OSError", test_case.expected_error), - (test_case.changed_rule, "ok", ""), - } - contents: tuple[str, ...] = tuple( - (tmp_path / value).read_text(encoding="utf-8") for value in RULES_SKILL_PATHS - ) - assert len(set(contents)) == 1 - assert test_case.changed_rule in contents[0] - assert test_case.selected_rule not in contents[0]