refactor: host lint fixes, migration proofs and dbt pipelines in Rust

.github/workflows/verify.yml

@@ -412,7 +412,7 @@ jobs:
     needs: [changes, lockfile]
     if: needs.changes.outputs.format_performance == 'true'
     runs-on: ubuntu-24.04
-    timeout-minutes: 10
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/setup-uv@v5
@@ -430,6 +430,7 @@ jobs:
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras --python 3.12
       - name: Run formatter performance guards
+        timeout-minutes: 10
         run: >-
           uv run pytest
           tests/e2e/src/sqlbuild/cli/commands/main/format/test_format_performance.py
@@ -659,7 +660,7 @@ jobs:
     needs: [changes, lockfile]
     if: needs.changes.outputs.performance == 'true'
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -681,6 +682,7 @@ jobs:
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
       - name: Run fresh-process compile cache guard under 4 GiB
+        timeout-minutes: 15
         env:
           BENCHMARK_MODELS: ${{ matrix.models }}
           MEMORY_LIMIT_BYTES: "4294967296"
@@ -703,6 +705,7 @@ jobs:
 
       - name: Run varied 1k warm-cache guard under 4 GiB
         if: matrix.models == 3000
+        timeout-minutes: 15
         env:
           MEMORY_LIMIT_BYTES: "4294967296"
         run: |

crates/sqlbuild-analysis/src/lib.rs

@@ -21,6 +21,7 @@ pub mod lineage_graph;
 pub mod manifest;
 pub mod planner_changes;
 pub mod planner_identity;
+pub mod planner_migrations;
 pub mod planner_scope;
 pub mod planner_selection;
 pub mod planner_strategy;

crates/sqlbuild-analysis/src/planner_migrations/_helpers/ast.rs

@@ -0,0 +1,109 @@
+//! Canonical migration AST values.
+
+use crate::planner_migrations::constants::{HAVING_MAX_KEY, OPAQUE_TUPLE_FIELDS};
+use serde_json::Value;
+use std::collections::BTreeSet;
+
+/// Tuple-valued AST fields are opaque to the historical Python dict/list walkers.
+pub(crate) fn opaque_field(key: &str, value: &Value) -> bool {
+    key == HAVING_MAX_KEY
+        || (OPAQUE_TUPLE_FIELDS.contains(&key)
+            && value
+                .as_array()
+                .is_some_and(|items| !items.is_empty() && items.iter().all(Value::is_array)))
+}
+
+pub(crate) fn strip_formatting(node: &mut Value) {
+    match node {
+        Value::Object(object) => {
+            object.shift_remove("span");
+            for (key, value) in object {
+                if opaque_field(key, value) {
+                    continue;
+                }
+                if key.ends_with("comments") {
+                    *value = Value::Array(Vec::new());
+                } else {
+                    strip_formatting(value);
+                }
+            }
+        }
+        Value::Array(items) => items.iter_mut().for_each(strip_formatting),
+        _ => {}
+    }
+}
+
+pub(crate) fn canonical(node: &mut Value) {
+    match node {
+        Value::Array(items) => items.iter_mut().for_each(canonical),
+        Value::Object(object) => {
+            for (key, value) in object.iter_mut() {
+                if !opaque_field(key, value) {
+                    canonical(value);
+                }
+            }
+            if object.get("quoted") == Some(&Value::Bool(false))
+                && let Some(Value::String(name)) = object.get_mut("name")
+            {
+                *name = name.to_lowercase();
+            }
+        }
+        _ => {}
+    }
+}
+
+pub(crate) fn identifier(node: &Value) -> Option<&str> {
+    node.get("name")?.as_str()
+}
+
+pub(crate) fn identifiers(node: &Value) -> Vec<String> {
+    items(node)
+        .iter()
+        .filter_map(identifier)
+        .map(str::to_lowercase)
+        .collect()
+}
+
+pub(crate) fn items(node: &Value) -> &[Value] {
+    node.as_array().map_or(&[], Vec::as_slice)
+}
+
+pub(crate) fn clause_items(node: &Value) -> &[Value] {
+    if node.is_array() {
+        items(node)
+    } else {
+        items(&node["expressions"])
+    }
+}
+
+pub(crate) fn references(node: &Value, unqualified: bool) -> BTreeSet<String> {
+    matching_fields(node, "column")
+        .into_iter()
+        .filter(|column| !unqualified || column["table"].is_null())
+        .filter_map(|column| identifier(&column["name"]))
+        .map(str::to_lowercase)
+        .collect()
+}
+
+pub(crate) fn matching_fields<'a>(node: &'a Value, key: &str) -> Vec<&'a Value> {
+    let mut found: Vec<&Value> = Vec::new();
+    let mut pending: Vec<&Value> = vec![node];
+    while let Some(current) = pending.pop() {
+        match current {
+            Value::Array(items) => pending.extend(items),
+            Value::Object(object) => {
+                if let Some(value) = object.get(key) {
+                    found.push(value);
+                }
+                pending.extend(
+                    object
+                        .iter()
+                        .filter(|(key, value)| !opaque_field(key, value))
+                        .map(|(_, value)| value),
+                );
+            }
+            _ => {}
+        }
+    }
+    found
+}

crates/sqlbuild-analysis/src/planner_migrations/_helpers/binding_proof.rs

@@ -0,0 +1,76 @@
+//! Conservative alias resolution in unchanged and renamed projections.
+
+use crate::planner_migrations::_helpers::ast::identifier;
+use crate::planner_migrations::models::QueryShape;
+use serde_json::Value;
+use std::collections::BTreeSet;
+
+pub(crate) fn other_references(shape: &QueryShape, excluded: &str) -> BTreeSet<String> {
+    shape
+        .projections
+        .iter()
+        .filter(|item| item.name.to_lowercase() != excluded)
+        .flat_map(|item| item.references.iter().cloned())
+        .collect()
+}
+
+pub(crate) fn binding_proven<E>(
+    shapes: (&QueryShape, &QueryShape),
+    names: (&str, &str),
+    input_columns: &mut impl FnMut(&str, &str) -> Result<Option<BTreeSet<String>>, E>,
+) -> Result<bool, E> {
+    let (previous, current) = shapes;
+    let (origin, destination) = names;
+    let mut referenced = previous.clause_references.clone();
+    referenced.extend(current.clause_references.iter().cloned());
+    referenced.extend(other_references(previous, origin));
+    referenced.extend(other_references(current, destination));
+    if !referenced.contains(origin) && !referenced.contains(destination) {
+        return Ok(true);
+    }
+    let Some(relations) = &current.input_relations else {
+        return Ok(false);
+    };
+    let mut inputs = current.local_input_columns.clone();
+    for (kind, name) in relations {
+        let Some(columns) = input_columns(kind, name)? else {
+            return Ok(false);
+        };
+        inputs.extend(columns);
+    }
+    if inputs.contains(destination) {
+        return Ok(false);
+    }
+    if inputs.contains(origin) {
+        return Ok(previous
+            .projections
+            .iter()
+            .any(|item| item.name.to_lowercase() == origin && item.passthrough));
+    }
+    Ok(!current.clause_references.contains(origin)
+        && !other_references(current, "").contains(origin))
+}
+
+pub(crate) fn substitute(node: &mut Value, names: &[(String, String)]) {
+    match node {
+        Value::Array(items) => items.iter_mut().for_each(|item| substitute(item, names)),
+        Value::Object(object) => {
+            if object.contains_key("select") {
+                return;
+            }
+            if let Some(column) = object.get_mut("column").filter(|column| column.is_object())
+                && column["table"].is_null()
+                && column["name"]["quoted"] == false
+                && let Some(name) = identifier(&column["name"])
+                && let Some((_, replacement)) = names.iter().rev().find(|(key, _)| key == name)
+            {
+                column["name"]["name"] = Value::String(replacement.clone());
+                return;
+            }
+            object
+                .values_mut()
+                .for_each(|value| substitute(value, names));
+        }
+        _ => {}
+    }
+}

crates/sqlbuild-analysis/src/planner_migrations/_helpers/clauses.rs

@@ -0,0 +1,37 @@
+//! Non-projection clauses and positional references of a SELECT.
+
+use crate::planner_migrations::_helpers::ast::{clause_items, items, references};
+use crate::planner_migrations::constants::NUMBER_LITERAL;
+use serde_json::Value;
+use std::collections::BTreeSet;
+
+pub(crate) fn clause_references(select: &Value) -> BTreeSet<String> {
+    let clauses: Vec<Value> = select
+        .as_object()
+        .into_iter()
+        .flat_map(|object| object.iter())
+        .filter(|(key, _)| !["expressions", "with", "from", "joins"].contains(&key.as_str()))
+        .map(|(_, value)| value.clone())
+        .chain(
+            items(&select["joins"])
+                .iter()
+                .filter(|join| join.is_object())
+                .map(|join| join["on"].clone()),
+        )
+        .collect();
+    references(&Value::Array(clauses), true)
+}
+
+pub(crate) fn positional(select: &Value) -> bool {
+    clause_items(&select["group_by"])
+        .iter()
+        .chain(clause_items(&select["order_by"]).iter().map(|item| {
+            if item.is_object() {
+                &item["this"]
+            } else {
+                item
+            }
+        }))
+        .chain(clause_items(&select["distinct_on"]))
+        .any(|item| item["literal"]["literal_type"] == NUMBER_LITERAL)
+}

crates/sqlbuild-analysis/src/planner_migrations/_helpers/encoding.rs

@@ -0,0 +1,42 @@
+//! Python-compatible compact sorted JSON for migration proofs.
+
+use serde_json::Value;
+use sqlbuild_core::json::main::dumps::dumps;
+use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions};
+
+pub(crate) fn encode(value: &Value) -> Result<String, String> {
+    dumps(
+        &converted(value)?,
+        &JsonDialect::Stdlib(StdlibJsonOptions {
+            sort_keys: true,
+            ..StdlibJsonOptions::new(None).with_separators(",", ":")
+        }),
+    )
+    .map_err(|error| format!("{error:?}"))
+}
+
+fn converted(value: &Value) -> Result<JsonValue, String> {
+    Ok(match value {
+        Value::Null => JsonValue::Null,
+        Value::Bool(flag) => JsonValue::Bool(*flag),
+        Value::String(text) => JsonValue::String(text.clone()),
+        Value::Number(number) => {
+            if !number.is_f64() {
+                JsonValue::Integer(
+                    JsonInteger::parse(&number.to_string()).ok_or("invalid integer")?,
+                )
+            } else {
+                JsonValue::Float(number.as_f64().ok_or("invalid float")?)
+            }
+        }
+        Value::Array(items) => {
+            JsonValue::Array(items.iter().map(converted).collect::<Result<_, _>>()?)
+        }
+        Value::Object(object) => JsonValue::Object(
+            object
+                .iter()
+                .map(|(key, value)| Ok((key.clone(), converted(value)?)))
+                .collect::<Result<_, String>>()?,
+        ),
+    })
+}