refactor: port planner change detection and strategy, progress reporters and compiler glue to the native extension

Makefile

@@ -8,7 +8,7 @@ SHELL := /bin/bash
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
 	test-e2e-dense-compile-performance test-e2e-varied-cache-performance \
 	compiler-differential compiler-differential-dense \
-	compiler-baselines
+	compiler-baselines planner-differential
 
 format:
 	uv run ruff format .
@@ -145,6 +145,7 @@ E2E_DUCKDB_CLI_PATHS := \
 	tests/e2e/scripts/cli_preview \
 	tests/e2e/scripts/compile_performance_ratio \
 	tests/e2e/scripts/compiler_differential \
+	tests/e2e/scripts/planner_differential \
 	tests/e2e/scripts/release_performance \
 	tests/e2e/src/sqlbuild/cli/commands/main/adapters \
 	tests/e2e/src/sqlbuild/cli/commands/main/audit \
@@ -326,6 +327,17 @@ compiler-differential-dense:
 		--corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \
 		--jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS)
 
+# Stateful planner differential (see scripts/planner_differential/README.md): record a
+# baseline on the tree before a planner change, then check the changed tree against it.
+PLANNER_DIFFERENTIAL_MODE ?= check
+PLANNER_DIFFERENTIAL_JOBS ?= 2
+PLANNER_DIFFERENTIAL_SEEDS ?= 6
+PLANNER_DIFFERENTIAL_ARGS ?=
+planner-differential:
+	env PYTHONUNBUFFERED=1 uv run python -m scripts.run_planner_differential \
+		$(PLANNER_DIFFERENTIAL_MODE) --jobs $(PLANNER_DIFFERENTIAL_JOBS) \
+		--seeds $(PLANNER_DIFFERENTIAL_SEEDS) $(PLANNER_DIFFERENTIAL_ARGS)
+
 
 skills:
 	uv run python -m scripts.generate_docs_skill

crates/sqlbuild-analysis/src/assembly/project/constants.rs

@@ -47,3 +47,16 @@ pub(crate) const AUDIT_TARGET_KIND_ENUM: &str = "AttachedAuditTargetKind";
 pub(crate) const ENVIRONMENT_READ_PREFIX: &str = "ENV:";
 /// Characters that end an `ENV:` name besides Python whitespace.
 pub(crate) const ENVIRONMENT_NAME_TERMINATORS: [char; 6] = ['(', ')', ',', '\'', '"', '}'];
+/// The MODEL header opt-out help of SQL the parser rejects.
+pub(crate) const MODEL_OPT_OUT_HELP: &str = "if this SQL is valid for your warehouse, skip SQL \
+    analysis for this model, add this to the MODEL header:\n            MODEL (\n              \
+    sql_analysis false,\n              ...\n            );\n          and please report the SQL \
+    so the parser can support it";
+/// The report-only help of rejected SQL that no model header owns.
+pub(crate) const REPORT_ONLY_HELP: &str =
+    "if this SQL is valid for your warehouse, please report it so the parser can support it";
+/// The project-wide and single-run opt-out helps, each after its own help label.
+pub(crate) const PROJECT_AND_RUN_OPT_OUT_HELP: &str = "\n  = help: SQL analysis is on for this \
+    project; to turn it off for every model, set this in sqlbuild_project.toml:\n            \
+    [settings]\n            sql_analysis = false\n  = help: to skip SQL analysis for one run, \
+    use `--no-sql-analysis`";

crates/sqlbuild-analysis/src/assembly/project/main/mod.rs

@@ -3,4 +3,5 @@ pub mod check_sql_syntax;
 pub mod configured_namespace;
 pub mod environment_template_names;
 pub mod preserved_logical_namespace;
+pub mod sql_analysis_opt_out_help;
 pub mod sql_syntax_error;

crates/sqlbuild-analysis/src/assembly/project/main/sql_analysis_opt_out_help.rs

@@ -0,0 +1,16 @@
+//! Help for SQL the parser rejects: the exact header, project and single-run opt-outs.
+
+use crate::assembly::project::constants::{
+    MODEL_OPT_OUT_HELP, PROJECT_AND_RUN_OPT_OUT_HELP, REPORT_ONLY_HELP,
+};
+
+/// The opt-out help; only SQL a model owns offers the MODEL header entry.
+#[must_use]
+pub fn sql_analysis_opt_out_help(model_owned: bool) -> String {
+    let first: &str = if model_owned {
+        MODEL_OPT_OUT_HELP
+    } else {
+        REPORT_ONLY_HELP
+    };
+    format!("{first}{PROJECT_AND_RUN_OPT_OUT_HELP}")
+}

crates/sqlbuild-analysis/src/assembly/project/tests.rs

@@ -4,6 +4,8 @@ mod configured_namespace;
 mod environment_names;
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/test_opt_out_help.rs"]
+mod opt_out_help;
 #[path = "tests/test_project_assembly.rs"]
 mod project_assembly;
 #[path = "tests/test_types.rs"]

crates/sqlbuild-analysis/src/assembly/project/tests/test_opt_out_help.rs

@@ -0,0 +1,34 @@
+use crate::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help;
+use crate::assembly::project::tests::test_types::OptOutHelpTestCase;
+
+#[test]
+fn given_ownership_when_rendering_opt_out_help_then_matches_python_help() {
+    let test_cases = [
+        OptOutHelpTestCase {
+            description: "model-owned SQL offers the MODEL header entry",
+            model_owned: true,
+            expected_help: "if this SQL is valid for your warehouse, skip SQL analysis for this \
+                model, add this to the MODEL header:\n            MODEL (\n              \
+                sql_analysis false,\n              ...\n            );\n          and please \
+                report the SQL so the parser can support it\n  = help: SQL analysis is on for \
+                this project; to turn it off for every model, set this in \
+                sqlbuild_project.toml:\n            [settings]\n            sql_analysis = \
+                false\n  = help: to skip SQL analysis for one run, use `--no-sql-analysis`",
+        },
+        OptOutHelpTestCase {
+            description: "other SQL only asks for a report",
+            model_owned: false,
+            expected_help: "if this SQL is valid for your warehouse, please report it so the \
+                parser can support it\n  = help: SQL analysis is on for this project; to turn \
+                it off for every model, set this in sqlbuild_project.toml:\n            \
+                [settings]\n            sql_analysis = false\n  = help: to skip SQL analysis \
+                for one run, use `--no-sql-analysis`",
+        },
+    ];
+
+    for test_case in test_cases {
+        let help: String = sql_analysis_opt_out_help(test_case.model_owned);
+
+        assert_eq!(help, test_case.expected_help, "{}", test_case.description);
+    }
+}

crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs

@@ -92,3 +92,9 @@ pub(crate) struct ConfiguredNamespaceTestCase {
     pub(crate) value: &'static str,
     pub(crate) expected_namespace: Option<&'static str>,
 }
+
+pub(crate) struct OptOutHelpTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) model_owned: bool,
+    pub(crate) expected_help: &'static str,
+}

crates/sqlbuild-analysis/src/compiled_project/main/map_expanded_offset.rs

@@ -0,0 +1,10 @@
+//! Resolve an offset in expanded SQL back onto the authored body.
+
+use crate::compiled_project::_helpers::locating::map_through_passes;
+use crate::compiled_project::models::SplicedSpan;
+
+/// The offset in the first pass's source, and whether an expansion generated it.
+#[must_use]
+pub fn map_expanded_offset(offset: usize, passes: &[Vec<SplicedSpan>]) -> (usize, bool) {
+    map_through_passes(offset, passes)
+}

crates/sqlbuild-analysis/src/compiled_project/main/mod.rs

@@ -5,6 +5,7 @@ pub mod declaration_error_message;
 pub mod enum_member_literal;
 pub mod header_metadata_keys;
 pub mod located_scan_error;
+pub mod map_expanded_offset;
 pub mod merge_call_site_references;
 pub mod model_facts;
 pub mod opt_out_position;

crates/sqlbuild-analysis/src/lib.rs

@@ -19,6 +19,8 @@ pub mod lineage;
 pub mod lineage_command;
 pub mod lineage_graph;
 pub mod manifest;
+pub mod planner_changes;
+pub mod planner_strategy;
 pub mod python_plan;
 pub mod python_selection;
 pub mod python_values;

crates/sqlbuild-analysis/src/planner_changes/_helpers/function_hashes.rs

@@ -0,0 +1,47 @@
+//! Function hashes and identity payload equality of fingerprint metadata.
+
+use serde_json::Value;
+
+use crate::planner_changes::_helpers::json_payload::{decoded_object, python_equal};
+use crate::planner_changes::constants::LOCAL_FUNCTION_HASHES_METADATA_KEY;
+use crate::planner_changes::models::IdentityPayload;
+
+/// Python equality of two identity payloads.
+pub(crate) fn payloads_equal(left: &IdentityPayload, right: &IdentityPayload) -> bool {
+    match (left, right) {
+        (IdentityPayload::Object(left), IdentityPayload::Object(right)) => {
+            python_equal(&Value::Object(left.clone()), &Value::Object(right.clone()))
+        }
+        _ => left == right,
+    }
+}
+
+/// Directly called functions whose recorded hash changed, sorted.
+pub(crate) fn changed_local_function_names(
+    metadata_json: &str,
+    previous_metadata_json: &str,
+) -> Vec<String> {
+    let previous: Vec<(String, Value)> = local_function_hashes(previous_metadata_json);
+    let mut changed: Vec<String> = Vec::new();
+    for (name, hash) in local_function_hashes(metadata_json) {
+        let recorded: Option<&Value> = previous
+            .iter()
+            .find(|(previous_name, _)| *previous_name == name)
+            .map(|(_, previous_hash)| previous_hash);
+        if recorded.is_some_and(|recorded| !python_equal(recorded, &hash)) {
+            changed.push(name);
+        }
+    }
+    changed.sort();
+    changed
+}
+
+fn local_function_hashes(metadata_json: &str) -> Vec<(String, Value)> {
+    match decoded_object(metadata_json) {
+        Ok(Some(object)) => match object.get(LOCAL_FUNCTION_HASHES_METADATA_KEY) {
+            Some(Value::Object(hashes)) => hashes.clone().into_iter().collect(),
+            _ => Vec::new(),
+        },
+        _ => Vec::new(),
+    }
+}

crates/sqlbuild-analysis/src/planner_changes/_helpers/json_payload.rs

@@ -0,0 +1,80 @@
+//! Decode fingerprint metadata as Python's `json.loads` does and write it as `json.dumps(sort_keys=True, separators=(",", ":"))`.
+
+use serde_json::{Map, Value};
+use sqlbuild_core::json::main::dumps::dumps;
+use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions};
+
+/// The decoded top-level object, or None when the text is not a JSON object.
+pub(crate) fn decoded_object(metadata_json: &str) -> Result<Option<Map<String, Value>>, ()> {
+    match serde_json::from_str::<Value>(metadata_json) {
+        Ok(Value::Object(object)) => Ok(Some(object)),
+        Ok(_) => Ok(None),
+        Err(_) => Err(()),
+    }
+}
+
+/// `json.dumps(object, sort_keys=True, separators=(",", ":"), default=str)`.
+pub(crate) fn canonical_text(object: Map<String, Value>) -> String {
+    let dialect: JsonDialect = JsonDialect::Stdlib(StdlibJsonOptions {
+        sort_keys: true,
+        ..StdlibJsonOptions::new(None).with_separators(",", ":")
+    });
+    dumps(&json_value(Value::Object(object)), &dialect).unwrap_or_default()
+}
+
+/// Python equality of two decoded JSON values: numbers by value and booleans as 0 and 1.
+pub(crate) fn python_equal(left: &Value, right: &Value) -> bool {
+    match (left, right) {
+        (Value::Array(left), Value::Array(right)) => {
+            left.len() == right.len()
+                && left
+                    .iter()
+                    .zip(right)
+                    .all(|(left, right)| python_equal(left, right))
+        }
+        (Value::Object(left), Value::Object(right)) => {
+            left.len() == right.len()
+                && left.iter().all(|(key, value)| {
+                    right
+                        .get(key)
+                        .is_some_and(|other| python_equal(value, other))
+                })
+        }
+        (Value::Number(left_number), Value::Number(right_number))
+            if !left_number.is_f64() && !right_number.is_f64() =>
+        {
+            left_number == right_number
+        }
+        (Value::Number(_) | Value::Bool(_), Value::Number(_) | Value::Bool(_)) => {
+            numeric(left) == numeric(right)
+        }
+        _ => left == right,
+    }
+}
+
+fn numeric(value: &Value) -> Option<f64> {
+    match value {
+        Value::Bool(flag) => Some(f64::from(u8::from(*flag))),
+        Value::Number(number) => number.as_f64(),
+        _ => None,
+    }
+}
+
+fn json_value(value: Value) -> JsonValue {
+    match value {
+        Value::Null => JsonValue::Null,
+        Value::Bool(flag) => JsonValue::Bool(flag),
+        Value::Number(number) => match JsonInteger::parse(&number.to_string()) {
+            Some(integer) if !number.is_f64() => JsonValue::Integer(integer),
+            _ => JsonValue::Float(number.as_f64().unwrap_or_default()),
+        },
+        Value::String(text) => JsonValue::String(text),
+        Value::Array(items) => JsonValue::Array(items.into_iter().map(json_value).collect()),
+        Value::Object(object) => JsonValue::Object(
+            object
+                .into_iter()
+                .map(|(key, value)| (key, json_value(value)))
+                .collect(),
+        ),
+    }
+}

crates/sqlbuild-analysis/src/planner_changes/_helpers/mod.rs

@@ -0,0 +1,6 @@
+pub(crate) mod function_hashes;
+pub(crate) mod json_payload;
+pub(crate) mod pick_more_aggressive;
+pub(crate) mod rename_mappings;
+pub(crate) mod schema_comparer;
+pub(crate) mod with_origin_cursor_input_names;

crates/sqlbuild-analysis/src/planner_changes/_helpers/pick_more_aggressive.rs

@@ -0,0 +1,20 @@
+//! Choose between two backfills.
+
+use crate::planner_changes::models::{Backfill, BackfillAction};
+
+/// The more aggressive backfill (full over bounded over forward only); a tie keeps `first`.
+pub(crate) fn pick_more_aggressive(first: Backfill, second: Backfill) -> Backfill {
+    if priority(first.action) >= priority(second.action) {
+        first
+    } else {
+        second
+    }
+}
+
+fn priority(action: BackfillAction) -> u8 {
+    match action {
+        BackfillAction::ForwardOnly => 0,
+        BackfillAction::Bounded => 1,
+        BackfillAction::Full => 2,
+    }
+}

crates/sqlbuild-analysis/src/planner_changes/_helpers/rename_mappings.rs

@@ -0,0 +1,63 @@
+//! Candidate rename mappings and their substituted queries.
+
+use crate::planner_changes::constants::MAX_CANDIDATE_MAPPINGS;
+use crate::planner_changes::models::NameSpan;
+
+/// The name followed by every origin it was renamed from, breadth first.
+pub(crate) fn lineage(name: &str, origins: &[(String, Vec<String>)]) -> Vec<String> {
+    let mut seen: Vec<String> = vec![name.to_owned()];
+    let mut index: usize = 0;
+    while index < seen.len() {
+        let current: String = seen[index].clone();
+        if let Some((_, list)) = origins.iter().find(|(key, _)| *key == current) {
+            for origin in list {
+                if !seen.contains(origin) {
+                    seen.push(origin.clone());
+                }
+            }
+        }
+        index += 1;
+    }
+    seen
+}
+
+/// `itertools.product(*choices)`, truncated to the first mappings tried.
+pub(crate) fn product(choices: &[Vec<String>]) -> Vec<Vec<String>> {
+    let mut combinations: Vec<Vec<String>> = vec![Vec::new()];
+    for options in choices {
+        let mut next: Vec<Vec<String>> = Vec::new();
+        for prefix in &combinations {
+            for option in options {
+                if next.len() == MAX_CANDIDATE_MAPPINGS {
+                    break;
+                }
+                let mut combination: Vec<String> = prefix.clone();
+                combination.push(option.clone());
+                next.push(combination);
+            }
+        }
+        combinations = next;
+    }
+    combinations
+}
+
+/// The query with each span replaced by its mapped origin name.
+pub(crate) fn substitute(
+    characters: &[char],
+    spans: &[NameSpan],
+    mapping: &[(String, String)],
+) -> String {
+    let mut text: String = String::new();
+    let mut copied_to: usize = 0;
+    for span in spans {
+        text.extend(&characters[copied_to.min(span.start)..span.start]);
+        let replacement: &str = mapping
+            .iter()
+            .find(|(name, _)| *name == span.name)
+            .map_or(span.name.as_str(), |(_, origin)| origin.as_str());
+        text.push_str(replacement);
+        copied_to = span.end;
+    }
+    text.extend(&characters[copied_to.min(characters.len())..]);
+    text
+}