refactor: move runtime capture, history and orchestration logic to Rust

crates/README.md

@@ -8,7 +8,7 @@ that can be tested and benchmarked with `cargo test` alone.
 |---|---|
 | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. |
 | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. |
-| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. |
+| `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. It also owns execution history: lifecycle event batch digests and conflicts, run projection, event and run cursors, filter and page validation, and the SQLite and PostgreSQL statements the native history backends run. |
 | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. |
 | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. |
 | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. |

crates/sqlbuild-attachments/src/audits/constants.rs

@@ -13,3 +13,22 @@ pub(crate) const FUNCTION_RESOURCE: &str = "function";
 pub(crate) const CROSS_MODEL_MINIMUM: usize = 2;
 /// The run scope Python falls back to without an instance or project value.
 pub(crate) const FALLBACK_RUN_SCOPE: &str = "delta_and_final";
+/// Built-in generic audit names and SQL bodies, which project definitions override by name.
+pub const BUILTIN_AUDITS: [(&str, &str); 4] = [
+    (
+        "accepted_values",
+        "SELECT @column\nFROM @relation\nWHERE @column IS NOT NULL\n  AND @column NOT IN (@'values')",
+    ),
+    (
+        "not_null",
+        "SELECT @column\nFROM @relation\nWHERE @column IS NULL",
+    ),
+    (
+        "relationships",
+        "SELECT @column\nFROM @relation\nWHERE @column IS NOT NULL\n  AND @column NOT IN (\n    SELECT @field\n    FROM @to\n    WHERE @field IS NOT NULL\n  )",
+    ),
+    (
+        "unique",
+        "SELECT @column, COUNT(*) AS duplicate_count\nFROM @relation\nWHERE @column IS NOT NULL\nGROUP BY @column\nHAVING COUNT(*) > 1",
+    ),
+];

crates/sqlbuild-attachments/src/audits/mod.rs

@@ -1,5 +1,5 @@
 pub(crate) mod _helpers;
-pub(crate) mod constants;
+pub mod constants;
 pub mod main;
 pub mod models;
 

crates/sqlbuild-cache/Cargo.toml

@@ -10,6 +10,7 @@ rust-version.workspace = true
 [dependencies]
 blake3.workspace = true
 rayon.workspace = true
+serde_json.workspace = true
 sha2.workspace = true
 sqlbuild-core.workspace = true
 tempfile.workspace = true

crates/sqlbuild-cache/src/digest/_helpers/distributions.rs

@@ -0,0 +1,124 @@
+//! `importlib.metadata` distribution lookup over search paths, for code identity facts.
+
+use std::io::ErrorKind;
+use std::path::{Path, PathBuf};
+
+use crate::digest::constants::{METADATA_DIRECTORY_SUFFIXES, METADATA_FILES};
+
+/// The first metadata folder of `name` along `search_paths`, as `importlib.metadata` finds it.
+pub(crate) fn distribution_directory(search_paths: &[PathBuf], name: &str) -> Option<PathBuf> {
+    let wanted: String = normalized(name);
+    for root in search_paths {
+        let directory: &Path = if root.as_os_str().is_empty() {
+            Path::new(".")
+        } else {
+            root
+        };
+        if let Some(child) = children(directory)
+            .into_iter()
+            .find(|child| metadata_name(child).is_some_and(|found| normalized(&found) == wanted))
+        {
+            return Some(root.join(child));
+        }
+    }
+    None
+}
+
+/// The `Version` header of a distribution's metadata, when it has one.
+pub(crate) fn distribution_version(directory: &Path) -> Option<String> {
+    for file in METADATA_FILES {
+        if let Some(text) = optional_text(&directory.join(file)) {
+            return header_value(&text, "version");
+        }
+    }
+    None
+}
+
+/// A file's bytes, or `None` where `importlib.metadata` treats the file as absent.
+pub(crate) fn optional_bytes(path: &Path) -> Option<Vec<u8>> {
+    match std::fs::read(path) {
+        Ok(bytes) => Some(bytes),
+        Err(error) if absent(&error) => None,
+        Err(_) => None,
+    }
+}
+
+/// Entry names of a search path folder; folders Python cannot list have none.
+fn children(directory: &Path) -> Vec<String> {
+    let entries: std::fs::ReadDir = match std::fs::read_dir(directory) {
+        Ok(entries) => entries,
+        Err(error) if absent(&error) => return Vec::new(),
+        Err(_) => return Vec::new(),
+    };
+    let mut names: Vec<String> = Vec::new();
+    for entry in entries.flatten() {
+        names.push(entry.file_name().to_string_lossy().into_owned());
+    }
+    names
+}
+
+fn optional_text(path: &Path) -> Option<String> {
+    optional_bytes(path).map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
+}
+
+fn absent(error: &std::io::Error) -> bool {
+    matches!(
+        error.kind(),
+        ErrorKind::NotFound | ErrorKind::NotADirectory | ErrorKind::IsADirectory
+    )
+}
+
+/// The distribution name a metadata folder entry names, before its version.
+fn metadata_name(child: &str) -> Option<String> {
+    let lowered: String = child.to_lowercase();
+    METADATA_DIRECTORY_SUFFIXES
+        .iter()
+        .any(|suffix| lowered.ends_with(suffix))
+        .then(|| {
+            let stem: &str = lowered.rsplit_once('.').map_or("", |(stem, _)| stem);
+            stem.split('-').next().unwrap_or_default().to_owned()
+        })
+}
+
+/// `re.sub(r"[-_.]+", "-", name).lower().replace("-", "_")`.
+fn normalized(name: &str) -> String {
+    let mut text: String = String::with_capacity(name.len());
+    let mut separator: bool = false;
+    for character in name.chars() {
+        if matches!(character, '-' | '_' | '.') {
+            if !separator {
+                text.push('_');
+            }
+            separator = true;
+        } else {
+            text.extend(character.to_lowercase());
+            separator = false;
+        }
+    }
+    text
+}
+
+/// The first value of a case-insensitive header in an RFC 822 style header block.
+fn header_value(text: &str, name: &str) -> Option<String> {
+    let mut value: Option<String> = None;
+    for line in text.lines() {
+        if line.is_empty() {
+            break;
+        }
+        if line.starts_with([' ', '\t']) {
+            if let Some(current) = value.as_mut() {
+                current.push_str(line);
+            }
+            continue;
+        }
+        if value.is_some() {
+            break;
+        }
+        if let Some((header, rest)) = line.split_once(':')
+            && header.eq_ignore_ascii_case(name)
+        {
+            value = Some(rest.trim_start().to_owned());
+        }
+    }
+    value.map(|text| text.trim_end().to_owned())
+}

crates/sqlbuild-cache/src/digest/_helpers/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod code_files;
+pub(crate) mod distributions;
 pub(crate) mod walk;

crates/sqlbuild-cache/src/digest/constants.rs

@@ -35,3 +35,16 @@ pub const PRESENCE_TAG: &[u8] = b"presence";
 pub(crate) const CODE_SOURCE_SUFFIXES: &[&str] = &["py", "so", "pyd"];
 /// The bytecode folder an editable install's code walk skips.
 pub(crate) const BYTECODE_CACHE_DIRECTORY: &str = "__pycache__";
+
+/// Distributions whose installed versions identify the code behind cached results.
+pub const CODE_IDENTITY_DISTRIBUTIONS: [&str; 3] = ["sqlbuild", "polyglot-sql-chio", "PyYAML"];
+/// The distribution an editable install's `direct_url.json` describes.
+pub(crate) const SQLBUILD_DISTRIBUTION: &str = "sqlbuild";
+/// The version reported for a distribution no search path holds.
+pub(crate) const UNKNOWN_VERSION: &str = "unknown";
+/// Metadata folder suffixes `importlib.metadata` searches, lowercased.
+pub(crate) const METADATA_DIRECTORY_SUFFIXES: [&str; 2] = [".dist-info", ".egg-info"];
+/// Metadata files read for a distribution's version, in order.
+pub(crate) const METADATA_FILES: [&str; 2] = ["METADATA", "PKG-INFO"];
+/// The installer's record of where a distribution was installed from.
+pub(crate) const DIRECT_URL_FILE: &str = "direct_url.json";

crates/sqlbuild-cache/src/digest/main/installed_code_facts.rs

@@ -0,0 +1,44 @@
+//! The installed distribution versions and editable flag behind the installed code identity.
+
+use std::path::PathBuf;
+
+use crate::digest::_helpers::distributions::{
+    distribution_directory, distribution_version, optional_bytes,
+};
+use crate::digest::constants::{
+    CODE_IDENTITY_DISTRIBUTIONS, DIRECT_URL_FILE, SQLBUILD_DISTRIBUTION, UNKNOWN_VERSION,
+};
+
+/// Each identity distribution's version along `search_paths`, and whether SQLBuild is editable.
+#[must_use]
+pub fn installed_code_facts(search_paths: &[PathBuf]) -> (Vec<String>, bool) {
+    let versions: Vec<String> = CODE_IDENTITY_DISTRIBUTIONS
+        .iter()
+        .map(|name| {
+            distribution_directory(search_paths, name)
+                .and_then(|directory| distribution_version(&directory))
+                .unwrap_or_else(|| UNKNOWN_VERSION.to_owned())
+        })
+        .collect();
+    let editable: bool =
+        distribution_directory(search_paths, SQLBUILD_DISTRIBUTION).is_none_or(|directory| {
+            optional_bytes(&directory.join(DIRECT_URL_FILE))
+                .is_some_and(|bytes| editable_direct_url(&bytes))
+        });
+    (versions, editable)
+}
+
+/// Whether `direct_url.json` marks an editable install; unreadable payloads count as editable.
+fn editable_direct_url(bytes: &[u8]) -> bool {
+    let Ok(payload) = serde_json::from_slice::<serde_json::Value>(bytes) else {
+        return true;
+    };
+    let Some(payload) = payload.as_object() else {
+        return true;
+    };
+    payload
+        .get("dir_info")
+        .and_then(serde_json::Value::as_object)
+        .and_then(|dir_info| dir_info.get("editable"))
+        .is_some_and(|editable| editable == &serde_json::Value::Bool(true))
+}

crates/sqlbuild-cache/src/digest/main/mod.rs

@@ -4,5 +4,6 @@ pub mod content_digest;
 pub mod digest_files;
 pub mod fingerprint_project_files;
 pub mod hex_digest;
+pub mod installed_code_facts;
 pub mod installed_code_identity;
 pub mod sha256_hex;

crates/sqlbuild-cache/src/digest/tests.rs

@@ -8,5 +8,7 @@ mod digest_files;
 mod fingerprint_project_files;
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/test_installed_code_facts.rs"]
+mod installed_code_facts;
 #[path = "tests/test_types.rs"]
 mod test_types;

crates/sqlbuild-cache/src/digest/tests/test_installed_code_facts.rs

@@ -0,0 +1,88 @@
+use std::path::PathBuf;
+
+use crate::digest::main::installed_code_facts::installed_code_facts;
+use crate::digest::tests::helpers::write_file;
+use crate::digest::tests::test_types::InstalledCodeFactsTestCase;
+
+#[test]
+fn given_search_paths_when_reading_installed_facts_then_importlib_metadata_is_reproduced() {
+    let test_cases = [
+        InstalledCodeFactsTestCase {
+            description: "nothing installed is unknown and editable",
+            files: &[(0, "other.txt", "")],
+            expected_versions: ["unknown", "unknown", "unknown"],
+            expected_editable: true,
+        },
+        InstalledCodeFactsTestCase {
+            description: "the first search path wins and names normalise",
+            files: &[
+                (
+                    0,
+                    "sqlbuild-1.2.0.dist-info/METADATA",
+                    "Name: sqlbuild\nVersion: 1.2.0\n\nbody",
+                ),
+                (1, "sqlbuild-9.0.0.dist-info/METADATA", "Version: 9.0.0\n"),
+                (
+                    1,
+                    "polyglot_sql_chio-0.4.dist-info/METADATA",
+                    "version:  0.4 \n",
+                ),
+                (
+                    1,
+                    "PyYAML-6.0.2.dist-info/METADATA",
+                    "Metadata-Version: 2.1\nVersion: 6.0.2\n",
+                ),
+            ],
+            expected_versions: ["1.2.0", "0.4", "6.0.2"],
+            expected_editable: false,
+        },
+        InstalledCodeFactsTestCase {
+            description: "an editable direct URL marks the install editable",
+            files: &[
+                (0, "sqlbuild-1.0.dist-info/METADATA", "Version: 1.0\n"),
+                (
+                    0,
+                    "sqlbuild-1.0.dist-info/direct_url.json",
+                    "{\"url\": \"file:///src\", \"dir_info\": {\"editable\": true}}",
+                ),
+            ],
+            expected_versions: ["1.0", "unknown", "unknown"],
+            expected_editable: true,
+        },
+        InstalledCodeFactsTestCase {
+            description: "an unreadable direct URL counts as editable",
+            files: &[
+                (0, "sqlbuild-1.0.dist-info/METADATA", "Version: 1.0\n"),
+                (0, "sqlbuild-1.0.dist-info/direct_url.json", "not json"),
+            ],
+            expected_versions: ["1.0", "unknown", "unknown"],
+            expected_editable: true,
+        },
+    ];
+    for test_case in test_cases {
+        let directories = [
+            tempfile::tempdir().expect("a temporary folder"),
+            tempfile::tempdir().expect("a temporary folder"),
+        ];
+        for (index, relative_path, contents) in test_case.files {
+            write_file(directories[*index].path(), relative_path, contents);
+        }
+        let search_paths: Vec<PathBuf> = directories
+            .iter()
+            .map(|directory| directory.path().to_path_buf())
+            .collect();
+
+        let (versions, editable) = installed_code_facts(&search_paths);
+
+        assert_eq!(
+            versions, test_case.expected_versions,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(
+            editable, test_case.expected_editable,
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-cache/src/digest/tests/test_types.rs

@@ -50,3 +50,12 @@ pub(crate) struct CompiledCodeIdentityTestCase {
     pub(crate) build_identity: &'static str,
     pub(crate) expected_identity: &'static str,
 }
+
+/// Search path folders with their files, and the facts `importlib.metadata` reads from them.
+pub(crate) struct InstalledCodeFactsTestCase {
+    pub(crate) description: &'static str,
+    /// `(search path index, relative path, contents)`.
+    pub(crate) files: &'static [(usize, &'static str, &'static str)],
+    pub(crate) expected_versions: [&'static str; 3],
+    pub(crate) expected_editable: bool,
+}