From 334fe373c422ce7658463b5369f4a49feef9fecb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:11:13 +0100 Subject: [PATCH 01/32] refactor: render coded errors and the lineage status natively --- .../src/cli_text/tests/test_cli_text.rs | 81 +++++++++++++- .../src/cli_text/tests/test_types.rs | 8 ++ .../build_progress/transient_status.rs | 33 ++++-- .../src/bindings/_helpers/cli/entry.rs | 37 ++++--- .../_helpers/lineage/lineage_command.rs | 27 ++--- .../presentation/_helpers/error_text.py | 74 ------------- .../presentation/_helpers/progress.py | 12 --- src/sqlbuild/presentation/constants.py | 1 - .../presentation/main/_maybe_status.py | 25 ----- .../main/_progress_spinners_disabled.py | 13 --- .../presentation/main/coded_error_text.py | 24 ----- .../presentation/main/inline_error_lines.py | 27 ----- .../presentation/classes/test_status.py | 51 --------- .../sqlbuild/presentation/main/_test_types.py | 18 ---- .../presentation/main/test_coded_errors.py | 100 ------------------ .../main/test_inline_error_lines.py | 39 ------- 16 files changed, 147 insertions(+), 423 deletions(-) delete mode 100644 src/sqlbuild/presentation/_helpers/error_text.py delete mode 100644 src/sqlbuild/presentation/_helpers/progress.py delete mode 100644 src/sqlbuild/presentation/main/_maybe_status.py delete mode 100644 src/sqlbuild/presentation/main/_progress_spinners_disabled.py delete mode 100644 src/sqlbuild/presentation/main/coded_error_text.py delete mode 100644 src/sqlbuild/presentation/main/inline_error_lines.py delete mode 100644 tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py delete mode 100644 tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index ddac7563c2..3e2632fd59 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -1,4 +1,5 @@ use crate::cli_text::_helpers::diagnostics::diagnostics_text; +use crate::cli_text::main::format_coded_error::format_coded_error; use crate::cli_text::main::format_compile_text::format_compile_text; use crate::cli_text::main::format_dbt_interop_plan::format_dbt_interop_plan; use crate::cli_text::main::format_plan_completion::format_plan_completion; @@ -10,8 +11,8 @@ use crate::cli_text::models::{ }; use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ - CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, - PlanTextTestCase, + CodedErrorTestCase, CompileTextTestCase, DbtPlanTextTestCase, DiagnosticsTextTestCase, + NoticeTestCase, PlanTextTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -232,3 +233,79 @@ fn given_dbt_interop_plan_when_formatting_then_matches_python_text() { ); } } + +/// Expectations recorded from the retired Python coded-error formatter. +#[test] +fn given_coded_error_when_formatting_then_renders_consistently() { + let plain: CliStyle = CliStyle { use_color: false }; + let color: CliStyle = CliStyle { use_color: true }; + let test_cases = [ + CodedErrorTestCase { + description: "renders plain coded error", + parts: ("R002", "materialization failed", None), + include_error_label: true, + style: plain, + expected_rendered: "error[R002]: materialization failed", + }, + CodedErrorTestCase { + description: "renders coded detail beneath an existing error label", + parts: ("T002", "test execution failed", None), + include_error_label: false, + style: plain, + expected_rendered: "[T002] test execution failed", + }, + CodedErrorTestCase { + description: "renders plain coded error with help", + parts: ( + "K011", + "contract requires staged promotion", + Some("set table_promotion_mode to staged"), + ), + include_error_label: true, + style: plain, + expected_rendered: "error[K011]: contract requires staged promotion\n = help: set table_promotion_mode to staged", + }, + CodedErrorTestCase { + description: "renders colorized coded error with help", + parts: ("R006", "audit failed", Some("fix failing audit rows")), + include_error_label: true, + style: color, + expected_rendered: "\x1b[38;5;167m\x1b[1merror[R006]:\x1b[0m audit failed\n \x1b[2m= help:\x1b[0m fix failing audit rows", + }, + CodedErrorTestCase { + description: "renders multiline coded error without color unchanged", + parts: ( + "D012", + "Provider 'slack_provider' in providers/slack.py has invalid settings:\n1 validation error for SlackProvider\nSLACK_TOKEN", + None, + ), + include_error_label: true, + style: plain, + expected_rendered: "error[D012]: Provider 'slack_provider' in providers/slack.py has invalid settings:\n1 validation error for SlackProvider\nSLACK_TOKEN", + }, + CodedErrorTestCase { + description: "renders multiline coded error with muted continuation lines", + parts: ( + "D012", + "Provider 'slack_provider' in providers/slack.py has invalid settings:\n1 validation error for SlackProvider\nSLACK_TOKEN", + None, + ), + include_error_label: true, + style: color, + expected_rendered: "\x1b[38;5;167m\x1b[1merror[D012]:\x1b[0m Provider 'slack_provider' in providers/slack.py has invalid settings:\n\x1b[2m1 validation error for SlackProvider\x1b[0m\n\x1b[2mSLACK_TOKEN\x1b[0m", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_coded_error( + test_case.parts, + test_case.include_error_label, + test_case.style + ), + test_case.expected_rendered, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index ed807492b8..65a373b900 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -32,3 +32,11 @@ pub(super) struct DbtPlanTextTestCase { pub(super) max_entries: Option, pub(super) expected_text: &'static str, } + +pub(super) struct CodedErrorTestCase { + pub(super) description: &'static str, + pub(super) parts: (&'static str, &'static str, Option<&'static str>), + pub(super) include_error_label: bool, + pub(super) style: CliStyle, + pub(super) expected_rendered: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs index bbf3d306bf..c255be2ec5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/transient_status.rs @@ -14,7 +14,8 @@ use crate::bindings::_helpers::build_progress::terminal::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{imported, within}; -const SPINNERS_DISABLED_MODULE: &str = "sqlbuild.presentation.main._progress_spinners_disabled"; +const NO_PROGRESS_VARIABLE: &str = "SQLBUILD_NO_PROGRESS"; +const PROGRESS_DISABLED_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; const PREFLIGHT_COMPLETION_PREFIX: &str = "Prepared "; /// Mutable reporter state; only held briefly and never across Python calls. @@ -149,12 +150,8 @@ impl TransientStatusReporter { enabled: bool, ) -> PyResult { let lines: Bound<'_, PyAny> = shared_lines(py)?; - let spinners_disabled: bool = - imported(py, SPINNERS_DISABLED_MODULE, "progress_spinners_disabled")? - .call0()? - .is_truthy()?; Ok(Self { - enabled: enabled && !spinners_disabled && tty_flag(stream)?, + enabled: enabled && !progress_spinners_disabled() && tty_flag(stream)?, stream: stream.clone().unbind(), lock: lines.getattr("lock")?.unbind(), lines: lines.unbind(), @@ -293,6 +290,30 @@ impl TransientStatusReporter { } } +/// Whether `SQLBUILD_NO_PROGRESS` turns transient spinners off. +pub(crate) fn progress_spinners_disabled() -> bool { + std::env::var(NO_PROGRESS_VARIABLE) + .is_ok_and(|value| PROGRESS_DISABLED_VALUES.contains(&value.trim().to_lowercase().as_str())) +} + +/// Run `body` under a stderr spinner when `enabled` and stderr is an interactive terminal. +pub(crate) fn maybe_status<'py, T>( + py: Python<'py>, + (message, enabled): (&str, bool), + body: impl FnOnce() -> PyResult, +) -> PyResult { + let stderr: Bound<'py, PyAny> = py.import("sys")?.getattr("stderr")?; + if !enabled || progress_spinners_disabled() || !tty_flag(&stderr)? { + return body(); + } + let reporter: Bound<'py, TransientStatusReporter> = + Bound::new(py, TransientStatusReporter::new(py, &stderr, false, true)?)?; + TransientStatusReporter::start(&reporter, message.to_owned())?; + let result: PyResult = body(); + TransientStatusReporter::close(&reporter)?; + result +} + /// A native status reporter writing to `stream`. pub(crate) fn status_reporter<'py>( stream: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index c947ab597f..eec3e3caea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -14,6 +14,8 @@ use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPol use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; +use sqlbuild_core::cli_text::models::CliStyle; const NO_COLOR_OPTION: &str = "--no-color"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; @@ -29,7 +31,6 @@ const INVOCATION_CONTEXT_MODULE: &str = "sqlbuild.runtime.output_capture.main.in const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exceptions"; const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const CODED_ERROR_MODULE: &str = "sqlbuild.presentation.main.coded_error_text"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; @@ -142,19 +143,27 @@ fn print_expected_error( ) -> PyResult<()> { let py: Python<'_> = invocation.py(); let (code, message, help) = error_parts(error, fallback_code)?; - let text: Bound<'_, PyAny> = call_with( - &imported(py, CODED_ERROR_MODULE, "format_coded_error")?, - &[ - ("code", code), - ("message", message), - ("help", help), - ( - "use_color", - PyBool::new(py, invocation.use_color).to_owned().into_any(), - ), - ], - )?; - print_object(&text.add("\n")?, Some("stderr"), None) + let help: Option = if help.is_none() { + None + } else { + Some(help.str()?.extract()?) + }; + let text: String = format_coded_error( + ( + &code.extract::()?, + &message.extract::()?, + help.as_deref(), + ), + true, + CliStyle { + use_color: invocation.use_color, + }, + ); + print_object( + &PyString::new(py, &format!("{text}\n")).into_any(), + Some("stderr"), + None, + ) } /// The compile JSON error document `sqb compile --json` prints for a failure. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 0296e7af8b..fe1361a9e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -31,6 +31,7 @@ use sqlbuild_analysis::lineage_command::models::{ }; use sqlbuild_core::cli_text::models::CliStyle; +use crate::bindings::_helpers::build_progress::transient_status::maybe_status; use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; @@ -39,7 +40,6 @@ use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const STATUS_MODULE: &str = "sqlbuild.presentation.main._maybe_status"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; @@ -384,7 +384,7 @@ fn traced_edges( Ok(result) } -/// `with maybe_status(...)` around building column lineage. +/// Build column lineage under a stderr spinner for large rich-mode requests. fn column_lineage<'py>( py: Python<'py>, (graph, mode): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -392,26 +392,19 @@ fn column_lineage<'py>( ) -> PyResult> { let count: usize = candidates.model_names.len(); let rich: bool = mode.str()?.extract::()? == RICH_MODE; - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item( - "message", - format!("Analyzing column lineage for {count} models..."), - )?; - options.set_item("enabled", rich && count >= RICH_STATUS_MODEL_THRESHOLD)?; - let status: Bound<'py, PyAny> = - imported(py, STATUS_MODULE, "maybe_status")?.call((), Some(&options))?; - status.call_method0("__enter__")?; - let built: PyResult> = - imported(py, NATIVE_MODULE, "build_project_column_lineage").and_then(|build| { - build.call1(( + let message: String = format!("Analyzing column lineage for {count} models..."); + maybe_status( + py, + (&message, rich && count >= RICH_STATUS_MODEL_THRESHOLD), + || { + imported(py, NATIVE_MODULE, "build_project_column_lineage")?.call1(( graph.getattr("project")?, py.None(), mode, PyFrozenSet::new(py, &candidates.model_names)?, )) - }); - status.call_method1("__exit__", (py.None(), py.None(), py.None()))?; - built + }, + ) } /// The column trace of a `model.column` target, or `None` when the target is not a model column. diff --git a/src/sqlbuild/presentation/_helpers/error_text.py b/src/sqlbuild/presentation/_helpers/error_text.py deleted file mode 100644 index 9e5e945b1a..0000000000 --- a/src/sqlbuild/presentation/_helpers/error_text.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Coded-error rendering implementation.""" - -from __future__ import annotations - -import textwrap - -from sqlbuild.presentation.classes.cli_style import CliStyle - -_MAX_INLINE_ERROR_LINES: int = 2 - - -def format_coded_error( - *, - code: str, - message: str, - help: str | None = None, - use_color: bool = False, - include_error_label: bool = True, -) -> str: - style: CliStyle = CliStyle(use_color=use_color) - prefix_text: str = f"error[{code}]:" if include_error_label else f"[{code}]" - prefix: str = style.error_strong(prefix_text) - lines: list[str] = message.split("\n") - rendered_message: str = message - if len(lines) > 1: - rendered_message = "\n".join([lines[0], *[style.muted(line) for line in lines[1:]]]) - rendered: str = f"{prefix} {rendered_message}" - if help is not None: - help_label: str = style.muted("= help:") - rendered = f"{rendered}\n {help_label} {help}" - return rendered - - -def format_inline_error_lines( - *, - error_code: str | None, - error_message: str, - error_help: str | None, - content_width: int, - style: CliStyle, -) -> list[str]: - """Wrap a compact inline diagnostic while preserving semantic code styling.""" - - code_prefix: str = f"[{error_code}]" if error_code is not None else "" - message: str = f"{code_prefix} {error_message}" if code_prefix else error_message - if error_help is not None: - message = f"{message}\n= help: {error_help}" - wrapped_lines: list[str] = [] - paragraph: str - for paragraph in message.splitlines() or [message]: - wrapped_lines.extend( - textwrap.wrap( - paragraph, - width=content_width, - break_long_words=True, - break_on_hyphens=False, - ) - or [""] - ) - truncated: bool = len(wrapped_lines) > _MAX_INLINE_ERROR_LINES - visible_lines: list[str] = wrapped_lines[:_MAX_INLINE_ERROR_LINES] - if truncated: - final_line: str = visible_lines[-1] - ellipsis: str = "." * min(3, content_width) - visible_lines[-1] = ( - f"{final_line[: max(0, content_width - len(ellipsis))].rstrip()}{ellipsis}" - ) - if code_prefix and visible_lines: - visible_lines[0] = visible_lines[0].replace( - code_prefix, - style.error_strong(code_prefix), - 1, - ) - return visible_lines diff --git a/src/sqlbuild/presentation/_helpers/progress.py b/src/sqlbuild/presentation/_helpers/progress.py deleted file mode 100644 index 948369c498..0000000000 --- a/src/sqlbuild/presentation/_helpers/progress.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Progress environment implementations.""" - -from __future__ import annotations - -import os - -from sqlbuild.presentation.constants import PROGRESS_DISABLED_VALUES - - -def progress_spinners_disabled() -> bool: - value: str | None = os.environ.get("SQLBUILD_NO_PROGRESS") - return value is not None and value.strip().lower() in PROGRESS_DISABLED_VALUES diff --git a/src/sqlbuild/presentation/constants.py b/src/sqlbuild/presentation/constants.py index c79c8b6d41..a06117ca75 100644 --- a/src/sqlbuild/presentation/constants.py +++ b/src/sqlbuild/presentation/constants.py @@ -6,7 +6,6 @@ ERROR_STATUS_LABELS: frozenset[str] = frozenset({"ERROR", "FAIL", "FAILED"}) PHASE_FAIL_GLYPH: str = "\u2717" PHASE_OK_GLYPH: str = "\u2713" -PROGRESS_DISABLED_VALUES: frozenset[str] = frozenset({"1", "on", "true", "yes"}) SKIPPED_STATUS_LABELS: frozenset[str] = frozenset({"SKIP", "SKIPPED"}) SUCCESS_STATUS_LABELS: frozenset[str] = frozenset({"OK", "PASS", "SUCCESS"}) TREE_BRANCH_GLYPH: str = "\u251c\u2500\u2500" diff --git a/src/sqlbuild/presentation/main/_maybe_status.py b/src/sqlbuild/presentation/main/_maybe_status.py deleted file mode 100644 index d9cb7214cd..0000000000 --- a/src/sqlbuild/presentation/main/_maybe_status.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public transient stderr status context entry.""" - -from __future__ import annotations - -import sys -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild._native import TransientStatusReporter -from sqlbuild.presentation._helpers.progress import progress_spinners_disabled - - -@contextmanager -def maybe_status(*, message: str, enabled: bool) -> Iterator[None]: - """Render a stderr spinner for long-running interactive operations.""" - - if not enabled or progress_spinners_disabled() or not sys.stderr.isatty(): - yield - return - reporter: TransientStatusReporter = TransientStatusReporter(stream=sys.stderr, enabled=enabled) - reporter.start(message) - try: - yield - finally: - reporter.close() diff --git a/src/sqlbuild/presentation/main/_progress_spinners_disabled.py b/src/sqlbuild/presentation/main/_progress_spinners_disabled.py deleted file mode 100644 index 06ba947e68..0000000000 --- a/src/sqlbuild/presentation/main/_progress_spinners_disabled.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public progress spinner environment check entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.progress import ( - progress_spinners_disabled as _progress_spinners_disabled, -) - - -def progress_spinners_disabled() -> bool: - """Return whether transient spinners are disabled via the environment.""" - - return _progress_spinners_disabled() diff --git a/src/sqlbuild/presentation/main/coded_error_text.py b/src/sqlbuild/presentation/main/coded_error_text.py deleted file mode 100644 index 6fff672e59..0000000000 --- a/src/sqlbuild/presentation/main/coded_error_text.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public coded-error formatting entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.error_text import format_coded_error as _format_coded_error - - -def format_coded_error( - *, - code: str, - message: str, - help: str | None = None, - use_color: bool = False, - include_error_label: bool = True, -) -> str: - """Render a result-stored coded error consistently with CLI expected errors.""" - - return _format_coded_error( - code=code, - message=message, - help=help, - use_color=use_color, - include_error_label=include_error_label, - ) diff --git a/src/sqlbuild/presentation/main/inline_error_lines.py b/src/sqlbuild/presentation/main/inline_error_lines.py deleted file mode 100644 index 7735e70da9..0000000000 --- a/src/sqlbuild/presentation/main/inline_error_lines.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public inline error-line formatting operation.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.error_text import ( - format_inline_error_lines as _format_inline_error_lines, -) -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_inline_error_lines( - *, - error_code: str | None, - error_message: str, - error_help: str | None, - content_width: int, - style: CliStyle, -) -> list[str]: - """Wrap an inline error to two display lines before truncating.""" - - return _format_inline_error_lines( - error_code=error_code, - error_message=error_message, - error_help=error_help, - content_width=content_width, - style=style, - ) diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index 9f0c1e5db5..a9051dc84f 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -7,7 +7,6 @@ import pytest from sqlbuild._native import TransientStatusReporter -from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from tests.unit.src.sqlbuild.presentation.classes._test_types import ( ProgressSpinnersDisabledTestCase, StatusReporterExitTestCase, @@ -28,56 +27,6 @@ def isatty(self) -> bool: return True -@pytest.mark.parametrize( - "test_case", - ( - ProgressSpinnersDisabledTestCase( - description="unset env keeps spinners enabled", - env_value=None, - expected_disabled=False, - ), - ProgressSpinnersDisabledTestCase( - description="truthy '1' disables spinners", - env_value="1", - expected_disabled=True, - ), - ProgressSpinnersDisabledTestCase( - description="truthy 'true' is case-insensitive and trimmed", - env_value=" TRUE ", - expected_disabled=True, - ), - ProgressSpinnersDisabledTestCase( - description="truthy 'on' disables spinners", - env_value="on", - expected_disabled=True, - ), - ProgressSpinnersDisabledTestCase( - description="falsey '0' keeps spinners enabled", - env_value="0", - expected_disabled=False, - ), - ProgressSpinnersDisabledTestCase( - description="arbitrary value keeps spinners enabled", - env_value="maybe", - expected_disabled=False, - ), - ProgressSpinnersDisabledTestCase( - description="empty value keeps spinners enabled", - env_value="", - expected_disabled=False, - ), - ), - ids=lambda case: case.description, -) -def test_given_no_progress_env_when_checking_then_reports_expected_disabled_state( - test_case: ProgressSpinnersDisabledTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - apply_no_progress_env(monkeypatch=monkeypatch, env_value=test_case.env_value) - - assert progress_spinners_disabled() == test_case.expected_disabled - - @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/presentation/main/_test_types.py b/tests/unit/src/sqlbuild/presentation/main/_test_types.py index 635b17ba07..1851522168 100644 --- a/tests/unit/src/sqlbuild/presentation/main/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/main/_test_types.py @@ -1,17 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class FormatCodedErrorTestCase: - description: str - code: str - message: str - help: str | None - use_color: bool - expected_rendered: str - include_error_label: bool = True - - @dataclass(frozen=True) class SummaryFooterTestCase: description: str @@ -21,13 +10,6 @@ class SummaryFooterTestCase: expected_color_fragments: tuple[str, ...] -@dataclass(frozen=True) -class InlineErrorLinesTestCase: - description: str - content_width: int - expected_lines: list[str] - - @dataclass(frozen=True) class CountHeaderTestCase: description: str diff --git a/tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py b/tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py deleted file mode 100644 index 56cc1aea2d..0000000000 --- a/tests/unit/src/sqlbuild/presentation/main/test_coded_errors.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Tests for coded error rendering.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.presentation.main.coded_error_text import format_coded_error -from tests.unit.src.sqlbuild.presentation.main._test_types import FormatCodedErrorTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - FormatCodedErrorTestCase( - description="renders plain coded error", - code="R002", - message="materialization failed", - help=None, - use_color=False, - expected_rendered="error[R002]: materialization failed", - ), - FormatCodedErrorTestCase( - description="renders coded detail beneath an existing error label", - code="T002", - message="test execution failed", - help=None, - use_color=False, - expected_rendered="[T002] test execution failed", - include_error_label=False, - ), - FormatCodedErrorTestCase( - description="renders plain coded error with help", - code="K011", - message="contract requires staged promotion", - help="set table_promotion_mode to staged", - use_color=False, - expected_rendered=( - "error[K011]: contract requires staged promotion\n" - " = help: set table_promotion_mode to staged" - ), - ), - FormatCodedErrorTestCase( - description="renders colorized coded error with help", - code="R006", - message="audit failed", - help="fix failing audit rows", - use_color=True, - expected_rendered=( - "\033[38;5;167m\033[1merror[R006]:\033[0m audit failed\n" - " \033[2m= help:\033[0m fix failing audit rows" - ), - ), - FormatCodedErrorTestCase( - description="renders multiline coded error without color unchanged", - code="D012", - message=( - "Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "1 validation error for SlackProvider\n" - "SLACK_TOKEN" - ), - help=None, - use_color=False, - expected_rendered=( - "error[D012]: Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "1 validation error for SlackProvider\n" - "SLACK_TOKEN" - ), - ), - FormatCodedErrorTestCase( - description="renders multiline coded error with muted continuation lines", - code="D012", - message=( - "Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "1 validation error for SlackProvider\n" - "SLACK_TOKEN" - ), - help=None, - use_color=True, - expected_rendered=( - "\033[38;5;167m\033[1merror[D012]:\033[0m " - "Provider 'slack_provider' in providers/slack.py has invalid settings:\n" - "\033[2m1 validation error for SlackProvider\033[0m\n" - "\033[2mSLACK_TOKEN\033[0m" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_coded_error_when_formatting_then_renders_consistently( - test_case: FormatCodedErrorTestCase, -) -> None: - rendered: str = format_coded_error( - code=test_case.code, - message=test_case.message, - help=test_case.help, - use_color=test_case.use_color, - include_error_label=test_case.include_error_label, - ) - - assert rendered == test_case.expected_rendered diff --git a/tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py b/tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py deleted file mode 100644 index ebe3473032..0000000000 --- a/tests/unit/src/sqlbuild/presentation/main/test_inline_error_lines.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Tests for bounded inline error rendering.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.inline_error_lines import format_inline_error_lines -from tests.unit.src.sqlbuild.presentation.main._test_types import InlineErrorLinesTestCase - - -@pytest.mark.parametrize( - "test_case", - ( - InlineErrorLinesTestCase( - description="one-column content keeps truncation marker within width", - content_width=1, - expected_lines=["a", "."], - ), - InlineErrorLinesTestCase( - description="wide content wraps twice before using an ellipsis", - content_width=8, - expected_lines=["abcdefgh", "ijklm..."], - ), - ), - ids=lambda case: case.description, -) -def test_given_content_width_when_formatting_inline_error_then_bounds_each_line( - test_case: InlineErrorLinesTestCase, -) -> None: - lines: list[str] = format_inline_error_lines( - error_code=None, - error_message="abcdefghijklmnopqrstuvwxyz", - error_help=None, - content_width=test_case.content_width, - style=CliStyle(use_color=False), - ) - - assert lines == test_case.expected_lines From 3e803fe2653cd55bf246c708a09d677c425cf5cf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:13:06 +0100 Subject: [PATCH 02/32] refactor: run the janitor phases natively --- .../_helpers/janitor/janitor_command.rs | 72 +---- .../_helpers/janitor/janitor_phases.rs | 303 ++++++++++++++++++ .../src/bindings/_helpers/janitor/mod.rs | 1 + .../_helpers/janitor_runtime/__init__.py | 0 .../_helpers/janitor_runtime/compilation.py | 53 --- .../_helpers/janitor_runtime/connections.py | 50 --- .../_helpers/janitor_runtime/execution.py | 26 -- .../_helpers/janitor_runtime/planning.py | 79 ----- src/sqlbuild/cli/commands/models.py | 26 -- 9 files changed, 317 insertions(+), 293 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/janitor_runtime/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 6d3bc6bb69..c41bf73def 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -19,6 +19,10 @@ use sqlbuild_core::janitor_text::main::render_janitor_plan::render_janitor_plan; use sqlbuild_core::janitor_text::main::validate_janitor_settings::validate_janitor_settings; use sqlbuild_core::janitor_text::models::JanitorPlanText; +use crate::bindings::_helpers::janitor::janitor_phases::{ + JanitorCompiled, build_janitor_execution_plan, close_janitor_warehouse, + compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, +}; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -28,10 +32,6 @@ const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; -const PLANNING_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.planning"; -const EXECUTION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.execution"; -const COMPILATION_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.compilation"; -const CONNECTIONS_MODULE: &str = "sqlbuild.cli.commands._helpers.janitor_runtime.connections"; const COMMAND_NAME: &str = "janitor"; fn imported<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { @@ -185,25 +185,14 @@ fn clean( py: Python<'_>, request: &Bound<'_, PyAny>, (invocation, settings): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - (compile_context, connection_context): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (compiled, connection): (&JanitorCompiled<'_>, &Bound<'_, PyAny>), ) -> PyResult { let style: CliStyle = CliStyle { use_color: invocation.getattr("use_color")?.extract()?, }; - let planning: Bound<'_, PyAny> = imported(py, PLANNING_MODULE, "build_janitor_execution_plan")? - .call( - (), - Some(&keywords( - py, - &[ - ("invocation", invocation), - ("settings", settings), - ("compile_context", compile_context), - ("connection_context", connection_context), - ], - )?), - )?; - let plan: JanitorPlanText = janitor_plan_text(&planning.getattr("plan")?)?; + let planning: Bound<'_, PyAny> = + build_janitor_execution_plan((invocation, settings), compiled, connection)?; + let plan: JanitorPlanText = janitor_plan_text(&planning)?; write(py, &render_janitor_plan(&plan, style))?; let blocked: i32 = i32::from(!plan.blocked_schemas.is_empty()); let as_target: Option = invocation.getattr("as_target")?.extract()?; @@ -225,18 +214,7 @@ fn clean( write(py, "Janitor cancelled.\n")?; return Ok(1); } - let result: Bound<'_, PyAny> = imported(py, EXECUTION_MODULE, "execute_janitor_cleanup")? - .call( - (), - Some(&keywords( - py, - &[ - ("compile_context", compile_context), - ("connection_context", connection_context), - ("planning_result", &planning), - ], - )?), - )?; + let result: Bound<'_, PyAny> = execute_janitor_cleanup(compiled, connection, &planning)?; write( py, &janitor_completion_text(&janitor_result_counts(&result)?, style), @@ -270,39 +248,15 @@ fn run_janitor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult = - imported(py, COMPILATION_MODULE, "compile_janitor_project")? - .call((), Some(&keywords(py, &[("invocation", &invocation)])?))?; - let connection_context: Bound<'_, PyAny> = - imported(py, CONNECTIONS_MODULE, "connect_janitor_warehouse")?.call( - (), - Some(&keywords( - py, - &[ - ("invocation", &invocation), - ("compile_context", &compile_context), - ], - )?), - )?; + let compiled: JanitorCompiled<'_> = compile_janitor_project(&invocation)?; + let connection: Bound<'_, PyAny> = connect_janitor_warehouse(&invocation, &compiled)?; let outcome: PyResult = clean( py, request, (&invocation, &settings), - (&compile_context, &connection_context), + (&compiled, &connection), ); - let closed: PyResult> = - imported(py, CONNECTIONS_MODULE, "close_janitor_warehouse").and_then(|close| { - close.call( - (), - Some(&keywords( - py, - &[ - ("compile_context", &compile_context), - ("connection_context", &connection_context), - ], - )?), - ) - }); + let closed: PyResult<()> = close_janitor_warehouse(&compiled, &connection); let code: i32 = outcome?; closed?; Ok(code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs new file mode 100644 index 0000000000..eb305192e6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -0,0 +1,303 @@ +//! `sqb janitor` phases: compile, connect, inspect and plan, clean up and close. + +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; + +use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const JANITOR_PLAN_MODULE: &str = "sqlbuild.executor.janitor.main.plan"; +const JANITOR_EXECUTE_MODULE: &str = "sqlbuild.executor.janitor.main.execute"; +const JANITOR_MODELS: &str = "sqlbuild.executor.janitor.models"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const UNKNOWN_VIEW_CODE: &str = "C503"; +const UNKNOWN_VIEW_HELP: &str = + "Run sqb janitor without --drop-old-name-view to list compatibility views."; +const CANDIDATE_FIELDS: [&str; 4] = [ + "candidates", + "archive_deletion_candidates", + "query_diff_artifact_candidates", + "direct_state_prune_candidates", +]; + +/// The compiled project, its adapter and the active target connection settings. +pub(crate) struct JanitorCompiled<'py> { + pub(crate) adapter_name: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) project: Bound<'py, PyAny>, + pub(crate) connection_config: Bound<'py, PyAny>, +} + +fn stdout(py: Python<'_>) -> PyResult> { + py.import("sys")?.getattr("stdout") +} + +fn use_color(invocation: &Bound<'_, PyAny>) -> PyResult { + invocation.getattr("use_color")?.extract() +} + +/// `OperationLifecycle(operation_kind=..., operation_name=...)`. +fn lifecycle<'py>(py: Python<'py>, (kind, name): (&str, &str)) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("operation_kind", kind)?; + options.set_item("operation_name", name)?; + imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?.call((), Some(&options)) +} + +/// Compile for the janitor or `--as` target and resolve the active target connection. +pub(crate) fn compile_janitor_project<'py>( + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let context: Bound<'py, PyAny> = call_with( + &imported( + py, + ADAPTER_CONTEXT_MODULE, + "resolve_adapter_connection_context", + )?, + &[ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ( + "effective_project_dir", + invocation.getattr("effective_project_dir")?, + ), + ("selected_target", invocation.getattr("selected_target")?), + ("cli_vars", py.None().into_bound(py)), + ], + )?; + let started: Instant = Instant::now(); + let status: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stdout(py)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?; + let as_target: Bound<'py, PyAny> = invocation.getattr("as_target")?; + let target: Bound<'py, PyAny> = if as_target.is_none() { + invocation.getattr("selected_target")? + } else { + as_target + }; + let project: Bound<'py, PyAny> = + within(&lifecycle(py, ("project", "project_compile"))?, |_| { + status.call_method1("on_progress", ("Compiling project...",))?; + let project: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ( + "discovered_inputs", + invocation.getattr("discovered_inputs")?, + ), + ("adapter", context.getattr("adapter")?), + ("selected_target", target), + ("resolved_connection", context.getattr("connection_config")?), + ], + )?; + status.call_method1( + "on_progress", + (format!( + "Compiled project. ({:.2}s)", + started.elapsed().as_secs_f64() + ),), + )?; + Ok(project) + })?; + Ok(JanitorCompiled { + adapter_name: context.getattr("adapter_name")?, + adapter: context.getattr("adapter")?, + project, + connection_config: context.getattr("connection_config")?, + }) +} + +fn connection_timing(progress: &Bound<'_, PyAny>, method: &str, started: Instant) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(progress.py()); + options.set_item("connection_count", 1)?; + options.set_item("elapsed_seconds", started.elapsed().as_secs_f64())?; + progress.call_method(method, (), Some(&options))?; + Ok(()) +} + +/// Connect to the janitor target warehouse. +pub(crate) fn connect_janitor_warehouse<'py>( + invocation: &Bound<'py, PyAny>, + compiled: &JanitorCompiled<'py>, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", compiled.adapter_name.clone()), + ("stream", stdout(py)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?; + let started: Instant = Instant::now(); + progress.call_method1("on_connection_start", (1,))?; + let connected: PyResult> = within( + &lifecycle(py, ("janitor", "janitor_target_connection"))?, + |_| { + compiled + .adapter + .call_method1("connect", (&compiled.connection_config,)) + }, + ); + match connected { + Ok(connection) => { + connection_timing(&progress, "on_connection_complete", started)?; + Ok(connection) + } + Err(error) => { + connection_timing(&progress, "on_connection_error", started)?; + Err(error) + } + } +} + +/// Close the janitor warehouse connection. +pub(crate) fn close_janitor_warehouse( + compiled: &JanitorCompiled<'_>, + connection: &Bound<'_, PyAny>, +) -> PyResult<()> { + compiled.adapter.call_method1("close", (connection,))?; + Ok(()) +} + +fn candidate_count(plan: &Bound<'_, PyAny>) -> PyResult { + let mut count: usize = plan.getattr("old_name_views")?.getattr("drops")?.len()?; + for field in CANDIDATE_FIELDS { + count += plan.getattr(field)?.len()?; + } + Ok(count) +} + +fn unknown_view_error(py: Python<'_>, requests: &Bound<'_, PyAny>) -> PyResult { + let names: Vec = requests.extract()?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", UNKNOWN_VIEW_CODE)?; + options.set_item("help", UNKNOWN_VIEW_HELP)?; + let error: Bound<'_, PyAny> = imported(py, ERRORS_MODULE, "CliUserError")?.call( + (format!( + "no compatibility view is recorded at {}", + names.join(", ") + ),), + Some(&options), + )?; + Ok(PyErr::from_value(error)) +} + +/// Build the janitor execution plan from warehouse facts. +pub(crate) fn build_janitor_execution_plan<'py>( + (invocation, settings): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + compiled: &JanitorCompiled<'py>, + connection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = invocation.py(); + let started: Instant = Instant::now(); + let status: Bound<'py, PyAny> = status_reporter(&stdout(py)?, use_color(invocation)?)?; + status.call_method1("start", ("Inspecting warehouse state...",))?; + let janitor: Bound<'py, PyAny> = invocation + .getattr("discovered_inputs")? + .getattr("project_config")? + .getattr("janitor")?; + let plan: Bound<'py, PyAny> = within( + &lifecycle(py, ("janitor", "janitor_candidate_planning"))?, + |lifecycle| { + let direct_settings: Bound<'py, PyAny> = call_with( + &imported(py, JANITOR_MODELS, "JanitorDirectModeSettings")?, + &[ + ("enabled", PyBool::new(py, true).to_owned().into_any()), + ( + "state_history_versions", + settings.getattr("direct_state_history_versions")?, + ), + ( + "archive_retention_days", + settings.getattr("archive_retention_days")?, + ), + ], + )?; + let plan: Bound<'py, PyAny> = call_with( + &imported(py, JANITOR_PLAN_MODULE, "build_janitor_plan")?, + &[ + ("project", compiled.project.clone()), + ("adapter", compiled.adapter.clone()), + ("connection", connection.clone()), + ("retention_days", settings.getattr("retention_days")?), + ( + "delete_tracked_only", + janitor.getattr("delete_tracked_only")?, + ), + ("exclude_patterns", janitor.getattr("exclude_patterns")?), + ("direct_settings", direct_settings), + ( + "early_old_name_view_drops", + settings.getattr("drop_old_name_views")?, + ), + ], + )?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", candidate_count(&plan)?)?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("metadata", metadata)?; + lifecycle.call_method("completed", (), Some(&options))?; + Ok(plan) + }, + )?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "message", + format!( + "Inspected warehouse state. ({:.2}s)", + started.elapsed().as_secs_f64() + ), + )?; + options.set_item("blank_line_after", true)?; + status.call_method("complete", (), Some(&options))?; + let unknown: Bound<'py, PyAny> = plan + .getattr("old_name_views")? + .getattr("unknown_requests")?; + if unknown.is_truthy()? { + return Err(unknown_view_error(py, &unknown)?); + } + Ok(plan) +} + +/// Execute the warehouse cleanup plan. +pub(crate) fn execute_janitor_cleanup<'py>( + compiled: &JanitorCompiled<'py>, + connection: &Bound<'py, PyAny>, + plan: &Bound<'py, PyAny>, +) -> PyResult> { + call_with( + &imported( + compiled.adapter.py(), + JANITOR_EXECUTE_MODULE, + "execute_janitor_plan", + )?, + &[ + ("plan", plan.clone()), + ("adapter", compiled.adapter.clone()), + ("connection", connection.clone()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs index af44f2ecdc..b3e8c36dc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod janitor_command; +pub(crate) mod janitor_phases; pub(crate) mod janitor_plan; pub(crate) mod registration; diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/__init__.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py deleted file mode 100644 index 17f5b4df22..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Janitor command compilation phase.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild._native import PlanningProgressReporter, build_compiled_project -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.models import ( - AdapterConnectionContext, - JanitorCompileContext, - JanitorInvocation, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileContext: - """Compile for the janitor or ``--as`` target and resolve the active target connection.""" - - adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=invocation.discovered_inputs, - effective_project_dir=invocation.effective_project_dir, - selected_target=invocation.selected_target, - cli_vars=None, - ) - compile_start: float = time.perf_counter() - status: PlanningProgressReporter = PlanningProgressReporter( - stream=sys.stdout, - use_color=invocation.use_color, - ) - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - status.on_progress("Compiling project...") - project: CompiledProjectView = build_compiled_project( - discovered_inputs=invocation.discovered_inputs, - adapter=adapter_context.adapter, - selected_target=( - invocation.as_target - if invocation.as_target is not None - else invocation.selected_target - ), - resolved_connection=adapter_context.connection_config, - ) - status.on_progress(f"Compiled project. ({time.perf_counter() - compile_start:.2f}s)") - return JanitorCompileContext( - adapter_name=adapter_context.adapter_name, - adapter=adapter_context.adapter, - project=project, - connection_config=adapter_context.connection_config, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py deleted file mode 100644 index 0c9e5f71f9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/connections.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Janitor command connection phases.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild._native import ConnectionProgressReporter -from sqlbuild.cli.commands.models import ( - JanitorCompileContext, - JanitorConnectionContext, - JanitorInvocation, -) -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def connect_janitor_warehouse( - *, invocation: JanitorInvocation, compile_context: JanitorCompileContext -) -> JanitorConnectionContext: - """Connect to the janitor target warehouse.""" - - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=compile_context.adapter_name, - stream=sys.stdout, - use_color=invocation.use_color, - ) - connection_start: float = time.perf_counter() - connection_progress.on_connection_start(1) - try: - with OperationLifecycle( - operation_kind="janitor", operation_name="janitor_target_connection" - ): - connection: object = compile_context.adapter.connect(compile_context.connection_config) - except BaseException: - connection_progress.on_connection_error( - connection_count=1, elapsed_seconds=time.perf_counter() - connection_start - ) - raise - connection_progress.on_connection_complete( - connection_count=1, elapsed_seconds=time.perf_counter() - connection_start - ) - return JanitorConnectionContext(connection=connection) - - -def close_janitor_warehouse( - *, compile_context: JanitorCompileContext, connection_context: JanitorConnectionContext -) -> None: - """Close the janitor warehouse connection.""" - - compile_context.adapter.close(connection_context.connection) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py deleted file mode 100644 index 06820c93fd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Janitor command warehouse cleanup execution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.models import ( - JanitorCompileContext, - JanitorConnectionContext, - JanitorPlanningResult, -) -from sqlbuild.executor.janitor.main.execute import execute_janitor_plan -from sqlbuild.executor.janitor.models import JanitorExecutionResult - - -def execute_janitor_cleanup( - *, - compile_context: JanitorCompileContext, - connection_context: JanitorConnectionContext, - planning_result: JanitorPlanningResult, -) -> JanitorExecutionResult: - """Execute the warehouse cleanup plan.""" - - return execute_janitor_plan( - plan=planning_result.plan, - adapter=compile_context.adapter, - connection=connection_context.connection, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py deleted file mode 100644 index 2a0a9f7fc9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Janitor command warehouse inspection and planning phases.""" - -from __future__ import annotations - -import sys -import time - -from sqlbuild._native import TransientStatusReporter -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - JanitorCompileContext, - JanitorConnectionContext, - JanitorInvocation, - JanitorPlanningResult, - JanitorSettings, -) -from sqlbuild.executor.janitor.main.plan import build_janitor_plan -from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - - -def build_janitor_execution_plan( - *, - invocation: JanitorInvocation, - settings: JanitorSettings, - compile_context: JanitorCompileContext, - connection_context: JanitorConnectionContext, -) -> JanitorPlanningResult: - """Build the janitor execution plan from warehouse facts.""" - - inspect_start: float = time.perf_counter() - status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stdout, - use_color=invocation.use_color, - ) - status.start("Inspecting warehouse state...") - with OperationLifecycle( - operation_kind="janitor", operation_name="janitor_candidate_planning" - ) as lifecycle: - plan: JanitorPlan = build_janitor_plan( - project=compile_context.project, - adapter=compile_context.adapter, - connection=connection_context.connection, - retention_days=settings.retention_days, - delete_tracked_only=invocation.discovered_inputs.project_config.janitor.delete_tracked_only, - exclude_patterns=invocation.discovered_inputs.project_config.janitor.exclude_patterns, - direct_settings=JanitorDirectModeSettings( - enabled=True, - state_history_versions=settings.direct_state_history_versions, - archive_retention_days=settings.archive_retention_days, - ), - early_old_name_view_drops=settings.drop_old_name_views, - ) - lifecycle.completed(metadata={"item_count": _janitor_candidate_count(plan)}) - status.complete( - message=f"Inspected warehouse state. ({time.perf_counter() - inspect_start:.2f}s)", - blank_line_after=True, - ) - if plan.old_name_views.unknown_requests: - raise CliUserError( - "no compatibility view is recorded at " - + ", ".join(plan.old_name_views.unknown_requests), - code="C503", - help="Run sqb janitor without --drop-old-name-view to list compatibility views.", - ) - return JanitorPlanningResult(plan=plan) - - -def _janitor_candidate_count(plan: JanitorPlan) -> int: - return sum( - len(candidates) - for candidates in ( - plan.candidates, - plan.archive_deletion_candidates, - plan.query_diff_artifact_candidates, - plan.direct_state_prune_candidates, - plan.old_name_views.drops, - ) - ) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index cab2bdd5c2..6d010b1750 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -46,11 +46,9 @@ from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, ) - from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.refactoring.models import RefactorProject - from sqlbuild.executor.janitor.models import JanitorPlan @dataclass(frozen=True) @@ -385,30 +383,6 @@ class JanitorSettings: drop_old_name_views: tuple[str, ...] = () -@dataclass(frozen=True) -class JanitorCompileContext: - """Compiled project and adapter context for janitor.""" - - adapter_name: str - adapter: BaseAdapter - project: CompiledProjectView - connection_config: dict[str, object] - - -@dataclass(frozen=True) -class JanitorConnectionContext: - """Janitor warehouse connection handle.""" - - connection: object - - -@dataclass(frozen=True) -class JanitorPlanningResult: - """Built janitor plan.""" - - plan: JanitorPlan - - @dataclass(frozen=True) class LoadCommandRequest: """CLI inputs for one load command invocation.""" From 1480e209b94019d6150be657d2f622d63b6d9a90 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:39:50 +0100 Subject: [PATCH 03/32] refactor: write runtime target artifacts and SQL test paths natively --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/target_paths/_helpers/mod.rs | 1 + .../src/target_paths/_helpers/path_text.rs | 21 ++ .../src/target_paths/constants.rs | 11 + .../target_paths/main/function_run_path.rs | 16 + .../src/target_paths/main/mod.rs | 4 + .../src/target_paths/main/model_run_path.rs | 12 + .../src/target_paths/main/runtime_sql_text.rs | 23 ++ .../target_paths/main/sql_test_output_path.rs | 54 ++++ .../sqlbuild-analysis/src/target_paths/mod.rs | 7 + .../src/target_paths/models.rs | 13 + .../src/target_paths/tests.rs | 6 + .../src/target_paths/tests/helpers.rs | 15 + .../target_paths/tests/test_target_paths.rs | 129 +++++++++ .../src/target_paths/tests/test_types.rs | 21 ++ .../_helpers/build_command/completion.rs | 23 +- .../bindings/_helpers/check/check_command.rs | 23 +- .../_helpers/compile_command/target_writer.rs | 20 +- .../bindings/_helpers/compile_outputs/mod.rs | 1 + .../_helpers/compile_outputs/registration.rs | 5 +- .../compile_outputs/runtime_target.rs | 274 ++++++++++++++++++ .../_helpers/sql_test_command/test_command.rs | 24 +- src/sqlbuild/_native.pyi | 3 + .../commands/_helpers/dbt/sqlbuild_work.py | 2 +- .../cli/paths/_helpers/artifact_path.py | 61 ---- .../main/_compiled_sql_test_output_path.py | 18 -- .../cli/paths/main/_sql_test_output_path.py | 12 - src/sqlbuild/cli/target_artifacts/__init__.py | 1 - .../cli/target_artifacts/_helpers/runtime.py | 192 ------------ .../cli/target_artifacts/main/__init__.py | 1 - .../_write_python_check_runtime_target.py | 18 -- .../main/_write_runtime_target.py | 19 -- .../main/_write_test_runtime_target.py | 29 -- .../test_runtime_target_writer.py | 2 +- 34 files changed, 645 insertions(+), 417 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/constants.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/mod.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/models.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/tests.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs create mode 100644 crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs delete mode 100644 src/sqlbuild/cli/paths/_helpers/artifact_path.py delete mode 100644 src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py delete mode 100644 src/sqlbuild/cli/paths/main/_sql_test_output_path.py delete mode 100644 src/sqlbuild/cli/target_artifacts/__init__.py delete mode 100644 src/sqlbuild/cli/target_artifacts/_helpers/runtime.py delete mode 100644 src/sqlbuild/cli/target_artifacts/main/__init__.py delete mode 100644 src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py delete mode 100644 src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py delete mode 100644 src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 383892f92e..84e825737a 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -33,4 +33,5 @@ pub mod semantic_usage; pub mod semantic_validation; pub mod sql_test_attachment; pub mod sql_tokens; +pub mod target_paths; pub mod type_system; diff --git a/crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs b/crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs new file mode 100644 index 0000000000..87c481a3a7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod path_text; diff --git a/crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs b/crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs new file mode 100644 index 0000000000..40e5daa881 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/_helpers/path_text.rs @@ -0,0 +1,21 @@ +//! `pathlib` suffix handling on POSIX path text. + +/// `PurePath.with_suffix(suffix)`: the last suffix of the final component replaced or added. +pub(crate) fn with_suffix(path: &str, suffix: &str) -> String { + let start: usize = path.rfind('/').map_or(0, |index| index + 1); + let name: &str = &path[start..]; + let stem_length: usize = match name.rfind('.') { + Some(index) if index > 0 && index < name.len() - 1 => index, + _ => name.len(), + }; + format!("{}{}{suffix}", &path[..start], &name[..stem_length]) +} + +/// The first `max_bytes` bytes of `value`, without a trailing partial character. +pub(crate) fn truncate_utf8(value: &str, max_bytes: usize) -> &str { + let mut end: usize = max_bytes.min(value.len()); + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} diff --git a/crates/sqlbuild-analysis/src/target_paths/constants.rs b/crates/sqlbuild-analysis/src/target_paths/constants.rs new file mode 100644 index 0000000000..99f4861abc --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/constants.rs @@ -0,0 +1,11 @@ +//! Directory names and limits of the paths SQLBuild writes under `target/`. + +pub(crate) const CHAIN_DIR: &str = "_chain_"; +pub(crate) const SQL_FILE_SUFFIX: &str = ".sql"; +/// A model chain directory name longer than this many bytes is shortened to a digest. +pub(crate) const MAX_CHAIN_COMPONENT_BYTES: usize = 200; +pub(crate) const CHAIN_DIGEST_LENGTH: usize = 12; +/// Bytes kept from the first and last model names of a shortened chain directory. +pub(crate) const CHAIN_EDGE_NAME_BYTES: usize = 80; +pub(crate) const MODELS_DIR: &str = "models"; +pub(crate) const FUNCTIONS_DIR: &str = "functions"; diff --git a/crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs b/crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs new file mode 100644 index 0000000000..6b46b65f1b --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/function_run_path.rs @@ -0,0 +1,16 @@ +//! Where a function's executed SQL is written under `target/run`. + +use crate::target_paths::_helpers::path_text::with_suffix; +use crate::target_paths::constants::{FUNCTIONS_DIR, SQL_FILE_SUFFIX}; + +/// The function's path under `functions//`, with a `.sql` suffix. +#[must_use] +pub fn function_run_path(relative_path: &str, language: &str) -> String { + let mut parts = relative_path.split('/'); + let path: String = if parts.next() == Some(FUNCTIONS_DIR) && parts.next() == Some(language) { + relative_path.to_owned() + } else { + format!("{FUNCTIONS_DIR}/{language}/{relative_path}") + }; + with_suffix(&path, SQL_FILE_SUFFIX) +} diff --git a/crates/sqlbuild-analysis/src/target_paths/main/mod.rs b/crates/sqlbuild-analysis/src/target_paths/main/mod.rs new file mode 100644 index 0000000000..43cc006909 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/mod.rs @@ -0,0 +1,4 @@ +pub mod function_run_path; +pub mod model_run_path; +pub mod runtime_sql_text; +pub mod sql_test_output_path; diff --git a/crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs b/crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs new file mode 100644 index 0000000000..1b65f25dd1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/model_run_path.rs @@ -0,0 +1,12 @@ +//! Where a model's executed SQL is written under `target/run`. + +use crate::target_paths::constants::MODELS_DIR; + +/// The model's relative path, under `models/` unless it already starts there. +#[must_use] +pub fn model_run_path(relative_path: &str) -> String { + if relative_path.split('/').next() == Some(MODELS_DIR) { + return relative_path.to_owned(); + } + format!("{MODELS_DIR}/{relative_path}") +} diff --git a/crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs b/crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs new file mode 100644 index 0000000000..2d0a3d51f1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/runtime_sql_text.rs @@ -0,0 +1,23 @@ +//! The text of a runtime SQL artifact: its statements, each terminated. + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +/// The statements joined by blank lines, each ending in `;`, with one final newline. +#[must_use] +pub fn runtime_sql_text(statements: &[String]) -> String { + let joined: String = statements + .iter() + .map(|statement| { + let stripped: &str = statement.trim_end_matches(python_isspace); + if stripped.is_empty() { + statement.clone() + } else if stripped.ends_with(';') { + stripped.to_owned() + } else { + format!("{stripped};") + } + }) + .collect::>() + .join("\n\n"); + format!("{}\n", joined.trim_end_matches(python_isspace)) +} diff --git a/crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs b/crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs new file mode 100644 index 0000000000..d615afc011 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/main/sql_test_output_path.rs @@ -0,0 +1,54 @@ +//! The readable, filesystem-safe path of one SQL test artifact. + +use sha2::{Digest, Sha256}; + +use crate::target_paths::_helpers::path_text::{truncate_utf8, with_suffix}; +use crate::target_paths::constants::{ + CHAIN_DIGEST_LENGTH, CHAIN_DIR, CHAIN_EDGE_NAME_BYTES, MAX_CHAIN_COMPONENT_BYTES, + SQL_FILE_SUFFIX, +}; +use crate::target_paths::models::SqlTestPathFacts; + +fn chain_folder(name: &str, model_names: &[String]) -> String { + let mut unique: Vec<&str> = model_names.iter().map(String::as_str).collect(); + unique.sort_unstable(); + unique.dedup(); + match unique.as_slice() { + [] => return name.to_owned(), + [only] => return (*only).to_owned(), + _ => {} + } + let chain: String = unique.join("__"); + if chain.len() <= MAX_CHAIN_COMPONENT_BYTES { + return format!("{CHAIN_DIR}/{chain}"); + } + let digest: String = Sha256::digest(chain.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::()[..CHAIN_DIGEST_LENGTH] + .to_owned(); + format!( + "{CHAIN_DIR}/{}__{}__{digest}", + truncate_utf8(unique[0], CHAIN_EDGE_NAME_BYTES), + truncate_utf8(unique[unique.len() - 1], CHAIN_EDGE_NAME_BYTES) + ) +} + +/// The test's chain folder and name, or its source stem and case for an expanded case. +#[must_use] +pub fn sql_test_output_path(facts: &SqlTestPathFacts) -> String { + match (&facts.case_name, &facts.source_path) { + (Some(case_name), Some(source_path)) => format!( + "{}/block_{}__{case_name}{SQL_FILE_SUFFIX}", + with_suffix(source_path, ""), + facts + .block_index + .map_or_else(|| "None".to_owned(), |index| index.to_string()) + ), + _ => format!( + "{}/{}{SQL_FILE_SUFFIX}", + chain_folder(&facts.name, &facts.model_names), + facts.name + ), + } +} diff --git a/crates/sqlbuild-analysis/src/target_paths/mod.rs b/crates/sqlbuild-analysis/src/target_paths/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/target_paths/models.rs b/crates/sqlbuild-analysis/src/target_paths/models.rs new file mode 100644 index 0000000000..3b5ce2205d --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/models.rs @@ -0,0 +1,13 @@ +//! The SQL test facts its target path is built from. + +/// One SQL test or expanded case, as its target path needs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqlTestPathFacts { + pub name: String, + /// The authoring file, relative and in POSIX form. + pub source_path: Option, + pub block_index: Option, + pub case_name: Option, + /// The models of the test's chain, in any order and possibly repeated. + pub model_names: Vec, +} diff --git a/crates/sqlbuild-analysis/src/target_paths/tests.rs b/crates/sqlbuild-analysis/src/target_paths/tests.rs new file mode 100644 index 0000000000..a27406ab34 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_target_paths.rs"] +mod target_paths; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs b/crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs new file mode 100644 index 0000000000..f6f1ea0e23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests/helpers.rs @@ -0,0 +1,15 @@ +use crate::target_paths::models::SqlTestPathFacts; + +pub(super) fn facts( + name: &str, + case: Option<(&str, Option, &str)>, + models: &[&str], +) -> SqlTestPathFacts { + SqlTestPathFacts { + name: name.to_owned(), + source_path: case.map(|(source, _, _)| source.to_owned()), + block_index: case.and_then(|(_, block, _)| block), + case_name: case.map(|(_, _, case_name)| case_name.to_owned()), + model_names: models.iter().map(|model| (*model).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs b/crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs new file mode 100644 index 0000000000..2d8f49a8c8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests/test_target_paths.rs @@ -0,0 +1,129 @@ +use crate::target_paths::main::function_run_path::function_run_path; +use crate::target_paths::main::model_run_path::model_run_path; +use crate::target_paths::main::runtime_sql_text::runtime_sql_text; +use crate::target_paths::main::sql_test_output_path::sql_test_output_path; +use crate::target_paths::tests::helpers::facts; +use crate::target_paths::tests::test_types::{ + RunPathTestCase, RuntimeSqlTestCase, SqlTestPathTestCase, +}; + +/// Expectations recorded from the retired Python path builders. +#[test] +fn given_sql_test_when_building_output_path_then_matches_python() { + let long: Vec = (0..9) + .map(|index| format!("m\u{e9}tric_{index:02}_xxxxxxxxxxxxxxxxxxxx")) + .collect(); + let last: String = format!("{}a", "\u{e9}".repeat(50)); + let mut long_chain: Vec<&str> = vec![last.as_str()]; + long_chain.extend(long.iter().map(String::as_str)); + let test_cases = [ + SqlTestPathTestCase { + description: "test without models uses its own name", + facts: facts("t1", None, &[]), + expected_path: "t1/t1.sql", + }, + SqlTestPathTestCase { + description: "one repeated model names the folder", + facts: facts("t2", None, &["orders", "orders"]), + expected_path: "orders/t2.sql", + }, + SqlTestPathTestCase { + description: "a chain is sorted and deduplicated", + facts: facts("t3", None, &["zeta", "alpha", "zeta"]), + expected_path: "_chain_/alpha__zeta/t3.sql", + }, + SqlTestPathTestCase { + description: "a long chain keeps truncated edge names and a digest", + facts: facts("t4", None, &long_chain), + expected_path: "_chain_/m\u{e9}tric_00_xxxxxxxxxxxxxxxxxxxx__\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}\u{e9}__132c41e73f1c/t4.sql", + }, + SqlTestPathTestCase { + description: "an expanded case lives under its source stem", + facts: facts( + "t5", + Some(("tests/orders.sql", Some(2), "happy")), + &["orders"], + ), + expected_path: "tests/orders/block_2__happy.sql", + }, + SqlTestPathTestCase { + description: "a dot file keeps its name and a missing block prints None", + facts: facts("t6", Some(("tests/.hidden", None, "c")), &[]), + expected_path: "tests/.hidden/block_None__c.sql", + }, + ]; + + for test_case in test_cases { + assert_eq!( + sql_test_output_path(&test_case.facts), + test_case.expected_path, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_path_when_building_run_paths_then_matches_python() { + let test_cases = [ + RunPathTestCase { + description: "paths already under their directory are kept", + relative_path: "models/a/b.sql", + language: "sql", + expected_model_path: "models/a/b.sql", + expected_function_path: "functions/sql/models/a/b.sql", + }, + RunPathTestCase { + description: "other paths are nested and functions get a sql suffix", + relative_path: "udfs/f.py", + language: "python", + expected_model_path: "models/udfs/f.py", + expected_function_path: "functions/python/udfs/f.sql", + }, + RunPathTestCase { + description: "a function under its language directory is kept", + relative_path: "functions/sql/f.sql", + language: "sql", + expected_model_path: "models/functions/sql/f.sql", + expected_function_path: "functions/sql/f.sql", + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + model_run_path(test_case.relative_path), + function_run_path(test_case.relative_path, test_case.language) + ), + ( + test_case.expected_model_path.to_owned(), + test_case.expected_function_path.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_statements_when_rendering_runtime_sql_then_matches_python() { + let test_cases = [RuntimeSqlTestCase { + description: "statements are terminated and blank ones kept", + statements: vec![ + "select 1 ".to_owned(), + "create table x as select 2;\n".to_owned(), + " ".to_owned(), + "drop view y\u{1f}".to_owned(), + ], + expected_text: "select 1;\n\ncreate table x as select 2;\n\n \n\ndrop view y;\n", + }]; + + for test_case in test_cases { + assert_eq!( + runtime_sql_text(&test_case.statements), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs b/crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs new file mode 100644 index 0000000000..0a90265759 --- /dev/null +++ b/crates/sqlbuild-analysis/src/target_paths/tests/test_types.rs @@ -0,0 +1,21 @@ +use crate::target_paths::models::SqlTestPathFacts; + +pub(super) struct SqlTestPathTestCase { + pub(super) description: &'static str, + pub(super) facts: SqlTestPathFacts, + pub(super) expected_path: &'static str, +} + +pub(super) struct RunPathTestCase { + pub(super) description: &'static str, + pub(super) relative_path: &'static str, + pub(super) language: &'static str, + pub(super) expected_model_path: &'static str, + pub(super) expected_function_path: &'static str, +} + +pub(super) struct RuntimeSqlTestCase { + pub(super) description: &'static str, + pub(super) statements: Vec, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 7dd3ddb644..34b0529700 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -18,6 +18,9 @@ use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, none, text, utc_now, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_outputs::runtime_target::{ + write_python_check_runtime_target, write_runtime_target, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; @@ -25,9 +28,6 @@ const COST_COLLECTION_MODULE: &str = "sqlbuild.cli.commands._helpers.cost.collec const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; -const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_runtime_target"; -const CHECK_TARGET_MODULE: &str = - "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; const SUCCESS_STATUS: &str = "success"; const PENDING_MESSAGE: &str = "Build cost collection has not completed."; const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; @@ -326,21 +326,8 @@ pub(crate) fn complete_direct_build<'py>( let exit_code: i32 = build_exit_code(succeeded, (python_failed, checks_failed)); let target_dir: Bound<'py, PyAny> = joined(&context.get("effective_project_dir")?, &["target"])?; - call_with( - &imported(py, RUNTIME_TARGET_MODULE, "write_runtime_target")?, - &[ - ("target_dir", target_dir.clone()), - ("plan_output", plan.clone()), - ("result", result.clone()), - ], - )?; - call_with( - &imported(py, CHECK_TARGET_MODULE, "write_python_check_runtime_target")?, - &[ - ("target_dir", target_dir), - ("results", check_results.clone()), - ], - )?; + write_runtime_target(&target_dir, &plan, result)?; + write_python_check_runtime_target(&target_dir, check_results)?; let cost_started: Instant = Instant::now(); let record: PyResult> = finalize_build_cost( context, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 609fbd9225..a2917f3156 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -39,8 +40,6 @@ const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; -const RUNTIME_TARGET_MODULE: &str = - "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ select checks directly without '+', '~', or path-between expansion"; @@ -658,21 +657,11 @@ fn complete<'py>( stream.call_method1("write", (render_check_summary(&texts, style),))?; stream.call_method0("flush")?; let tuple: Bound<'py, PyAny> = PyTuple::new(py, results)?.into_any(); - call_with( - &imported( - py, - RUNTIME_TARGET_MODULE, - "write_python_check_runtime_target", - )?, - &[ - ( - "target_dir", - invocation - .getattr("effective_project_dir")? - .call_method1("__truediv__", ("target",))?, - ), - ("results", tuple), - ], + write_python_check_runtime_target( + &invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + &tuple, )?; call_with( &imported(py, NATIVE_MODULE, "write_execution_json_output")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 00eb66a67c..9e6bffcfb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -27,6 +27,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::compile_outputs::artifacts::{ publish_staged_artifacts, remove_stale_artifacts, }; +use crate::bindings::_helpers::compile_outputs::runtime_target::{ + compiled_sql_test_path, planned_sql_test_path, +}; const COMPILED_DIR: &str = "compiled"; const MODELS_DIR: &str = "models"; @@ -43,8 +46,6 @@ const PUBLISHED_STAGING_CHANGED: &str = "changed"; const PUBLISHED_TREE: &str = "tree"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COMPILED_TEST_PATH_MODULE: &str = "sqlbuild.cli.paths.main._compiled_sql_test_output_path"; -const TEST_PATH_MODULE: &str = "sqlbuild.cli.paths.main._sql_test_output_path"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const MODEL_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.execution.sql_test_model_chain"; @@ -269,7 +270,7 @@ fn write_planned_target<'py>( let entry: Bound<'py, PyAny> = entry?; let path: Bound<'py, PyAny> = compiled .div(TESTS_DIR)? - .div(imported(py, TEST_PATH_MODULE, "sql_test_output_path")?.call1((&entry,))?)?; + .div(planned_sql_test_path(&entry)?)?; let comparison_sql: String = timed(py, "comparison_render_ms", || { call_with( &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, @@ -537,16 +538,9 @@ fn write_static_tests<'py>( let mut test_paths: Vec> = Vec::new(); for ((test, _, _), artifact) in zipped(planned)? { let artifact: &Bound<'py, PyAny> = artifact.bind(py); - let path: Bound<'py, PyAny> = tests_root.div(call_with( - &imported( - py, - COMPILED_TEST_PATH_MODULE, - "compiled_sql_test_output_path", - )?, - &[ - ("test", test.bind(py).clone()), - ("model_names", artifact.getattr("model_names")?), - ], + let path: Bound<'py, PyAny> = tests_root.div(compiled_sql_test_path( + test.bind(py), + &artifact.getattr("model_names")?, )?)?; batch.queue_sql(&path, &artifact.getattr("sql")?.extract::()?)?; managed.push(path.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs index 0afc71b85d..6e37e35303 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod artifacts; pub(crate) mod json_report; pub(crate) mod registration; pub(crate) mod reuse; +pub(crate) mod runtime_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs index 604ca7f3d2..26fea3bc26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_outputs::{json_report, reuse}; +use crate::bindings::_helpers::compile_outputs::{json_report, reuse, runtime_target}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { json_report::register(module)?; - reuse::register(module) + reuse::register(module)?; + runtime_target::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs new file mode 100644 index 0000000000..8e8784396f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -0,0 +1,274 @@ +//! SQL test artifact paths, and the executed SQL and Python check results written to `target/run`. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use pyo3::wrap_pyfunction; +use sqlbuild_analysis::target_paths::main::function_run_path::function_run_path; +use sqlbuild_analysis::target_paths::main::model_run_path::model_run_path; +use sqlbuild_analysis::target_paths::main::runtime_sql_text::runtime_sql_text; +use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output_path; +use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const LIFECYCLE_TYPES: &str = "sqlbuild.adapter.contract.types"; +const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; +const RUN_DIR: &str = "run"; +const TESTS_DIR: &str = "tests"; +const CHECKS_DIR: &str = "checks"; +const PYTHON_CHECKS_FILE: &str = "python_checks.json"; +const PYTHON_CHECK_KIND: &str = "python_check"; +const JSON_INDENT: i64 = 2; + +/// One test's identity as the test runtime target matches entries to results. +type TestKey = (String, Option, Option, Option); + +fn posix(path: &Bound<'_, PyAny>) -> PyResult> { + if path.is_none() { + return Ok(None); + } + path.call_method0("as_posix")?.extract().map(Some) +} + +fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> { + imported(py, "pathlib", "Path")?.call1((text,)) +} + +fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { + Ok(( + value.getattr(name_field)?.extract()?, + posix(&value.getattr("source_path")?)?, + value.getattr("block_index")?.extract()?, + value.getattr("case_name")?.extract()?, + )) +} + +fn path_facts(test: &Bound<'_, PyAny>, model_names: Vec) -> PyResult { + let (name, source_path, block_index, case_name) = test_key(test, "name")?; + Ok(SqlTestPathFacts { + name, + source_path, + block_index, + case_name, + model_names, + }) +} + +/// The target path of one planned SQL test, relative to its artifact directory. +pub(crate) fn planned_sql_test_path<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let mut model_names: Vec = Vec::new(); + for step in entry.getattr("chain")?.try_iter()? { + model_names.push(step?.getattr("model_name")?.extract()?); + } + path_object( + entry.py(), + &sql_test_output_path(&path_facts(entry, model_names)?), + ) +} + +/// The target path of one compiled SQL test from its chain's model names. +pub(crate) fn compiled_sql_test_path<'py>( + test: &Bound<'py, PyAny>, + model_names: &Bound<'py, PyAny>, +) -> PyResult> { + path_object( + test.py(), + &sql_test_output_path(&path_facts(test, model_names.extract()?)?), + ) +} + +fn write_text(path: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { + let py: Python<'_> = path.py(); + let mkdir: Bound<'_, PyDict> = PyDict::new(py); + mkdir.set_item("parents", true)?; + mkdir.set_item("exist_ok", true)?; + path.getattr("parent")? + .call_method("mkdir", (), Some(&mkdir))?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("encoding", "utf-8")?; + path.call_method("write_text", (text,), Some(&options))?; + Ok(()) +} + +fn sql_statements(result: &Bound<'_, PyAny>) -> PyResult> { + let sql_kind: Bound<'_, PyAny> = + imported(result.py(), LIFECYCLE_TYPES, "LifeCycleEventKind")?.getattr("SQL")?; + let mut statements: Vec = Vec::new(); + for event in result.getattr("lifecycle_events")?.try_iter()? { + let event: Bound<'_, PyAny> = event?; + if event.getattr("kind")?.eq(&sql_kind)? { + statements.push(event.getattr("content")?.extract()?); + } + } + Ok(statements) +} + +fn entry_paths( + entries: &Bound<'_, PyAny>, + path_of: impl Fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult> { + let mut paths: HashMap = HashMap::new(); + for entry in entries.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + paths.insert(entry.getattr("name")?.extract()?, path_of(&entry)?); + } + Ok(paths) +} + +fn relative_posix(entry: &Bound<'_, PyAny>) -> PyResult { + entry + .getattr("relative_path")? + .call_method0("as_posix")? + .extract() +} + +/// Write executed model and function lifecycle SQL under `target/run`. +pub(crate) fn write_runtime_target( + target_dir: &Bound<'_, PyAny>, + plan: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + let run_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?; + let models: HashMap = entry_paths(&plan.getattr("model_entries")?, |entry| { + relative_posix(entry).map(|path| model_run_path(&path)) + })?; + let functions: HashMap = + entry_paths(&plan.getattr("function_entries")?, |entry| { + Ok(function_run_path( + &relative_posix(entry)?, + &entry + .getattr("language")? + .getattr("value")? + .extract::()?, + )) + })?; + for (results, name_field, paths) in [ + ("model_results", "model_name", &models), + ("function_results", "function_name", &functions), + ] { + for item in result.getattr(results)?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + let statements: Vec = sql_statements(&item)?; + if statements.is_empty() { + continue; + } + let name: String = item.getattr(name_field)?.extract()?; + if let Some(path) = paths.get(&name) { + write_text(&run_dir.div(path)?, &runtime_sql_text(&statements))?; + } + } + } + Ok(()) +} + +/// Write the executed SQL unit-test comparison statements under `target/run/tests`. +pub(crate) fn write_test_runtime_target( + (target_dir, adapter): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + plan: &Bound<'_, PyAny>, + results: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = target_dir.py(); + let mut executed: HashSet = HashSet::new(); + for result in results.try_iter()? { + executed.insert(test_key(&result?, "test_name")?); + } + let tests_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?.div(TESTS_DIR)?; + for entry in plan.getattr("test_entries")?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if !executed.contains(&test_key(&entry, "name")?) { + continue; + } + let sql: String = call_with( + &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, + &[ + ("test_entry", entry.clone()), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ], + )? + .extract()?; + write_text( + &tests_dir.div(planned_sql_test_path(&entry)?)?, + &format!("{}\n", sql.trim_end_matches(python_isspace)), + )?; + } + Ok(()) +} + +fn check_status(result: &Bound<'_, PyAny>) -> PyResult<&'static str> { + if result.getattr("passed")?.is_truthy()? { + return Ok("pass"); + } + if result.getattr("warned")?.is_truthy()? { + return Ok("warn"); + } + Ok("fail") +} + +/// Write Python check results under `target/run/checks` when any ran. +pub(crate) fn write_python_check_runtime_target( + target_dir: &Bound<'_, PyAny>, + results: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = target_dir.py(); + let checks: Bound<'_, PyList> = PyList::empty(py); + for result in results.try_iter()? { + let result: Bound<'_, PyAny> = result?; + let name: Bound<'_, PyAny> = result.getattr("node_name")?; + let check: Bound<'_, PyDict> = PyDict::new(py); + check.set_item("kind", PYTHON_CHECK_KIND)?; + check.set_item("name", &name)?; + check.set_item("display_name", &name)?; + check.set_item("check_id", format!("{PYTHON_CHECK_KIND}:{}", name.str()?))?; + check.set_item("status", check_status(&result)?)?; + check.set_item("passed", result.getattr("passed")?)?; + check.set_item("severity", result.getattr("severity")?.getattr("value")?)?; + check.set_item("message", result.getattr("message")?)?; + check.set_item("error_message", result.getattr("error_message")?)?; + check.set_item("metadata", result.getattr("metadata")?)?; + checks.append(check)?; + } + if checks.is_empty() { + return Ok(()); + } + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("checks", checks)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("indent", JSON_INDENT)?; + let text: String = imported(py, "json", "dumps")? + .call((payload,), Some(&options))? + .extract()?; + write_text( + &target_dir + .div(RUN_DIR)? + .div(CHECKS_DIR)? + .div(PYTHON_CHECKS_FILE)?, + &format!("{text}\n"), + ) +} + +/// Write executed model and function lifecycle SQL under `target/run`. +#[pyfunction(name = "write_runtime_target")] +#[pyo3(signature = (*, target_dir, plan_output, result))] +fn write_runtime_target_binding( + target_dir: &Bound<'_, PyAny>, + plan_output: &Bound<'_, PyAny>, + result: &Bound<'_, PyAny>, +) -> PyResult<()> { + write_runtime_target(target_dir, plan_output, result) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(write_runtime_target_binding, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index c28e0f40d9..c7b51d761b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::build_progress::transient_status::status_reporter use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runtime_target; use crate::bindings::_helpers::result_output::integration_values::json_value; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -41,7 +42,6 @@ const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; -const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_test_runtime_target"; const DECIMAL_KIND: &str = "decimal"; const PASS_OUTCOME: &str = "pass"; const EXPECT_LABEL: &str = "expect"; @@ -673,19 +673,15 @@ fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult header.set_item("use_color", invocation.getattr("use_color")?)?; imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; let results: Vec> = run_tests(&invocation, &pipeline, concurrency)?; - call_with( - &imported(py, RUNTIME_TARGET_MODULE, "write_test_runtime_target")?, - &[ - ( - "target_dir", - invocation - .getattr("effective_project_dir")? - .call_method1("__truediv__", ("target",))?, - ), - ("adapter", invocation.getattr("adapter")?), - ("plan_output", pipeline.getattr("plan_output")?), - ("results", PyTuple::new(py, &results)?.into_any()), - ], + write_test_runtime_target( + ( + &invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))?, + &invocation.getattr("adapter")?, + ), + &pipeline.getattr("plan_output")?, + &PyTuple::new(py, &results)?.into_any(), )?; complete(request, &invocation, &results) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b2116bd81e..5851044964 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3366,6 +3366,9 @@ def resolve_loader_defaults( target_config: TargetConfig | None = None, cli_vars: dict[str, object] | None = None, ) -> tuple[str | None, str | None]: ... +def write_runtime_target( + *, target_dir: Path, plan_output: PlanOutput, result: BuildExecutionResult +) -> None: ... class TransientStatusReporter: def __init__( diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index 8132be96ca..caff1a6b4d 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -12,6 +12,7 @@ enforce_plan_safety_policies, format_build_footer, format_execution_header, + write_runtime_target, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands.constants import ( @@ -22,7 +23,6 @@ TABLE_TYPE_DOWNGRADE_DBT_HELP, ) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext, PlanSafetyGates -from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import ( PlanOutput, diff --git a/src/sqlbuild/cli/paths/_helpers/artifact_path.py b/src/sqlbuild/cli/paths/_helpers/artifact_path.py deleted file mode 100644 index a5528bf8f1..0000000000 --- a/src/sqlbuild/cli/paths/_helpers/artifact_path.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Stable, filesystem-safe CLI artifact paths.""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -from sqlbuild.compiler.planner.models import SqlTestPlanEntry - -_CHAIN_DIR: str = "_chain_" -_SQL_FILE_SUFFIX: str = ".sql" -_MAX_CHAIN_COMPONENT_BYTES: int = 200 -_CHAIN_DIGEST_LENGTH: int = 12 -_CHAIN_EDGE_NAME_BYTES: int = 80 - - -def build_sql_test_output_path(entry: SqlTestPlanEntry) -> Path: - """Return a readable test path with bounded chain directory components.""" - - return build_sql_test_output_path_from_values( - name=entry.name, - source_path=entry.source_path, - block_index=entry.block_index, - case_name=entry.case_name, - model_names=tuple(step.model_name for step in entry.chain), - ) - - -def build_sql_test_output_path_from_values( - *, - name: str, - source_path: Path | None, - block_index: int | None, - case_name: str | None, - model_names: tuple[str, ...], -) -> Path: - """Return the stable test path from compact planner artifact metadata.""" - - if case_name is None or source_path is None: - return _test_folder_from_names(name=name, model_names=model_names) / ( - f"{name}{_SQL_FILE_SUFFIX}" - ) - source_stem: Path = source_path.with_suffix("") - return source_stem / f"block_{block_index}__{case_name}{_SQL_FILE_SUFFIX}" - - -def _test_folder_from_names(*, name: str, model_names: tuple[str, ...]) -> Path: - unique_names: list[str] = sorted(set(model_names)) - if len(unique_names) <= 1: - return Path(unique_names[0] if unique_names else name) - chain_name: str = "__".join(unique_names) - if len(chain_name.encode()) <= _MAX_CHAIN_COMPONENT_BYTES: - return Path(_CHAIN_DIR) / chain_name - digest: str = hashlib.sha256(chain_name.encode()).hexdigest()[:_CHAIN_DIGEST_LENGTH] - first_name: str = _truncate_utf8(value=unique_names[0], max_bytes=_CHAIN_EDGE_NAME_BYTES) - last_name: str = _truncate_utf8(value=unique_names[-1], max_bytes=_CHAIN_EDGE_NAME_BYTES) - return Path(_CHAIN_DIR) / f"{first_name}__{last_name}__{digest}" - - -def _truncate_utf8(*, value: str, max_bytes: int) -> str: - return value.encode()[:max_bytes].decode(errors="ignore") diff --git a/src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py b/src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py deleted file mode 100644 index 15a43b417c..0000000000 --- a/src/sqlbuild/cli/paths/main/_compiled_sql_test_output_path.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Compiled SQL test artifact path entrypoint.""" - -from pathlib import Path - -from sqlbuild.cli.paths._helpers.artifact_path import build_sql_test_output_path_from_values -from sqlbuild.compiler.compile.models import CompiledSqlTest - - -def compiled_sql_test_output_path(*, test: CompiledSqlTest, model_names: tuple[str, ...]) -> Path: - """Return a test path from a compiled test and compact native plan facts.""" - - return build_sql_test_output_path_from_values( - name=test.name, - source_path=test.source_path, - block_index=test.block_index, - case_name=test.case_name, - model_names=model_names, - ) diff --git a/src/sqlbuild/cli/paths/main/_sql_test_output_path.py b/src/sqlbuild/cli/paths/main/_sql_test_output_path.py deleted file mode 100644 index e28da2e1ba..0000000000 --- a/src/sqlbuild/cli/paths/main/_sql_test_output_path.py +++ /dev/null @@ -1,12 +0,0 @@ -"""SQL test artifact path entrypoint.""" - -from pathlib import Path - -from sqlbuild.cli.paths._helpers.artifact_path import build_sql_test_output_path -from sqlbuild.compiler.planner.models import SqlTestPlanEntry - - -def sql_test_output_path(entry: SqlTestPlanEntry) -> Path: - """Return the filesystem-safe target path for one SQL test artifact.""" - - return build_sql_test_output_path(entry) diff --git a/src/sqlbuild/cli/target_artifacts/__init__.py b/src/sqlbuild/cli/target_artifacts/__init__.py deleted file mode 100644 index 1011f89980..0000000000 --- a/src/sqlbuild/cli/target_artifacts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI target artifact capabilities.""" diff --git a/src/sqlbuild/cli/target_artifacts/_helpers/runtime.py b/src/sqlbuild/cli/target_artifacts/_helpers/runtime.py deleted file mode 100644 index a3e02a8e92..0000000000 --- a/src/sqlbuild/cli/target_artifacts/_helpers/runtime.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Write runtime SQL artifacts to target/run.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.paths.main._sql_test_output_path import sql_test_output_path -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import ( - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - SqlTestPlanEntry, -) -from sqlbuild.executor.build.models import BuildExecutionResult, FunctionExecutionResult -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.executor.testing.models import SqlTestExecutionResult - -_RUN_DIR: str = "run" -_MODELS_DIR: str = "models" -_FUNCTIONS_DIR: str = "functions" -_TESTS_DIR: str = "tests" -_CHECKS_DIR: str = "checks" -_SQL_FILE_SUFFIX: str = ".sql" - - -def write_runtime_target( - *, - target_dir: Path, - plan_output: PlanOutput, - result: BuildExecutionResult, -) -> None: - """Write executed model lifecycle SQL under target/run.""" - - run_dir: Path = target_dir / _RUN_DIR - - model_entry_map: dict[str, ModelPlanEntry] = { - entry.name: entry for entry in plan_output.model_entries - } - function_entry_map: dict[str, FunctionPlanEntry] = { - entry.name: entry for entry in plan_output.function_entries - } - - model_result: ModelExecutionResult - for model_result in result.model_results: - if not model_result.lifecycle_events: - continue - sql_events: tuple[LifeCycleEvent, ...] = tuple( - e for e in model_result.lifecycle_events if e.kind == LifeCycleEventKind.SQL - ) - if not sql_events: - continue - entry: ModelPlanEntry | None = model_entry_map.get(model_result.model_name) - if entry is None: - continue - run_path: Path = run_dir / _model_output_path(entry.relative_path) - _write_sql( - path=run_path, - sql="\n\n".join(_format_statement(e.content) for e in sql_events), - ) - - function_result: FunctionExecutionResult - for function_result in result.function_results: - if not function_result.lifecycle_events: - continue - function_sql_events: tuple[LifeCycleEvent, ...] = tuple( - e for e in function_result.lifecycle_events if e.kind == LifeCycleEventKind.SQL - ) - if not function_sql_events: - continue - function_entry: FunctionPlanEntry | None = function_entry_map.get( - function_result.function_name - ) - if function_entry is None: - continue - function_run_path: Path = run_dir / _function_output_path( - relative_path=function_entry.relative_path, - language=function_entry.language, - ) - _write_sql( - path=function_run_path, - sql="\n\n".join(_format_statement(e.content) for e in function_sql_events), - ) - - -def write_test_runtime_target( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - results: tuple[SqlTestExecutionResult, ...], -) -> None: - """Write executed SQL unit-test statements under target/run/tests.""" - - run_dir: Path = target_dir / _RUN_DIR - result_keys: frozenset[tuple[str, str | None, int | None, str | None]] = frozenset( - ( - result.test_name, - result.source_path.as_posix() if result.source_path is not None else None, - result.block_index, - result.case_name, - ) - for result in results - ) - entry: SqlTestPlanEntry - for entry in plan_output.test_entries: - entry_key: tuple[str, str | None, int | None, str | None] = ( - entry.name, - entry.source_path.as_posix() if entry.source_path is not None else None, - entry.block_index, - entry.case_name, - ) - if entry_key not in result_keys: - continue - test_run_path: Path = run_dir / _TESTS_DIR / sql_test_output_path(entry) - _write_sql( - path=test_run_path, - sql=build_sql_test_comparison_sql( - test_entry=entry, - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), - ), - ) - - -def write_python_check_runtime_target( - *, target_dir: Path, results: tuple[PythonCheckExecutionResult, ...] -) -> None: - """Write Python check runtime results under target/run/checks.""" - - if not results: - return - run_path: Path = target_dir / _RUN_DIR / _CHECKS_DIR / "python_checks.json" - run_path.parent.mkdir(parents=True, exist_ok=True) - payload: dict[str, object] = { - "checks": [ - { - "kind": "python_check", - "name": result.node_name, - "display_name": result.node_name, - "check_id": f"python_check:{result.node_name}", - "status": "pass" if result.passed else "warn" if result.warned else "fail", - "passed": result.passed, - "severity": result.severity.value, - "message": result.message, - "error_message": result.error_message, - "metadata": result.metadata, - } - for result in results - ] - } - run_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") - - -def _write_sql(*, path: Path, sql: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(sql.rstrip() + "\n", encoding="utf-8") - - -def _model_output_path(relative_path: Path) -> Path: - parts: tuple[str, ...] = relative_path.parts - if parts and parts[0] == _MODELS_DIR: - return Path(*parts) - return Path(_MODELS_DIR) / relative_path - - -def _function_output_path(*, relative_path: Path, language: FunctionLanguage) -> Path: - parts: tuple[str, ...] = relative_path.parts - language_dir: str = language.value - function_language_path_part_count: int = 2 - if ( - len(parts) >= function_language_path_part_count - and parts[0] == _FUNCTIONS_DIR - and parts[1] == language_dir - ): - return Path(*parts).with_suffix(_SQL_FILE_SUFFIX) - return (Path(_FUNCTIONS_DIR) / language_dir / relative_path).with_suffix(_SQL_FILE_SUFFIX) - - -def _format_statement(statement: str) -> str: - stripped: str = statement.rstrip() - if not stripped: - return statement - if stripped.endswith(";"): - return stripped - return f"{stripped};" diff --git a/src/sqlbuild/cli/target_artifacts/main/__init__.py b/src/sqlbuild/cli/target_artifacts/main/__init__.py deleted file mode 100644 index 5978e9c4e4..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Public CLI target artifact entrypoints.""" diff --git a/src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py b/src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py deleted file mode 100644 index cb92880475..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/_write_python_check_runtime_target.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public Python check runtime target writing entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.target_artifacts._helpers.runtime import ( - write_python_check_runtime_target as _write_python_check_runtime_target, -) -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult - - -def write_python_check_runtime_target( - *, target_dir: Path, results: tuple[PythonCheckExecutionResult, ...] -) -> None: - """Write Python check runtime results under target/run/checks.""" - - _write_python_check_runtime_target(target_dir=target_dir, results=results) diff --git a/src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py b/src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py deleted file mode 100644 index 292133e1de..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/_write_runtime_target.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public runtime target writing entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.target_artifacts._helpers.runtime import ( - write_runtime_target as _write_runtime_target, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.build.models import BuildExecutionResult - - -def write_runtime_target( - *, target_dir: Path, plan_output: PlanOutput, result: BuildExecutionResult -) -> None: - """Write executed model lifecycle SQL under target/run.""" - - _write_runtime_target(target_dir=target_dir, plan_output=plan_output, result=result) diff --git a/src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py b/src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py deleted file mode 100644 index 06d8b1fd2d..0000000000 --- a/src/sqlbuild/cli/target_artifacts/main/_write_test_runtime_target.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public SQL test runtime target writing entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.target_artifacts._helpers.runtime import ( - write_test_runtime_target as _write_test_runtime_target, -) -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.testing.models import SqlTestExecutionResult - - -def write_test_runtime_target( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - results: tuple[SqlTestExecutionResult, ...], -) -> None: - """Write executed SQL unit-test statements under target/run/tests.""" - - _write_test_runtime_target( - target_dir=target_dir, - adapter=adapter, - plan_output=plan_output, - results=results, - ) diff --git a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py index c65d50fd21..e4d9c27550 100644 --- a/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/target_artifacts/main/write_runtime_target/test_runtime_target_writer.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target +from sqlbuild._native import write_runtime_target from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.executor.build.models import BuildExecutionResult from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( From 49652bb759234f286974cbb650a59b3e0ad2412c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:04:54 +0100 Subject: [PATCH 04/32] chore: record the dbt subprocess, profile loading and output capture stays --- python-boundary.toml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python-boundary.toml b/python-boundary.toml index 503fd8bbd2..a99bd3a406 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -299,6 +299,36 @@ path = "integrations/dbt/main/runtime/_execute_json_event_stream.py" reason = "user-code" note = "Entry point of the dbt subprocess runner." +[[stays]] +path = "integrations/dbt/classes/dbt_runner.py" +reason = "user-code" +note = "Runs dbt compile, deps, debug and ls as subprocesses over the user's dbt project." + +[[stays]] +path = "integrations/dbt/_helpers/profile/load.py" +reason = "user-code" +note = "Loads the user's dbt_project.yml and profiles.yml with PyYAML." + +[[stays]] +path = "integrations/dbt/_helpers/profile/render.py" +reason = "user-code" +note = "Renders the selected dbt profile output with Jinja, env_var and var as dbt does." + +[[stays]] +path = "cli/commands/main/entrypoint/_dispatch_with_compute_logs.py" +reason = "user-code" +note = "Captures the command's stdout, stderr and logging into the local compute log around user code." + +[[stays]] +path = "cli/commands/main/entrypoint/_dispatch_with_output_capture.py" +reason = "user-code" +note = "Tees the command's Python output streams to configured output capture sinks." + +[[stays]] +path = "cli/commands/_helpers/entry/compute_log_diagnostics.py" +reason = "user-code" +note = "Logs compute log capture failures through Python logging during capture." + [[stays]] path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" From 41ea35c7c3109ea83ca62151513067c7851ca65a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:25:33 +0100 Subject: [PATCH 05/32] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index a99bd3a406..1df4a81de8 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 51236 +remaining_lines = 49845 [[stays]] path = "executor" From 2ad036f442cf97b82fd3920f85409223ed5a1d86 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:16:58 +0100 Subject: [PATCH 06/32] refactor: compute planner identities, stale-selection warnings and execution order natively Port planner version identity hashing (model metadata JSON, declared-column, function and seed identities, graph identity composition and honest write hashes), selection staleness classification, execution edges and their origins, topological order and buildability into sqlbuild-analysis planner_identity with PyO3 glue. Change detection now calls the identity helpers directly instead of through Python. Delete the Python modules they replace and pin persisted identity values in Rust and Python tests. --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../_helpers/compose_graph_identity.rs | 13 + .../_helpers/cycle_message.rs | 65 +++ .../_helpers/dependency_counts.rs | 35 ++ .../_helpers/identity_resolver.rs | 132 +++++ .../src/planner_identity/_helpers/mod.rs | 5 + .../_helpers/staleness_walk.rs | 78 +++ .../src/planner_identity/constants.rs | 77 +++ .../main/classify_selection_staleness.rs | 37 ++ .../main/expected_graph_identity_hashes.rs | 14 + .../main/function_fingerprint_sql.rs | 26 + .../main/graph_write_identity_hashes.rs | 25 + .../main/missing_upstream_message.rs | 70 +++ .../main/missing_upstreams.rs | 51 ++ .../src/planner_identity/main/mod.rs | 10 + .../main/model_local_identity_hash.rs | 8 + .../main/stable_identity_hash.rs | 15 + .../main/stale_upstream_warning_message.rs | 25 + .../main/topological_order.rs | 45 ++ .../src/planner_identity/mod.rs | 10 + .../src/planner_identity/models.rs | 76 +++ .../src/planner_identity/tests.rs | 6 + .../src/planner_identity/tests/helpers.rs | 89 +++ .../tests/test_planner_identity.rs | 392 ++++++++++++++ .../src/planner_identity/tests/test_types.rs | 63 +++ .../src/planner_identity/types.rs | 6 + .../src/bindings/_helpers/graph/algorithms.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_changes/detect.rs | 158 +++--- .../_helpers/planner_changes/renames.rs | 21 +- .../_helpers/planner_identity/graph.rs | 415 ++++++++++++++ .../_helpers/planner_identity/hashing.rs | 89 +++ .../_helpers/planner_identity/identities.rs | 394 ++++++++++++++ .../_helpers/planner_identity/json_values.rs | 150 +++++ .../_helpers/planner_identity/metadata.rs | 512 ++++++++++++++++++ .../bindings/_helpers/planner_identity/mod.rs | 7 + .../_helpers/planner_identity/registration.rs | 15 + .../_helpers/planner_identity/staleness.rs | 290 ++++++++++ .../src/bindings/main/register.rs | 7 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 96 +++- .../_helpers/command_planning/static.py | 10 +- .../planner/_helpers/graph/buildability.py | 97 ---- .../compiler/planner/_helpers/graph/core.py | 244 --------- .../planner/_helpers/graph/model_closure.py | 29 - .../compiler/planner/_helpers/graph/scope.py | 10 +- .../planner/_helpers/identity/direct.py | 178 ------ .../planner/_helpers/identity/functions.py | 100 ---- .../planner/_helpers/identity/graph.py | 44 -- .../planner/_helpers/identity/hashing.py | 109 ---- .../planner/_helpers/identity/honest.py | 256 --------- .../_helpers/identity/model_metadata.py | 88 --- .../planner/_helpers/identity/seed.py | 75 --- .../planner/_helpers/migrations/columns.py | 3 +- .../planner/_helpers/migrations/discovery.py | 12 +- .../planner/_helpers/planning/buildability.py | 8 +- .../planner/_helpers/planning/identities.py | 5 +- .../_helpers/planning/output_assembly.py | 4 +- .../_helpers/planning/reconciliation.py | 5 +- .../_helpers/pruning/selection_classifier.py | 150 ----- .../_helpers/pruning/selection_staleness.py | 229 -------- .../compiler/planner/_helpers/reuse/clone.py | 16 +- .../planner/_helpers/scenario/relations.py | 10 +- .../planner/_helpers/warehouse/snapshot.py | 4 +- .../classes/graph_identity_resolver.py | 85 --- .../main/changes/_selection_staleness.py | 16 - .../planner/main/execution/display_plan.py | 11 +- .../planner/main/identity/__init__.py | 0 .../planner/main/identity/_graph_identity.py | 26 - .../main/identity/_graph_write_identity.py | 30 - .../main/identity/_version_identity_config.py | 53 -- .../identity/_version_identity_metadata.py | 32 -- .../version_identity_function_hashes.py | 52 -- .../version_identity_model_metadata.py | 87 --- .../selection/_model_downstream_closure.py | 19 - .../planner/main/selection/_upstream.py | 4 +- .../source_freshness/_helpers/propagation.py | 6 +- .../compiler/planner/main/_test_types.py | 2 +- .../sqlbuild/compiler/planner/main/helpers.py | 4 +- .../planner/main/test_build_execution_plan.py | 19 +- .../compiler/compile/_helpers/helpers.py | 2 +- .../compile/_helpers/test_declarations.py | 4 +- .../compile/_helpers/test_model_schemas.py | 4 +- .../compiler/planner/_helpers/_test_types.py | 43 +- .../planner/_helpers/changes/_test_helpers.py | 5 +- .../planner/_helpers/changes/test_detect.py | 4 +- .../compiler/planner/_helpers/conftest.py | 10 +- .../compiler/planner/_helpers/helpers.py | 110 ++-- .../identity/test_definition_hashes.py | 16 +- .../test_merge_recomputed_model_changes.py | 2 +- .../planner/_helpers/test_buildability.py | 5 +- .../_helpers/test_function_fingerprints.py | 14 +- .../compiler/planner/_helpers/test_graph.py | 15 +- .../planner/_helpers/test_graph_identity.py | 259 --------- .../planner/_helpers/test_model_closure.py | 4 +- .../planner/_helpers/test_seed_identity.py | 2 +- .../_helpers/test_selection_staleness.py | 85 +-- .../planner/_helpers/test_selectors.py | 3 +- .../sqlbuild/compiler/planner/main/helpers.py | 9 +- .../planner/main/identity/_test_types.py | 9 + .../compiler/planner/main/identity/helpers.py | 74 ++- .../identity/test_persisted_identity_pins.py | 83 +++ .../identity/test_version_identity_config.py | 6 +- .../test_version_identity_function_hashes.py | 4 +- .../planner/main/test_selection_staleness.py | 4 +- 105 files changed, 3728 insertions(+), 2713 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/planner_identity/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/buildability.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/core.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/direct.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/functions.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/graph.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/hashing.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/honest.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/identity/seed.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py delete mode 100644 src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py delete mode 100644 src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/_graph_identity.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py delete mode 100644 src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py delete mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 84e825737a..3cf976f9b6 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,7 @@ pub mod lineage_command; pub mod lineage_graph; pub mod manifest; pub mod planner_changes; +pub mod planner_identity; pub mod planner_strategy; pub mod python_plan; pub mod python_selection; diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs new file mode 100644 index 0000000000..a8e54ec089 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/compose_graph_identity.rs @@ -0,0 +1,13 @@ +//! A node identity composed from its local hash and the hashes of its available upstreams. + +use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; + +/// The hash of the local hash and each upstream hash, one per line. +pub(crate) fn compose_graph_identity(local_hash: &str, upstream_hashes: &[&str]) -> String { + let mut text: String = local_hash.to_owned(); + for upstream_hash in upstream_hashes { + text.push('\n'); + text.push_str(upstream_hash); + } + stable_identity_hash(&text) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs new file mode 100644 index 0000000000..bf476b3d2a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/cycle_message.rs @@ -0,0 +1,65 @@ +//! The dependency-cycle message naming the keys left unordered and the injected edges between them. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_identity::models::GraphKey; + +/// Upstream edges and the origins of injected edges. +type CycleEdges<'a> = ( + &'a [(GraphKey, Vec)], + &'a HashMap<(GraphKey, GraphKey), String>, +); + +/// The message for the keys Kahn's algorithm could not order. +pub(crate) fn cycle_message( + ordered: &[GraphKey], + node_keys: HashSet<&GraphKey>, + (upstream, injected_edge_origins): CycleEdges<'_>, +) -> String { + let ordered_set: HashSet<&GraphKey> = ordered.iter().collect(); + let mut cycle_keys: Vec<&GraphKey> = node_keys + .into_iter() + .filter(|key| !ordered_set.contains(key)) + .collect(); + cycle_keys.sort(); + let names: Vec = cycle_keys + .iter() + .map(|key| format!("{}:{}", key.kind, key.name)) + .collect(); + format!( + "Dependency cycle detected involving: {}{}", + names.join(", "), + cycle_origin_notes(&cycle_keys, upstream, injected_edge_origins) + ) +} + +fn cycle_origin_notes( + cycle_keys: &[&GraphKey], + upstream: &[(GraphKey, Vec)], + injected_edge_origins: &HashMap<(GraphKey, GraphKey), String>, +) -> String { + if injected_edge_origins.is_empty() { + return String::new(); + } + let cycle_set: HashSet<&GraphKey> = cycle_keys.iter().copied().collect(); + let deps: HashMap<&GraphKey, &Vec> = + upstream.iter().map(|(key, deps)| (key, deps)).collect(); + let mut notes: BTreeSet<&str> = BTreeSet::new(); + for key in cycle_keys { + for dep_key in deps.get(key).into_iter().flat_map(|deps| deps.iter()) { + if !cycle_set.contains(dep_key) { + continue; + } + if let Some(origin) = injected_edge_origins.get(&((*key).clone(), dep_key.clone())) { + notes.insert(origin); + } + } + } + if notes.is_empty() { + return String::new(); + } + format!( + " (via {})", + notes.into_iter().collect::>().join("; ") + ) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs new file mode 100644 index 0000000000..ff60300336 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/dependency_counts.rs @@ -0,0 +1,35 @@ +//! Kahn's algorithm inputs: every key, each key's upstream count and its downstream keys. + +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::models::GraphKey; + +/// Every key named in the edges, its number of upstream entries and its downstream keys, one per edge. +pub(crate) struct DependencyCounts<'a> { + pub(crate) node_keys: HashSet<&'a GraphKey>, + pub(crate) indegree: HashMap<&'a GraphKey, usize>, + pub(crate) downstream: HashMap<&'a GraphKey, Vec<&'a GraphKey>>, +} + +/// The counts of `upstream`, counting repeated upstream entries once per entry. +pub(crate) fn dependency_counts(upstream: &[(GraphKey, Vec)]) -> DependencyCounts<'_> { + let mut counts = DependencyCounts { + node_keys: HashSet::new(), + indegree: HashMap::new(), + downstream: HashMap::new(), + }; + for (key, dep_keys) in upstream { + counts.node_keys.insert(key); + for dep_key in dep_keys { + counts.node_keys.insert(dep_key); + counts.downstream.entry(dep_key).or_default().push(key); + } + } + for key in &counts.node_keys { + counts.indegree.insert(key, 0); + } + for (key, dep_keys) in upstream { + counts.indegree.insert(key, dep_keys.len()); + } + counts +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs new file mode 100644 index 0000000000..de8f20a2c2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs @@ -0,0 +1,132 @@ +//! Memoized identity resolution over one dependency graph, each node once. + +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::_helpers::compose_graph_identity::compose_graph_identity; +use crate::planner_identity::models::{GraphKey, IdentityNode}; +use crate::planner_identity::types::IdentityHashes; + +/// Which hashes a resolution composes from. +#[derive(Clone, Copy)] +enum Mode { + Expected, + Write, +} + +/// Expected or selected-write identity resolution state. +pub(crate) struct IdentityResolver<'a> { + nodes: &'a HashMap, + selected: Option<&'a HashSet>, + hashes: IdentityHashes, + positions: HashMap, + resolved_selected: HashMap, + visiting: HashSet, +} + +impl<'a> IdentityResolver<'a> { + pub(crate) fn new( + nodes: &'a HashMap, + selected: Option<&'a HashSet>, + base_hashes: Vec<(GraphKey, String)>, + ) -> Self { + let mut resolver = Self { + nodes, + selected, + hashes: Vec::new(), + positions: HashMap::new(), + resolved_selected: HashMap::new(), + visiting: HashSet::new(), + }; + for (key, hash) in base_hashes { + resolver.record(&key, Some(hash)); + } + resolver + } + + /// Every hash recorded so far, in first-recorded order. + pub(crate) fn into_hashes(self) -> IdentityHashes { + self.hashes + } + + /// A node's expected hash, composed from its upstream expected hashes. + pub(crate) fn resolve_expected(&mut self, key: &GraphKey) -> Option { + if let Some(position) = self.positions.get(key) { + return self.hashes[*position].1.clone(); + } + let nodes: &'a HashMap = self.nodes; + let Some((node, local_hash)) = nodes + .get(key) + .and_then(|node| node.local_hash.as_deref().map(|local| (node, local))) + else { + self.record(key, None); + return None; + }; + if self.visiting.contains(key) { + self.record(key, Some(local_hash.to_owned())); + return Some(local_hash.to_owned()); + } + let composed: String = self.compose(key, (node, local_hash), Mode::Expected); + self.record(key, Some(composed.clone())); + Some(composed) + } + + /// A selected node's hash recomputed from upstream write hashes; other nodes keep theirs. + pub(crate) fn resolve_write(&mut self, key: &GraphKey) -> Option { + if !self.selected.is_some_and(|selected| selected.contains(key)) { + return self.recorded(key); + } + if let Some(hash) = self.resolved_selected.get(key) { + return Some(hash.clone()); + } + let nodes: &'a HashMap = self.nodes; + let Some((node, local_hash)) = nodes + .get(key) + .and_then(|node| node.local_hash.as_deref().map(|local| (node, local))) + else { + return self.recorded(key); + }; + if self.visiting.contains(key) { + return Some(local_hash.to_owned()); + } + let composed: String = self.compose(key, (node, local_hash), Mode::Write); + self.record(key, Some(composed.clone())); + self.resolved_selected.insert(key.clone(), composed.clone()); + Some(composed) + } + + fn compose( + &mut self, + key: &GraphKey, + (node, local_hash): (&IdentityNode, &str), + mode: Mode, + ) -> String { + self.visiting.insert(key.clone()); + let mut upstream_hashes: Vec = Vec::new(); + for upstream_key in &node.upstream_keys { + let upstream_hash: Option = match mode { + Mode::Expected => self.resolve_expected(upstream_key), + Mode::Write => self.resolve_write(upstream_key), + }; + upstream_hashes.extend(upstream_hash); + } + self.visiting.remove(key); + let upstream: Vec<&str> = upstream_hashes.iter().map(String::as_str).collect(); + compose_graph_identity(local_hash, &upstream) + } + + fn recorded(&self, key: &GraphKey) -> Option { + self.positions + .get(key) + .and_then(|position| self.hashes[*position].1.clone()) + } + + fn record(&mut self, key: &GraphKey, hash: Option) { + match self.positions.get(key) { + Some(position) => self.hashes[*position].1 = hash, + None => { + self.positions.insert(key.clone(), self.hashes.len()); + self.hashes.push((key.clone(), hash)); + } + } + } +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs new file mode 100644 index 0000000000..7cf4c87d38 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod compose_graph_identity; +pub(crate) mod cycle_message; +pub(crate) mod dependency_counts; +pub(crate) mod identity_resolver; +pub(crate) mod staleness_walk; diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs new file mode 100644 index 0000000000..7a4cb1ff1f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs @@ -0,0 +1,78 @@ +//! The upstream staleness walk of one selected model. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_identity::constants::{MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE}; +use crate::planner_identity::models::{GraphKey, StalenessGraph}; + +/// The upstream walk of one selected model. +pub(crate) struct StalenessWalk<'a> { + pub(crate) graph: &'a StalenessGraph, + pub(crate) model_key: &'a GraphKey, + pub(crate) names: BTreeSet, + pub(crate) visiting: HashSet, + pub(crate) stale: HashMap, +} + +impl StalenessWalk<'_> { + pub(crate) fn visit(&mut self, key: &GraphKey) -> bool { + if let Some(cached) = self.stale.get(key) { + return *cached; + } + if self.visiting.contains(key) { + return false; + } + self.visiting.insert(key.clone()); + let graph: &StalenessGraph = self.graph; + let is_stale: bool = match key.kind.as_str() { + MODEL_RESOURCE => { + let in_run_set: bool = graph.run.models.contains(&key.name); + if graph.changed.models.contains(&key.name) && !in_run_set { + self.names.insert(key.name.clone()); + true + } else { + let mut ancestor_stale: bool = false; + for parent_key in graph.upstream_deps.get(key).into_iter().flatten() { + let parent_stale: bool = self.visit(parent_key); + ancestor_stale = parent_stale || ancestor_stale; + if run_parent_changed(graph, parent_key) { + ancestor_stale = true; + } + } + if ancestor_stale && !in_run_set && key != self.model_key { + self.names.insert(key.name.clone()); + } + ancestor_stale + } + } + SEED_RESOURCE => self.leaf(key, (&graph.changed.seeds, &graph.run.seeds)), + SOURCE_RESOURCE => self.leaf(key, (&graph.changed.sources, &graph.run.sources)), + _ => false, + }; + self.visiting.remove(key); + self.stale.insert(key.clone(), is_stale); + is_stale + } + + fn leaf( + &mut self, + key: &GraphKey, + (changed, run): (&HashSet, &HashSet), + ) -> bool { + let is_changed: bool = changed.contains(&key.name); + if is_changed && !run.contains(&key.name) { + self.names.insert(key.name.clone()); + } + is_changed + } +} + +fn run_parent_changed(graph: &StalenessGraph, parent_key: &GraphKey) -> bool { + let (run, changed): (&HashSet, &HashSet) = match parent_key.kind.as_str() { + MODEL_RESOURCE => (&graph.run.models, &graph.changed.models), + SEED_RESOURCE => (&graph.run.seeds, &graph.changed.seeds), + SOURCE_RESOURCE => (&graph.run.sources, &graph.changed.sources), + _ => return false, + }; + run.contains(&parent_key.name) && changed.contains(&parent_key.name) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/constants.rs b/crates/sqlbuild-analysis/src/planner_identity/constants.rs new file mode 100644 index 0000000000..3faff6781e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/constants.rs @@ -0,0 +1,77 @@ +//! Planner identity constants shared with the Python planner. + +/// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`. +pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 34] = [ + "append_cursor_inclusive", + "batch_size", + "check_columns", + "cursor", + "cursor_grain", + "cursor_start", + "cursor_end", + "cursor_start_max_ahead", + "cursor_start_max_action", + "cursor_future_max_distance", + "cursor_future_action", + "cursor_type", + "historical_input", + "incremental_mode", + "microbatch_strategy", + "cursor_watermark_mode", + "max_microbatches", + "microbatch_limit", + "incremental_strategy", + "merge_exclude_columns", + "full_refresh", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "observed_at", + "on_schema_change", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", +]; +/// Model config key of cursor inputs, recorded before every other identity key. +pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; +/// Model config key of user custom config. +pub const MODEL_CUSTOM_CONFIG_KEY: &str = "config"; +/// Model config key of custom placeholders. +pub const MODEL_PLACEHOLDERS_CONFIG_KEY: &str = "placeholders"; +/// Model config key of pre-hooks. +pub const MODEL_PRE_HOOKS_CONFIG_KEY: &str = "pre_hooks"; +/// Model config key of post-hooks. +pub const MODEL_POST_HOOKS_CONFIG_KEY: &str = "post_hooks"; +/// Model config key of the contract policy. +pub const CONTRACT_CONFIG_KEY: &str = "contract"; +/// The contract policy that enforces declared columns. +pub const ENFORCED_CONTRACT_POLICY: &str = "enforced"; +/// Hook fields that participate in SQL hook identity. +pub const SQL_HOOK_IDENTITY_FIELDS: [&str; 4] = ["statement", "name", "definition_sql", "kwargs"]; +/// Error code of a model or function that cannot be fingerprinted. +pub const QUERY_FINGERPRINT_FAILED: &str = "S025"; +/// Help text of a model or function that cannot be fingerprinted. +pub const FINGERPRINT_HELP: &str = "SQLBuild fingerprints the dialect token stream of compiled SQL to detect changes. Fix the SQL so the warehouse dialect tokenizer accepts it."; +/// Resource type of models. +pub const MODEL_RESOURCE: &str = "model"; +/// Resource type of seeds. +pub const SEED_RESOURCE: &str = "seed"; +/// Resource type of sources. +pub const SOURCE_RESOURCE: &str = "source"; +/// Resource type of scalar functions. +pub const UDF_RESOURCE: &str = "udf"; +/// Resource type of table functions. +pub const TABLE_FN_RESOURCE: &str = "table_fn"; +/// Resource type of SQL tests. +pub const SQL_TEST_RESOURCE: &str = "sql_test"; +/// Resource type of dbt references. +pub const DBT_REF_RESOURCE: &str = "dbt_ref"; +/// At most this many stale triggers are listed in one warning. +pub const STALE_TRIGGER_DISPLAY_LIMIT: usize = 5; +/// At most this many items are named in a missing-upstream message. +pub const MISSING_MESSAGE_ITEM_LIMIT: usize = 5; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs b/crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs new file mode 100644 index 0000000000..9cde7e0cdf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/classify_selection_staleness.rs @@ -0,0 +1,37 @@ +//! Selected models stale through changed upstreams outside the run set. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use crate::planner_identity::_helpers::staleness_walk::StalenessWalk; + +use crate::planner_identity::constants::MODEL_RESOURCE; +use crate::planner_identity::models::{GraphKey, StaleWarning, StalenessGraph}; + +/// One warning per selected model, by name, whose upstream closure reaches a changed node that does not run. +pub fn classify_selection_staleness(graph: &StalenessGraph) -> Vec { + let selected: BTreeSet<&String> = graph.selected_model_names.iter().collect(); + let mut warnings: Vec = Vec::new(); + for model_name in selected { + let model_key = GraphKey { + kind: MODEL_RESOURCE.to_owned(), + name: model_name.clone(), + }; + let mut walk = StalenessWalk { + graph, + model_key: &model_key, + names: BTreeSet::new(), + visiting: HashSet::new(), + stale: HashMap::new(), + }; + for upstream_key in graph.upstream_deps.get(&model_key).into_iter().flatten() { + let _ = walk.visit(upstream_key); + } + if !walk.names.is_empty() { + warnings.push(StaleWarning { + model_name: model_name.clone(), + trigger_names: walk.names.into_iter().collect(), + }); + } + } + warnings +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs b/crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs new file mode 100644 index 0000000000..7535405f79 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/expected_graph_identity_hashes.rs @@ -0,0 +1,14 @@ +//! Expected identity hashes of every node of a dependency graph. + +use crate::planner_identity::_helpers::identity_resolver::IdentityResolver; +use crate::planner_identity::models::IdentityGraph; +use crate::planner_identity::types::IdentityHashes; + +/// Each node's hash composed from its upstream expected hashes, in first-recorded order; `None` for nodes without a local hash. +pub fn expected_graph_identity_hashes(graph: &IdentityGraph) -> IdentityHashes { + let mut resolver = IdentityResolver::new(&graph.nodes, None, Vec::new()); + for key in &graph.execution_order { + let _ = resolver.resolve_expected(key); + } + resolver.into_hashes() +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs b/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs new file mode 100644 index 0000000000..8317a9fbf2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs @@ -0,0 +1,26 @@ +//! Stable function definition text used for change fingerprints. + +use crate::planner_identity::models::FunctionFingerprintFields; + +/// The language, signature, runtime fields and body, one field per line. +pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String { + let render = |pairs: &[(String, String)]| -> String { + pairs + .iter() + .map(|(name, kind)| format!("{name}:{kind}")) + .collect::>() + .join(",") + }; + [ + format!("language={}", fields.language), + format!("arguments={}", render(&fields.arguments)), + format!("returns={}", fields.returns), + format!("return_columns={}", render(&fields.return_columns)), + format!("runtime_version={}", fields.runtime_version), + format!("entry_point={}", fields.entry_point), + format!("packages={}", fields.packages.join(",")), + "body=".to_owned(), + fields.body_sql.clone(), + ] + .join("\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs b/crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs new file mode 100644 index 0000000000..88b507fa00 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/graph_write_identity_hashes.rs @@ -0,0 +1,25 @@ +//! Write identity hashes from the upstream hashes actually available this run. + +use std::collections::HashSet; + +use crate::planner_identity::_helpers::identity_resolver::IdentityResolver; +use crate::planner_identity::models::{GraphKey, IdentityGraph}; + +/// The base hashes with each selected node recomposed from upstream write hashes, in first-recorded order. +pub fn graph_write_identity_hashes( + graph: &IdentityGraph, + selected_keys: &HashSet, + base_hashes: Vec<(GraphKey, String)>, +) -> Vec<(GraphKey, String)> { + let mut resolver = IdentityResolver::new(&graph.nodes, Some(selected_keys), base_hashes); + for key in &graph.execution_order { + let _ = resolver.resolve_write(key); + } + let mut hashes: Vec<(GraphKey, String)> = Vec::new(); + for (key, hash) in resolver.into_hashes() { + if let Some(hash) = hash { + hashes.push((key, hash)); + } + } + hashes +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs new file mode 100644 index 0000000000..5afe8280f0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs @@ -0,0 +1,70 @@ +//! The error message of a selected scope with missing upstreams. + +use std::collections::HashMap; + +use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT; +use crate::planner_identity::models::{GraphKey, MissingUpstream}; + +/// Missing lineage upstreams by name and, when only an audit or hook read requires one, that read's origin. +pub fn missing_upstream_message( + missing: &[MissingUpstream], + edge_origins: &HashMap<(GraphKey, GraphKey), String>, +) -> String { + let mut lineage_missing: Vec<&MissingUpstream> = Vec::new(); + let mut read_names: Vec<&str> = Vec::new(); + for entry in missing { + let origins: Vec<&str> = entry + .required_by + .iter() + .filter_map(|dependent| { + edge_origins + .get(&(dependent.clone(), entry.key.clone())) + .map(String::as_str) + }) + .collect(); + if origins.len() < entry.required_by.len() { + lineage_missing.push(entry); + continue; + } + for origin in origins { + if !read_names.contains(&origin) { + read_names.push(origin); + } + } + } + let mut parts: Vec = Vec::new(); + if !lineage_missing.is_empty() { + let names: Vec<&str> = lineage_missing + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .map(|entry| entry.key.name.as_str()) + .collect(); + parts.push(format!( + "{} missing upstream dependencies ({})", + lineage_missing.len(), + names.join(", ") + )); + } + if let [name] = read_names.as_slice() { + parts.push(format!( + "{name}, which is not selected and does not exist in the warehouse; select it or build it first" + )); + } else if !read_names.is_empty() { + let shown: String = read_names + .iter() + .take(MISSING_MESSAGE_ITEM_LIMIT) + .copied() + .collect::>() + .join("; "); + let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT { + "; ..." + } else { + "" + }; + parts.push(format!( + "{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first", + read_names.len() + )); + } + format!("cannot build selected scope: {}", parts.join("; ")) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs new file mode 100644 index 0000000000..59cb1c90a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs @@ -0,0 +1,51 @@ +//! Upstream dependencies of selected keys that neither scope nor warehouse provides. + +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::constants::{DBT_REF_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE}; +use crate::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; + +/// Missing upstreams, most dependents first, then by `(kind, name)`; dependents sorted by `(kind, name)`. +pub fn missing_upstreams( + selected_keys: &[GraphKey], + upstream_deps: &HashMap>, + available: &AvailableUpstreams, +) -> Vec { + let selected: HashSet<&GraphKey> = selected_keys.iter().collect(); + let mut missing: HashMap> = HashMap::new(); + for selected_key in selected_keys { + for dep_key in upstream_deps.get(selected_key).into_iter().flatten() { + if [SQL_TEST_RESOURCE, DBT_REF_RESOURCE, SOURCE_RESOURCE] + .contains(&dep_key.kind.as_str()) + || selected.contains(dep_key) + || available.satisfied_keys.contains(dep_key) + || available.existing_relations.contains(&dep_key.name) + || available + .deferred_relations + .as_ref() + .is_some_and(|deferred| deferred.contains(&dep_key.name)) + { + continue; + } + missing + .entry(dep_key.clone()) + .or_default() + .push(selected_key.clone()); + } + } + let mut entries: Vec = missing + .into_iter() + .map(|(key, mut required_by)| { + required_by.sort(); + MissingUpstream { key, required_by } + }) + .collect(); + entries.sort_by(|left, right| { + right + .required_by + .len() + .cmp(&left.required_by.len()) + .then_with(|| left.key.cmp(&right.key)) + }); + entries +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs new file mode 100644 index 0000000000..219d458e53 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/mod.rs @@ -0,0 +1,10 @@ +pub mod classify_selection_staleness; +pub mod expected_graph_identity_hashes; +pub mod function_fingerprint_sql; +pub mod graph_write_identity_hashes; +pub mod missing_upstream_message; +pub mod missing_upstreams; +pub mod model_local_identity_hash; +pub mod stable_identity_hash; +pub mod stale_upstream_warning_message; +pub mod topological_order; diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs b/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs new file mode 100644 index 0000000000..11148c52dc --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/model_local_identity_hash.rs @@ -0,0 +1,8 @@ +//! A model's local identity hash from its query fingerprint and non-query metadata. + +use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; + +/// The hash of the query fingerprint and metadata JSON, one per line. +pub fn model_local_identity_hash(query_fingerprint: &str, metadata_json: &str) -> String { + stable_identity_hash(&format!("{query_fingerprint}\n{metadata_json}")) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs b/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs new file mode 100644 index 0000000000..f119ec5c7f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/stable_identity_hash.rs @@ -0,0 +1,15 @@ +//! The SHA-256 hash used for local and composed version identities. + +use std::fmt::Write; + +use sha2::{Digest, Sha256}; + +/// The lowercase hex SHA-256 digest of the UTF-8 text. +pub fn stable_identity_hash(value: &str) -> String { + Sha256::digest(value.as_bytes()) + .iter() + .fold(String::with_capacity(64), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs b/crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs new file mode 100644 index 0000000000..612caef505 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/stale_upstream_warning_message.rs @@ -0,0 +1,25 @@ +//! A multi-line stale-selection warning: summary, capped bullet list and one hint. + +use crate::planner_identity::constants::STALE_TRIGGER_DISPLAY_LIMIT; + +/// The warning text for `model_name` building on stale `trigger_names`. +pub fn stale_upstream_warning_message( + (model_label, model_name): (&str, &str), + trigger_label: &str, + trigger_names: &[String], +) -> String { + let count: usize = trigger_names.len(); + let mut lines: Vec = vec![format!( + "{model_label} '{model_name}' will build on {count} stale {trigger_label} not selected for rebuild:" + )]; + for name in trigger_names.iter().take(STALE_TRIGGER_DISPLAY_LIMIT) { + lines.push(format!(" - {name}")); + } + if count > STALE_TRIGGER_DISPLAY_LIMIT { + lines.push(format!(" +{} more", count - STALE_TRIGGER_DISPLAY_LIMIT)); + } + lines.push(format!( + " rebuild the closure to refresh them: --select +{model_name}" + )); + lines.join("\n") +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs b/crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs new file mode 100644 index 0000000000..6a183eb1d7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/main/topological_order.rs @@ -0,0 +1,45 @@ +//! Stable dependency order with Kahn's algorithm, smallest `(kind, name)` first. + +use std::cmp::Reverse; +use std::collections::{BinaryHeap, HashMap}; + +use crate::planner_identity::_helpers::cycle_message::cycle_message; +use crate::planner_identity::_helpers::dependency_counts::{DependencyCounts, dependency_counts}; +use crate::planner_identity::models::GraphKey; + +/// Every key in dependency order, or the dependency-cycle message naming the keys left over and the injected edges between them. +pub fn topological_order( + upstream: &[(GraphKey, Vec)], + injected_edge_origins: &HashMap<(GraphKey, GraphKey), String>, +) -> Result, String> { + let DependencyCounts { + node_keys, + mut indegree, + downstream, + } = dependency_counts(upstream); + let mut ready: BinaryHeap> = indegree + .iter() + .filter(|(_, count)| **count == 0) + .map(|(key, _)| Reverse(*key)) + .collect(); + let mut ordered: Vec = Vec::with_capacity(node_keys.len()); + while let Some(Reverse(current)) = ready.pop() { + ordered.push(current.clone()); + for downstream_key in downstream.get(current).into_iter().flatten() { + if let Some(count) = indegree.get_mut(downstream_key) { + *count = count.saturating_sub(1); + if *count == 0 { + ready.push(Reverse(downstream_key)); + } + } + } + } + if ordered.len() == node_keys.len() { + return Ok(ordered); + } + Err(cycle_message( + &ordered, + node_keys, + (upstream, injected_edge_origins), + )) +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/mod.rs b/crates/sqlbuild-analysis/src/planner_identity/mod.rs new file mode 100644 index 0000000000..efa80954a5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/mod.rs @@ -0,0 +1,10 @@ +//! Planner identity, pruning and graph order: version identity hashes, dependency-aware identity composition, stale-selection warnings, buildability and execution order. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs new file mode 100644 index 0000000000..54a84b5fda --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -0,0 +1,76 @@ +//! Planner identity, pruning and graph-order data. + +use std::collections::{HashMap, HashSet}; + +/// A graph node identity: resource type (or fingerprint node type) and name, ordered as Python's `(resource_type, name)`. +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct GraphKey { + pub kind: String, + pub name: String, +} + +/// One node of an identity graph: its upstream keys in order and its local hash. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IdentityNode { + pub upstream_keys: Vec, + pub local_hash: Option, +} + +/// An identity graph: nodes by key and the order nodes are resolved in. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct IdentityGraph { + pub nodes: HashMap, + pub execution_order: Vec, +} + +/// A function definition's fingerprint fields, already rendered as text. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FunctionFingerprintFields { + pub language: String, + pub arguments: Vec<(String, String)>, + pub returns: String, + pub return_columns: Vec<(String, String)>, + pub runtime_version: String, + pub entry_point: String, + pub packages: Vec, + pub body_sql: String, +} + +/// Names that changed or run this time, per resource kind, for stale-selection classification. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StalenessNames { + pub models: HashSet, + pub seeds: HashSet, + pub sources: HashSet, +} + +/// Graph inputs of selected-model stale warning classification. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StalenessGraph { + pub upstream_deps: HashMap>, + pub selected_model_names: Vec, + pub run: StalenessNames, + pub changed: StalenessNames, +} + +/// A selected model that builds on changed upstreams outside the run set. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StaleWarning { + pub model_name: String, + pub trigger_names: Vec, +} + +/// What satisfies an upstream dependency of a selected key. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AvailableUpstreams { + pub satisfied_keys: HashSet, + pub existing_relations: HashSet, + pub deferred_relations: Option>, +} + +/// One upstream dependency missing from both scope and warehouse, with its dependents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MissingUpstream { + pub key: GraphKey, + pub required_by: Vec, +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests.rs b/crates/sqlbuild-analysis/src/planner_identity/tests.rs new file mode 100644 index 0000000000..a26a2fda78 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_identity.rs"] +mod planner_identity; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs new file mode 100644 index 0000000000..6b2872ee41 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs @@ -0,0 +1,89 @@ +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode, StalenessGraph}; + +pub(super) fn key(kind: &str, name: &str) -> GraphKey { + GraphKey { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +pub(super) fn model(name: &str) -> GraphKey { + key("model", name) +} + +pub(super) fn names(values: &[&str]) -> HashSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Nodes `a`, `b` over `a`, `c` over `a`, `b` and unknown `x`, and `d` over `c` without a local hash. +pub(super) fn identity_graph() -> IdentityGraph { + let node = |upstream: &[&str], local: Option<&str>| IdentityNode { + upstream_keys: upstream.iter().map(|name| model(name)).collect(), + local_hash: local.map(str::to_owned), + }; + IdentityGraph { + nodes: HashMap::from([ + (model("a"), node(&[], Some("la"))), + (model("b"), node(&["a"], Some("lb"))), + (model("c"), node(&["a", "b", "x"], Some("lc"))), + (model("d"), node(&["c"], None)), + ]), + execution_order: vec![model("c"), model("a"), model("b"), model("d")], + } +} + +pub(super) fn staleness_graph( + edges: &[(GraphKey, &[GraphKey])], + selected: &[&str], +) -> StalenessGraph { + StalenessGraph { + upstream_deps: edges + .iter() + .map(|(node, upstream)| (node.clone(), upstream.to_vec())) + .collect(), + selected_model_names: texts(selected), + ..StalenessGraph::default() + } +} + +/// Two models reading each other. +pub(super) fn cyclic_identity_graph() -> IdentityGraph { + let node = |upstream: &str, local: &str| IdentityNode { + upstream_keys: vec![model(upstream)], + local_hash: Some(local.to_owned()), + }; + IdentityGraph { + nodes: HashMap::from([(model("a"), node("b", "la")), (model("b"), node("a", "lb"))]), + execution_order: vec![model("a"), model("b")], + } +} + +/// A dense ladder: each layer's two nodes read both nodes of the previous layer. +pub(super) fn ladder_identity_graph(layer_count: usize) -> IdentityGraph { + let mut graph = IdentityGraph::default(); + let mut previous: Vec = Vec::new(); + for layer in 0..layer_count { + let current: Vec = ["a", "b"] + .iter() + .map(|suffix| model(&format!("L{layer}_{suffix}"))) + .collect(); + for key in ¤t { + graph.nodes.insert( + key.clone(), + IdentityNode { + upstream_keys: previous.clone(), + local_hash: Some(format!("local_{}", key.name)), + }, + ); + graph.execution_order.push(key.clone()); + } + previous = current; + } + graph +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs new file mode 100644 index 0000000000..69447c9b93 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -0,0 +1,392 @@ +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::_helpers::compose_graph_identity::compose_graph_identity; +use crate::planner_identity::main::classify_selection_staleness::classify_selection_staleness; +use crate::planner_identity::main::expected_graph_identity_hashes::expected_graph_identity_hashes; +use crate::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; +use crate::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; +use crate::planner_identity::main::missing_upstream_message::missing_upstream_message; +use crate::planner_identity::main::missing_upstreams::missing_upstreams; +use crate::planner_identity::main::model_local_identity_hash::model_local_identity_hash; +use crate::planner_identity::main::stable_identity_hash::stable_identity_hash; +use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; +use crate::planner_identity::main::topological_order::topological_order; +use crate::planner_identity::models::{ + AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, StaleWarning, + StalenessNames, +}; +use crate::planner_identity::tests::helpers::{ + cyclic_identity_graph, identity_graph, key, ladder_identity_graph, model, names, + staleness_graph, texts, +}; +use crate::planner_identity::tests::test_types::{ + BuildabilityTestCase, FunctionSqlTestCase, HashTestCase, IdentityGraphSizeTestCase, + IdentityGraphTestCase, OrderTestCase, StalenessTestCase, WarningTextTestCase, +}; + +#[test] +fn given_persisted_identity_inputs_when_hashing_then_matches_pinned_python_hashes() { + let test_cases = [ + HashTestCase { + description: "empty text", + actual: stable_identity_hash(""), + expected_hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + HashTestCase { + description: "unicode and carriage returns", + actual: stable_identity_hash("orders\r\ncafé ☃"), + expected_hash: "ba7e194792c57a2f2e3aa77441f680ac5823e5a19cbbef0113bd20d701aa4ab2", + }, + HashTestCase { + description: "composed with upstreams", + actual: compose_graph_identity("abc", &["d1", "d2"]), + expected_hash: "5be4a7558096ea1ae6d8c0f97fac33972a28951ded9ce7e9a2c1898c2b8389cc", + }, + HashTestCase { + description: "composed without upstreams", + actual: compose_graph_identity("abc", &[]), + expected_hash: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }, + HashTestCase { + description: "model local identity", + actual: model_local_identity_hash("q1", r#"{"config":{}}"#), + expected_hash: "3f16f63bb7e4c297529a7814cdce6737d9fcc1f3ad695905c58daa2438613c16", + }, + ]; + for test_case in test_cases { + assert_eq!( + test_case.actual, test_case.expected_hash, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_identity_graph_when_resolving_then_matches_pinned_python_hashes() { + let pinned = |hash: &str| hash.to_owned(); + let test_cases = [ + IdentityGraphTestCase { + description: "acyclic graph with an unknown upstream and a node without a local hash", + graph: identity_graph(), + selected: HashSet::from([model("b"), model("c")]), + base_hashes: vec![(model("a"), "base_a".to_owned())], + expected_hashes: vec![ + ( + model("a"), + Some(pinned( + "b737ca7ee563ae80e457bb3d1dfe64edd2b4c015a8f88b6f87d5c113b68897fd", + )), + ), + ( + model("b"), + Some(pinned( + "b1d269f24b600f5b30f9b68dc34f0305ebf3c23b3657ebb147b6cae7ffaf8282", + )), + ), + (model("x"), None), + ( + model("c"), + Some(pinned( + "3f7d5648557bef56e83e9b9b9cc1789cf758e32310f142d733b3d57a5caabc15", + )), + ), + (model("d"), None), + ], + expected_write_hashes: vec![ + (model("a"), pinned("base_a")), + ( + model("b"), + pinned("1780fb66cc0e91d38e4be3c63ffa49e52abaa5365a68aad535ff04218f58ecc9"), + ), + ( + model("c"), + pinned("d41afd29f6fdfaa9bc2a55fd9e5f1a75b2e8dbe100f431706a67938e21fd1b86"), + ), + ], + }, + IdentityGraphTestCase { + description: "cycle broken with the local hash of the node being resolved", + graph: cyclic_identity_graph(), + selected: HashSet::from([model("a"), model("b")]), + base_hashes: Vec::new(), + expected_hashes: vec![ + ( + model("a"), + Some(pinned( + "9c389106b00932fd7ff1b7dd3778747eac217990a9a73b16c80a9f0bbbf9f527", + )), + ), + ( + model("b"), + Some(pinned( + "b8c36ab63f3421b0ce0f3b1b55428334e42f1c1a8db65d10707bdc236a81da76", + )), + ), + ], + expected_write_hashes: vec![ + ( + model("b"), + pinned("b8c36ab63f3421b0ce0f3b1b55428334e42f1c1a8db65d10707bdc236a81da76"), + ), + ( + model("a"), + pinned("9c389106b00932fd7ff1b7dd3778747eac217990a9a73b16c80a9f0bbbf9f527"), + ), + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + ( + expected_graph_identity_hashes(&test_case.graph), + graph_write_identity_hashes( + &test_case.graph, + &test_case.selected, + test_case.base_hashes + ), + ), + (test_case.expected_hashes, test_case.expected_write_hashes), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dense_ladder_when_resolving_write_hashes_then_resolves_each_node_once() { + let test_cases = [IdentityGraphSizeTestCase { + description: "four hundred fully selected layers", + layer_count: 400, + expected_hash_count: 800, + }]; + for test_case in test_cases { + let graph = ladder_identity_graph(test_case.layer_count); + let selected: HashSet = graph.nodes.keys().cloned().collect(); + assert_eq!( + graph_write_identity_hashes(&graph, &selected, Vec::new()).len(), + test_case.expected_hash_count, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_function_fields_when_rendering_then_writes_one_field_per_line() { + let test_cases = [FunctionSqlTestCase { + description: "python function with packages and an unnamed argument", + fields: FunctionFingerprintFields { + language: "python".to_owned(), + arguments: vec![ + ("x".to_owned(), "int".to_owned()), + (String::new(), "text".to_owned()), + ], + returns: "table".to_owned(), + return_columns: vec![("total".to_owned(), "double".to_owned())], + runtime_version: "3.12".to_owned(), + entry_point: String::new(), + packages: texts(&["numpy", "pandas"]), + body_sql: "return x\r\n".to_owned(), + }, + expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\ + runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n", + }]; + for test_case in test_cases { + assert_eq!( + function_fingerprint_sql(&test_case.fields), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selection_when_classifying_staleness_then_names_changed_upstreams_outside_run() { + let mut changed_outside = staleness_graph( + &[ + (model("report"), &[model("orders"), key("seed", "regions")]), + (model("orders"), &[model("raw_orders")]), + ], + &["report"], + ); + changed_outside.changed = StalenessNames { + models: names(&["raw_orders"]), + seeds: names(&["regions"]), + sources: HashSet::new(), + }; + let mut run_parent = staleness_graph( + &[ + (model("report"), &[model("orders")]), + (model("orders"), &[model("raw_orders")]), + ], + &["report"], + ); + run_parent.run.models = names(&["raw_orders", "report"]); + run_parent.changed.models = names(&["raw_orders"]); + let mut source_in_run = staleness_graph( + &[(model("report"), &[key("source", "events")])], + &["report"], + ); + source_in_run.changed.sources = names(&["events"]); + source_in_run.run.sources = names(&["events"]); + let test_cases = [ + StalenessTestCase { + description: "changed model and seed outside the run set", + graph: changed_outside, + expected_warnings: vec![StaleWarning { + model_name: "report".to_owned(), + trigger_names: texts(&["orders", "raw_orders", "regions"]), + }], + }, + StalenessTestCase { + description: "changed parent runs but the middle model does not", + graph: run_parent, + expected_warnings: vec![StaleWarning { + model_name: "report".to_owned(), + trigger_names: texts(&["orders"]), + }], + }, + StalenessTestCase { + description: "changed source in the run set", + graph: source_in_run, + expected_warnings: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + classify_selection_staleness(&test_case.graph), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_trigger_names_when_formatting_then_caps_the_list() { + let test_cases = [ + WarningTextTestCase { + description: "one trigger", + trigger_names: texts(&["orders"]), + expected_message: "selected model 'report' will build on 1 stale upstream(s) not selected for rebuild:\n - orders\n rebuild the closure to refresh them: --select +report", + }, + WarningTextTestCase { + description: "more than the display limit", + trigger_names: texts(&["a", "b", "c", "d", "e", "f", "g"]), + expected_message: "selected model 'report' will build on 7 stale upstream(s) not selected for rebuild:\n - a\n - b\n - c\n - d\n - e\n +2 more\n rebuild the closure to refresh them: --select +report", + }, + ]; + for test_case in test_cases { + assert_eq!( + stale_upstream_warning_message( + ("selected model", "report"), + "upstream(s)", + &test_case.trigger_names + ), + test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_upstream_edges_when_ordering_then_orders_stably_or_reports_cycles() { + let test_cases = [ + OrderTestCase { + description: "smallest ready key first", + upstream: vec![ + (model("report"), vec![model("orders"), key("seed", "regions")]), + (model("orders"), vec![key("source", "raw")]), + ], + origins: HashMap::new(), + expected_order: Ok(vec![ + key("seed", "regions"), + key("source", "raw"), + model("orders"), + model("report"), + ]), + }, + OrderTestCase { + description: "cycle through an injected edge", + upstream: vec![ + (model("a"), vec![key("sql_test", "t")]), + (key("sql_test", "t"), vec![model("a")]), + (model("z"), vec![]), + ], + origins: HashMap::from([( + (model("a"), key("sql_test", "t")), + "SQL test 't' runs before 'a'".to_owned(), + )]), + expected_order: Err("Dependency cycle detected involving: model:a, sql_test:t (via SQL test 't' runs before 'a')".to_owned()), + }, + ]; + for test_case in test_cases { + assert_eq!( + topological_order(&test_case.upstream, &test_case.origins), + test_case.expected_order, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams() { + let test_cases = [ + BuildabilityTestCase { + description: "lineage upstream missing for two dependents", + selected: vec![model("b"), model("a")], + upstream: HashMap::from([ + (model("a"), vec![model("raw"), key("source", "events")]), + (model("b"), vec![model("raw"), model("built"), model("a")]), + ]), + available: AvailableUpstreams { + existing_relations: names(&["built"]), + ..AvailableUpstreams::default() + }, + origins: HashMap::new(), + expected_missing: vec![MissingUpstream { + key: model("raw"), + required_by: vec![model("a"), model("b")], + }], + expected_message: "cannot build selected scope: 1 missing upstream dependencies (raw)", + }, + BuildabilityTestCase { + description: "only an audit read requires the upstream", + selected: vec![model("a")], + upstream: HashMap::from([(model("a"), vec![model("lookup")])]), + available: AvailableUpstreams::default(), + origins: HashMap::from([( + (model("a"), model("lookup")), + "audit 'check' on 'a' reads 'lookup'".to_owned(), + )]), + expected_missing: vec![MissingUpstream { + key: model("lookup"), + required_by: vec![model("a")], + }], + expected_message: "cannot build selected scope: audit 'check' on 'a' reads 'lookup', which is not selected and does not exist in the warehouse; select it or build it first", + }, + ]; + for test_case in test_cases { + let missing = missing_upstreams( + &test_case.selected, + &test_case.upstream, + &test_case.available, + ); + assert_eq!( + ( + missing_upstream_message(&missing, &test_case.origins), + missing + ), + ( + test_case.expected_message.to_owned(), + test_case.expected_missing + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs new file mode 100644 index 0000000000..8ad36977d2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs @@ -0,0 +1,63 @@ +use std::collections::{HashMap, HashSet}; + +use crate::planner_identity::models::{ + AvailableUpstreams, FunctionFingerprintFields, GraphKey, IdentityGraph, MissingUpstream, + StaleWarning, StalenessGraph, +}; +use crate::planner_identity::types::IdentityHashes; + +pub(super) struct HashTestCase { + pub(super) description: &'static str, + pub(super) actual: String, + pub(super) expected_hash: &'static str, +} + +pub(super) struct StalenessTestCase { + pub(super) description: &'static str, + pub(super) graph: StalenessGraph, + pub(super) expected_warnings: Vec, +} + +pub(super) struct OrderTestCase { + pub(super) description: &'static str, + pub(super) upstream: Vec<(GraphKey, Vec)>, + pub(super) origins: HashMap<(GraphKey, GraphKey), String>, + pub(super) expected_order: Result, String>, +} + +pub(super) struct BuildabilityTestCase { + pub(super) description: &'static str, + pub(super) selected: Vec, + pub(super) upstream: HashMap>, + pub(super) available: AvailableUpstreams, + pub(super) origins: HashMap<(GraphKey, GraphKey), String>, + pub(super) expected_missing: Vec, + pub(super) expected_message: &'static str, +} + +pub(super) struct WarningTextTestCase { + pub(super) description: &'static str, + pub(super) trigger_names: Vec, + pub(super) expected_message: &'static str, +} + +pub(super) struct IdentityGraphTestCase { + pub(super) description: &'static str, + pub(super) graph: IdentityGraph, + pub(super) selected: HashSet, + pub(super) base_hashes: Vec<(GraphKey, String)>, + pub(super) expected_hashes: IdentityHashes, + pub(super) expected_write_hashes: Vec<(GraphKey, String)>, +} + +pub(super) struct IdentityGraphSizeTestCase { + pub(super) description: &'static str, + pub(super) layer_count: usize, + pub(super) expected_hash_count: usize, +} + +pub(super) struct FunctionSqlTestCase { + pub(super) description: &'static str, + pub(super) fields: FunctionFingerprintFields, + pub(super) expected_sql: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/types.rs b/crates/sqlbuild-analysis/src/planner_identity/types.rs new file mode 100644 index 0000000000..a60fe417d6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_identity/types.rs @@ -0,0 +1,6 @@ +//! Planner identity type aliases. + +use crate::planner_identity::models::GraphKey; + +/// Identity hashes in first-recorded order. +pub type IdentityHashes = Vec<(GraphKey, Option)>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs index 54c8e6f8f7..dbeb9c3956 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs @@ -69,7 +69,7 @@ fn transitive_closure<'py>( } /// Reversed edges: every source key first, then each new target in first-seen order. -fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn inverted<'py>(edges: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = edges.py(); let inverted: Bound<'py, PyDict> = PyDict::new(py); for key in edges.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 71b7bb0094..0588b71cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -45,6 +45,7 @@ pub(crate) mod manifest; pub(crate) mod model_config; pub(crate) mod plan_output; pub(crate) mod planner_changes; +pub(crate) mod planner_identity; pub(crate) mod planner_strategy; pub(crate) mod presentation_text; pub(crate) mod project_assembly; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 19c22aa036..3269ee20cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -27,15 +27,15 @@ use crate::bindings::_helpers::planner_changes::renames::origin_names; use crate::bindings::_helpers::planner_changes::results::{ MODELS_MODULE, backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, }; +use crate::bindings::_helpers::planner_identity::hashing::{ + function_definition_hash, model_query_hash, +}; +use crate::bindings::_helpers::planner_identity::metadata::{ + compiled_function_fingerprint_sql, function_local_hashes, model_declared_columns_hash, + model_version_identity_metadata_json, +}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const MODEL_METADATA_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.model_metadata"; -const VERSION_METADATA_MODULE: &str = - "sqlbuild.compiler.planner.main.identity.version_identity_model_metadata"; -const FUNCTION_HASHES_MODULE: &str = - "sqlbuild.compiler.planner.main.identity.version_identity_function_hashes"; -const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; -const FUNCTIONS_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.functions"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; const DYNAMIC_FAMILIES_MODULE: &str = "sqlbuild._native"; @@ -46,12 +46,12 @@ const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; /// Inputs shared by every model of one detection pass. -struct ModelPass<'py> { - snapshot: Bound<'py, PyAny>, - query_change_tracking: bool, - full_refresh: bool, - function_local_hashes: Bound<'py, PyAny>, - hook_version_hashes: Bound<'py, PyAny>, +pub(crate) struct ModelPass<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) query_change_tracking: bool, + pub(crate) full_refresh: bool, + pub(crate) function_local_hashes: Bound<'py, PyAny>, + pub(crate) hook_version_hashes: Bound<'py, PyAny>, } /// Selected model and function changes. @@ -87,47 +87,47 @@ fn detect_changes<'py>( )?; let functions: Bound<'py, PyDict> = PyDict::new(py); let result_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "FunctionChangeResult")?; - let fingerprint_sql_of: Bound<'py, PyAny> = imported( - py, - FUNCTIONS_MODULE, - "build_compiled_function_fingerprint_sql", - )?; - let detect_function: Bound<'py, PyAny> = - imported(py, FUNCTIONS_MODULE, "detect_function_change")?; let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let recorded: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("functions")?; + let query_change_tracking: bool = project + .getattr("settings")? + .getattr("query_change_tracking")? + .is_truthy()?; + let dialect: Option = snapshot.getattr("column_dialect")?.extract()?; for function in project.getattr("functions")?.try_iter()? { let function: Bound<'py, PyAny> = function?; let name: Bound<'py, PyAny> = function.getattr("name")?; - let fingerprint_sql: Bound<'py, PyAny> = fingerprint_sql_of.call1((&function,))?; - let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = - vec![("fingerprint_sql", fingerprint_sql.clone())]; + let fingerprint_sql: String = compiled_function_fingerprint_sql(&function)?; + let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = vec![( + "fingerprint_sql", + PyString::new(py, &fingerprint_sql).into_any(), + )]; if selected.contains(function.getattr("key")?)? { - let reason: Bound<'py, PyAny> = call_with( - &detect_function, - &[ - ("function", function.clone()), - ("fingerprint_sql", fingerprint_sql), - ( - "fingerprint", - snapshot - .getattr("fingerprints")? - .getattr("functions")? - .call_method1("get", (&name,))?, - ), - ( - "query_change_tracking", - project - .getattr("settings")? - .getattr("query_change_tracking")?, - ), - ( - "full_refresh", - full_refresh.into_pyobject(py)?.to_owned().into_any(), - ), - ("dialect", snapshot.getattr("column_dialect")?), - ], - )?; - arguments.push(("reason", reason)); + let reason: &str = if full_refresh { + "full_refresh" + } else { + let fingerprint: Bound<'py, PyAny> = recorded.call_method1("get", (&name,))?; + if fingerprint.is_none() { + "first_run" + } else if query_change_tracking + && function_definition_hash( + py, + &name.extract::()?, + ( + &fingerprint_sql, + &function.getattr("language")?.str()?.extract::()?, + dialect.as_deref(), + ), + )? != fingerprint + .getattr("definition_hash")? + .extract::()? + { + "query_changed" + } else { + "no_change" + } + }; + arguments.push(("reason", planner_enum(py, "PlanReason", reason)?)); } functions.set_item(name, call_with(&result_class, &arguments)?)?; } @@ -231,13 +231,14 @@ fn scope_model_changes<'py>( .getattr("query_change_tracking")? .extract()?, }; - let function_local_hashes: Bound<'py, PyAny> = call_with( - &imported(py, FUNCTION_HASHES_MODULE, "build_function_local_hashes")?, - &[ - ("functions", project.getattr("functions")?), - ("dialect", project.getattr("sql_analysis_dialect")?), - ], - )?; + let function_local_hashes: Bound<'py, PyAny> = function_local_hashes( + &project.getattr("functions")?, + project + .getattr("sql_analysis_dialect")? + .extract::>()? + .as_deref(), + )? + .into_any(); let hook_version_hashes: Bound<'py, PyDict> = PyDict::new(py); let identities: Bound<'py, PyAny> = imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")? @@ -288,7 +289,7 @@ fn lookup(mapping: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult( +pub(crate) fn model_change<'py>( pass: &ModelPass<'py>, model: &Bound<'py, PyAny>, (expected_version_hash, expected_metadata_json): (Option, Option), @@ -296,26 +297,14 @@ fn model_change<'py>( let py: Python<'py> = model.py(); let snapshot: &Bound<'py, PyAny> = &pass.snapshot; let model_name: Bound<'py, PyAny> = model.getattr("name")?; - let columns_hash: String = call_with( - &imported(py, MODEL_METADATA_MODULE, "declared_columns_hash")?, - &[("model", model.clone())], - )? - .extract()?; + let columns_hash: String = model_declared_columns_hash(model)?; let base_metadata: String = match expected_metadata_json.filter(|text| !text.is_empty()) { Some(text) => text, - None => call_with( - &imported( - py, - VERSION_METADATA_MODULE, - "build_model_version_identity_metadata_json", - )?, - &[ - ("model", model.clone()), - ("function_local_hashes", pass.function_local_hashes.clone()), - ("hook_version_hashes", pass.hook_version_hashes.clone()), - ], - )? - .extract()?, + None => model_version_identity_metadata_json( + model, + Some(&pass.function_local_hashes), + Some(&pass.hook_version_hashes), + )?, }; let metadata_json: String = with_declared_columns_hash(&base_metadata, &columns_hash); let mut result = ChangeResult::new( @@ -343,17 +332,14 @@ fn model_change<'py>( let recorded: Option = recorded_fingerprint(&fingerprint)?; let dialect: Bound<'py, PyAny> = snapshot.getattr("column_dialect")?; let compiled_query_hash: Option = if pass.query_change_tracking && recorded.is_some() { - Some( - call_with( - &imported(py, HASHING_MODULE, "model_definition_hash")?, - &[ - ("model_name", model_name.clone()), - ("query_sql", model.getattr("query_sql")?), - ("dialect", dialect.clone()), - ], - )? - .extract()?, - ) + Some(model_query_hash( + py, + &model_name.extract::()?, + ( + &model.getattr("query_sql")?.extract::()?, + dialect.extract::>()?.as_deref(), + ), + )?) } else { None }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index d19d59345b..32dfca7039 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; const NATIVE_MODULE: &str = "sqlbuild._native"; -const HASHING_MODULE: &str = "sqlbuild.compiler.planner._helpers.identity.hashing"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; const GENERIC_DIALECT: &str = "generic"; const FINAL_NEWLINE: &str = "\n"; @@ -238,7 +238,7 @@ fn changed_referencing_query( let recorded_models: Bound<'_, PyAny> = snapshot.getattr("fingerprints")?.getattr("models")?; let ref_kind: Bound<'_, PyAny> = imported(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?.getattr("REF")?; - let definition_hash: Bound<'_, PyAny> = imported(py, HASHING_MODULE, "model_definition_hash")?; + let dialect: Option = snapshot.getattr("column_dialect")?.extract()?; for model in selected { let recorded: Bound<'_, PyAny> = recorded_models.call_method1("get", (model.getattr("name")?,))?; @@ -253,15 +253,14 @@ fn changed_referencing_query( } } if references_models - && call_with( - &definition_hash, - &[ - ("model_name", model.getattr("name")?), - ("query_sql", model.getattr("query_sql")?), - ("dialect", snapshot.getattr("column_dialect")?), - ], - )? - .ne(recorded.getattr("definition_hash")?)? + && model_query_hash( + py, + &model.getattr("name")?.extract::()?, + ( + &model.getattr("query_sql")?.extract::()?, + dialect.as_deref(), + ), + )? != recorded.getattr("definition_hash")?.extract::()? { return Ok(true); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs new file mode 100644 index 0000000000..7257d905c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -0,0 +1,415 @@ +//! Planner execution edges, their injected origins, stable execution order and buildability over compiled keys. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, + PyTuple, +}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::missing_upstream_message::missing_upstream_message as message_of; +use sqlbuild_analysis::planner_identity::main::missing_upstreams::missing_upstreams; +use sqlbuild_analysis::planner_identity::main::topological_order::topological_order; +use sqlbuild_analysis::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream}; + +use crate::bindings::_helpers::analysis::explicit_references::{ + GateRow, HookRow, native_execution_gate_edges, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_identity::hashing::graph_key; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const TABLE_FN_TEST_MODE: &str = "table_fn"; +const INLINE_SQL_HOOK: &str = "inline SQL"; + +/// Python keys by graph key, keeping the first object seen for each. +struct KeyObjects<'py> { + objects: HashMap>, +} + +impl<'py> KeyObjects<'py> { + fn new() -> Self { + Self { + objects: HashMap::new(), + } + } + + fn key(&mut self, object: Bound<'py, PyAny>) -> PyResult { + let key: GraphKey = graph_key(&object)?; + self.objects.entry(key.clone()).or_insert(object); + Ok(key) + } + + fn keys(&mut self, objects: &Bound<'py, PyAny>) -> PyResult> { + objects + .try_iter()? + .map(|object| self.key(object?)) + .collect() + } + + fn object(&self, key: &GraphKey) -> Bound<'py, PyAny> { + self.objects[key].clone() + } + + fn tuple(&self, keys: &[GraphKey], py: Python<'py>) -> PyResult> { + PyTuple::new(py, keys.iter().map(|key| self.object(key))) + } +} + +fn function_resource(key: &Bound<'_, PyAny>) -> PyResult { + let kind: String = key.getattr("resource_type")?.str()?.extract()?; + Ok(kind == UDF_RESOURCE || kind == TABLE_FN_RESOURCE) +} + +fn hook_label(hook_name: &str, kind: &str) -> String { + if kind == INLINE_SQL_HOOK { + return format!("inline SQL hook {hook_name}"); + } + format!("{kind} hook '{hook_name}'") +} + +/// Execution-order edges from lineage and woven SQL test nodes. +#[pyfunction] +fn build_execution_upstream_deps<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let upstream: Bound<'py, PyDict> = PyDict::new(py); + for collection in ["models", "sources", "seeds", "functions"] { + for resource in project.getattr(collection)?.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + upstream.set_item( + resource.getattr("key")?, + PyList::new( + py, + resource + .getattr("deps")? + .try_iter()? + .collect::>>()?, + )?, + )?; + } + } + for test in project.getattr("sql_tests")?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + let test_key: Bound<'py, PyAny> = test.getattr("key")?; + let scope_deps: Bound<'py, PyAny> = test.getattr("scope_deps")?; + let deps: Bound<'py, PyList> = PyList::empty(py); + if test.getattr("mode")?.str()?.extract::()? == TABLE_FN_TEST_MODE { + for scope_dep in scope_deps.try_iter()? { + let scope_dep: Bound<'py, PyAny> = scope_dep?; + if function_resource(&scope_dep)? { + deps.append(scope_dep)?; + } + } + upstream.set_item(test_key, deps)?; + continue; + } + let seen: Bound<'py, PySet> = PySet::empty(py)?; + for target_key in scope_deps.try_iter()? { + let Some(target_deps) = upstream.get_item(target_key?)? else { + continue; + }; + for dep_key in target_deps.try_iter()? { + let dep_key: Bound<'py, PyAny> = dep_key?; + if !function_resource(&dep_key)? || seen.contains(&dep_key)? { + continue; + } + seen.add(&dep_key)?; + deps.append(dep_key)?; + } + } + upstream.set_item(&test_key, deps)?; + for target_key in scope_deps.try_iter()? { + if let Some(target_deps) = upstream.get_item(target_key?)? { + target_deps.downcast::()?.append(&test_key)?; + } + } + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let cache: KeyCache<'py> = KeyCache::new(py)?; + let mut extra: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for (_, _, gated, read) in &gates { + extra.push((cache.key(gated)?, cache.key(read)?)); + } + for (_, gated, read, _, _) in &hooks { + extra.push((cache.key(gated)?, cache.key(read)?)); + } + for (gated, read) in extra { + let gated_upstream: Bound<'py, PyAny> = + upstream.call_method1("setdefault", (gated, PyList::empty(py)))?; + if !gated_upstream.contains(&read)? { + gated_upstream.downcast::()?.append(read)?; + } + } + let result: Bound<'py, PyDict> = PyDict::new(py); + for (key, deps) in upstream.iter() { + result.set_item( + key, + PyTuple::new(py, deps.try_iter()?.collect::>>()?)?, + )?; + } + Ok(result) +} + +/// Human-readable origins for execution edges injected beyond pure lineage. +#[pyfunction] +fn build_execution_edge_origins<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let origins: Bound<'py, PyDict> = PyDict::new(py); + for test in project.getattr("sql_tests")?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + let test_name: String = test.getattr("name")?.extract()?; + for target_key in test.getattr("scope_deps")?.try_iter()? { + let target_key: Bound<'py, PyAny> = target_key?; + let target_name: String = target_key.getattr("name")?.extract()?; + origins.set_item( + (target_key, test.getattr("key")?), + format!("SQL test '{test_name}' runs before '{target_name}'"), + )?; + } + } + let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; + let cache: KeyCache<'py> = KeyCache::new(py)?; + let lineage: Bound<'py, PyAny> = project.getattr("lineage_graph")?; + let is_lineage = |gated: &(String, String), read: &(String, String)| -> PyResult { + lineage + .call_method1("upstream", (gated.clone(),))? + .contains(read.clone()) + }; + for (audit_name, target, gated, read) in &gates { + if is_lineage(gated, read)? { + continue; + } + let _ = origins.call_method1( + "setdefault", + ( + (cache.key(gated)?, cache.key(read)?), + format!("audit '{audit_name}' on '{}' reads '{}'", target.1, read.1), + ), + )?; + } + for (hook_name, gated, read, kind, _) in &hooks { + if is_lineage(gated, read)? { + continue; + } + let _ = origins.call_method1( + "setdefault", + ( + (cache.key(gated)?, cache.key(read)?), + format!( + "{} on '{}' reads '{}'", + hook_label(hook_name, kind), + gated.1, + read.1 + ), + ), + )?; + } + Ok(origins) +} + +/// Downstream edges keyed by upstream key, each list sorted by `(resource_type, name)`. +#[pyfunction] +fn build_downstream_deps<'py>(upstream: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = upstream.py(); + let result: Bound<'py, PyDict> = PyDict::new(py); + for (key, sources) in inverted(upstream)?.iter() { + let mut keyed: Vec<(GraphKey, Bound<'py, PyAny>)> = sources + .try_iter()? + .map(|source| { + let source: Bound<'py, PyAny> = source?; + Ok((graph_key(&source)?, source)) + }) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + result.set_item( + key, + PyTuple::new(py, keyed.into_iter().map(|(_, source)| source))?, + )?; + } + Ok(result) +} + +fn planner_input_error(py: Python<'_>, message: String) -> PyErr { + match imported(py, EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| class.call1((message,))) + { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn edge_origins<'py>( + origins: &Bound<'py, PyAny>, + objects: &mut KeyObjects<'py>, +) -> PyResult> { + let mut edges: HashMap<(GraphKey, GraphKey), String> = HashMap::new(); + if origins.is_none() { + return Ok(edges); + } + for item in origins.call_method0("items")?.try_iter()? { + let ((dependent, dependency), origin): ((Bound<'py, PyAny>, Bound<'py, PyAny>), String) = + item?.extract()?; + edges.insert((objects.key(dependent)?, objects.key(dependency)?), origin); + } + Ok(edges) +} + +/// All keys in stable dependency order using Kahn's algorithm. +#[pyfunction] +#[pyo3(signature = (*, upstream, injected_edge_origins = None))] +fn topologically_order_keys<'py>( + upstream: &Bound<'py, PyAny>, + injected_edge_origins: Option>, +) -> PyResult> { + let py: Python<'py> = upstream.py(); + let mut objects = KeyObjects::new(); + let mut edges: Vec<(GraphKey, Vec)> = Vec::new(); + for item in upstream.call_method0("items")?.try_iter()? { + let (key, deps): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + edges.push((objects.key(key)?, objects.keys(&deps)?)); + } + let origins: HashMap<(GraphKey, GraphKey), String> = match injected_edge_origins { + Some(origins) => edge_origins(&origins, &mut objects)?, + None => HashMap::new(), + }; + let ordered: Vec = + topological_order(&edges, &origins).map_err(|message| planner_input_error(py, message))?; + objects.tuple(&ordered, py) +} + +/// Model names reachable downstream from the given keys, including model roots. +#[pyfunction] +#[pyo3(signature = (*, start_keys, downstream_deps))] +fn build_downstream_model_name_closure<'py>( + start_keys: &Bound<'py, PyAny>, + downstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = downstream_deps.py(); + let starts: Vec> = start_keys.try_iter()?.collect::>()?; + let mut names: Vec = Vec::new(); + for key in closure(starts, downstream_deps, (true, None))?.iter() { + let key: GraphKey = graph_key(&key)?; + if key.kind == MODEL_RESOURCE { + names.push(key.name); + } + } + PyFrozenSet::new(py, names) +} + +fn name_set(names: &Bound<'_, PyAny>) -> PyResult> { + names.try_iter()?.map(|name| name?.extract()).collect() +} + +fn missing_rows<'py>( + missing: &Bound<'py, PyAny>, + objects: &mut KeyObjects<'py>, +) -> PyResult> { + missing + .try_iter()? + .map(|entry| { + let entry: Bound<'py, PyAny> = entry?; + Ok(MissingUpstream { + key: objects.key(entry.getattr("key")?)?, + required_by: objects.keys(&entry.getattr("required_by")?)?, + }) + }) + .collect() +} + +/// Upstream deps of selected keys that exist neither in scope nor in the warehouse. +#[pyfunction] +#[pyo3(signature = (**options))] +fn check_buildability<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "check_buildability", + options, + ( + &["selected_keys", "upstream_deps", "snapshot"], + &["deferred_relations", "satisfied_keys"], + ), + )?; + let selected_keys: Bound<'py, PyAny> = arguments.value("selected_keys")?; + let upstream_deps: Bound<'py, PyAny> = arguments.value("upstream_deps")?; + let snapshot: Bound<'py, PyAny> = arguments.value("snapshot")?; + let deferred_relations: Bound<'py, PyAny> = arguments.value("deferred_relations")?; + let satisfied_keys: Bound<'py, PyAny> = arguments.value("satisfied_keys")?; + let mut objects = KeyObjects::new(); + let selected: Vec = objects.keys(&selected_keys)?; + let mut deps: HashMap> = HashMap::new(); + for key in &selected { + let dep_keys: Bound<'py, PyAny> = + upstream_deps.call_method1("get", (objects.object(key), PyTuple::empty(py)))?; + deps.insert(key.clone(), objects.keys(&dep_keys)?); + } + let available = AvailableUpstreams { + satisfied_keys: if satisfied_keys.is_none() { + HashSet::new() + } else { + objects.keys(&satisfied_keys)?.into_iter().collect() + }, + existing_relations: name_set(&snapshot.getattr("existing_relations")?)?, + deferred_relations: if deferred_relations.is_none() { + None + } else { + Some(name_set(&deferred_relations)?) + }, + }; + let entry_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "MissingUpstream")?; + let mut entries: Vec> = Vec::new(); + for missing in missing_upstreams(&selected, &deps, &available) { + entries.push(call_with( + &entry_class, + &[ + ("key", objects.object(&missing.key)), + ( + "required_by", + objects.tuple(&missing.required_by, py)?.into_any(), + ), + ], + )?); + } + PyTuple::new(py, entries) +} + +/// Describe missing upstream keys, naming the audit when only an audit read requires one. +#[pyfunction] +#[pyo3(signature = (*, missing, edge_origins))] +fn missing_upstream_message<'py>( + missing: &Bound<'py, PyAny>, + edge_origins: &Bound<'py, PyAny>, +) -> PyResult> { + let mut objects = KeyObjects::new(); + let rows: Vec = missing_rows(missing, &mut objects)?; + let origins: HashMap<(GraphKey, GraphKey), String> = + self::edge_origins(edge_origins, &mut objects)?; + Ok(PyString::new(missing.py(), &message_of(&rows, &origins))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_execution_upstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(build_execution_edge_origins, module)?)?; + module.add_function(wrap_pyfunction!(build_downstream_deps, module)?)?; + module.add_function(wrap_pyfunction!(topologically_order_keys, module)?)?; + module.add_function(wrap_pyfunction!( + build_downstream_model_name_closure, + module + )?)?; + module.add_function(wrap_pyfunction!(check_buildability, module)?)?; + module.add_function(wrap_pyfunction!(missing_upstream_message, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs new file mode 100644 index 0000000000..af4645c209 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/hashing.rs @@ -0,0 +1,89 @@ +//! Query and function definition fingerprints for change detection, failing as planner input errors. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyString; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{FINGERPRINT_HELP, QUERY_FINGERPRINT_FAILED}; +use sqlbuild_analysis::planner_identity::models::GraphKey; +use sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; +use sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint; +use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::types::CompilerDetach; + +const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; + +/// The query fingerprint of a model, or the planner input error naming the model. +pub(crate) fn model_query_hash( + py: Python<'_>, + model_name: &str, + (query_sql, dialect): (&str, Option<&str>), +) -> PyResult { + py.compiler_detach(|| compiler_query_fingerprint(query_sql, dialect)) + .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) +} + +/// The definition fingerprint of a function, or the planner input error naming the function. +pub(crate) fn function_definition_hash( + py: Python<'_>, + function_name: &str, + (fingerprint_sql, language, dialect): (&str, &str, Option<&str>), +) -> PyResult { + py.compiler_detach(|| definition_fingerprint(fingerprint_sql, language, dialect)) + .map_err(|error| fingerprint_error(py, &format!("Function '{function_name}'"), error)) +} + +fn fingerprint_error(py: Python<'_>, subject: &str, message: String) -> PyErr { + if is_compiler_panic(&message) { + return compiler_error(message); + } + let text = |value: &str| PyString::new(py, value).into_any(); + let raised: PyResult> = imported(py, EXCEPTIONS_MODULE, "PlannerInputError") + .and_then(|class| { + class.call( + (format!( + "{subject} cannot be fingerprinted for change detection: {message}" + ),), + Some(&keywords( + py, + &[ + ("code", text(QUERY_FINGERPRINT_FAILED)), + ("help", text(FINGERPRINT_HELP)), + ], + )?), + ) + }); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The graph key of a `CompiledObjectKey`-like value: its resource type text and name. +pub(crate) fn graph_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(GraphKey { + kind: key.getattr("resource_type")?.str()?.extract()?, + name: key.getattr("name")?.extract()?, + }) +} + +/// Fingerprint a model query for change detection, naming the model when it cannot. +#[pyfunction] +#[pyo3(signature = (*, model_name, query_sql, dialect))] +fn model_definition_hash( + py: Python<'_>, + model_name: &str, + query_sql: &str, + dialect: Option<&str>, +) -> PyResult { + model_query_hash(py, model_name, (query_sql, dialect)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_definition_hash, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs new file mode 100644 index 0000000000..0f06e11456 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -0,0 +1,394 @@ +//! Direct model version identities and honest write hashes over the planner scope. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, TABLE_FN_RESOURCE, UDF_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::expected_graph_identity_hashes::expected_graph_identity_hashes; +use sqlbuild_analysis::planner_identity::main::graph_write_identity_hashes::graph_write_identity_hashes; +use sqlbuild_analysis::planner_identity::main::model_local_identity_hash::model_local_identity_hash; +use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; +use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_query_hash}; +use crate::bindings::_helpers::planner_identity::metadata::{ + function_local_hashes, model_version_identity_metadata_json, seed_identity, +}; + +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const DATACLASSES_MODULE: &str = "dataclasses"; + +fn key_of(kind: &str, name: &str) -> GraphKey { + GraphKey { + kind: kind.to_owned(), + name: name.to_owned(), + } +} + +fn graph_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| graph_key(&key?)).collect() +} + +fn string_pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { + mapping + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract::<(String, String)>()) + .collect() +} + +/// `{name: identity.version_hash}` of the discovered Python hooks. +fn hook_version_hashes<'py>(hook_functions: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = hook_functions.py(); + let hashes: Bound<'py, PyDict> = PyDict::new(py); + let identities: Bound<'py, PyAny> = + imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")?.call1((hook_functions,))?; + for item in identities.call_method0("items")?.try_iter()? { + let (name, identity): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + hashes.set_item(name, identity.getattr("version_hash")?)?; + } + Ok(hashes) +} + +/// Current direct model identities from code and upstream identities. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_direct_model_version_identities<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_direct_model_version_identities", + options, + ( + &["functions", "scope", "dialect"], + &["seeds", "source_version_hashes", "hook_functions"], + ), + )?; + let functions: Bound<'py, PyAny> = arguments.value("functions")?; + let scope: Bound<'py, PyAny> = arguments.value("scope")?; + let dialect: Option = arguments.value("dialect")?.extract()?; + let function_hashes: Bound<'py, PyDict> = + function_local_hashes(&functions, dialect.as_deref())?; + let hooks: Bound<'py, PyAny> = arguments.value("hook_functions")?; + let hook_hashes: Bound<'py, PyDict> = if hooks.is_none() { + PyDict::new(py) + } else { + hook_version_hashes(&hooks)? + }; + let mut graph = IdentityGraph::default(); + for function in functions.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let name: String = function.getattr("name")?.extract()?; + let Some(function_hash) = function_hashes.get_item(&name)? else { + continue; + }; + graph.nodes.insert( + graph_key(&function.getattr("key")?)?, + IdentityNode { + upstream_keys: graph_keys(&function.getattr("deps")?)?, + local_hash: Some(function_hash.extract()?), + }, + ); + } + let seed_hashes: Bound<'py, PyDict> = PyDict::new(py); + let seed_metadata: Bound<'py, PyDict> = PyDict::new(py); + let seeds: Bound<'py, PyAny> = arguments.value("seeds")?; + if !seeds.is_none() { + for seed in seeds.try_iter()? { + let seed: Bound<'py, PyAny> = seed?; + let (seed_hash, metadata_json): (String, String) = seed_identity(&seed)?; + let name: Bound<'py, PyAny> = seed.getattr("name")?; + seed_hashes.set_item(&name, &seed_hash)?; + seed_metadata.set_item(&name, metadata_json)?; + } + } + for (name, seed_hash) in string_pairs(seed_hashes.as_any())? { + graph.nodes.insert( + key_of(SEED_RESOURCE, &name), + IdentityNode { + upstream_keys: Vec::new(), + local_hash: Some(seed_hash), + }, + ); + } + let source_hashes: Bound<'py, PyAny> = arguments.value("source_version_hashes")?; + if source_hashes.is_truthy()? { + for (name, source_hash) in string_pairs(&source_hashes)? { + graph.nodes.insert( + key_of(SOURCE_RESOURCE, &name), + IdentityNode { + upstream_keys: Vec::new(), + local_hash: Some(source_hash), + }, + ); + } + } + let model_metadata: Bound<'py, PyDict> = PyDict::new(py); + let model_local: Bound<'py, PyDict> = PyDict::new(py); + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + let order_key: GraphKey = graph_key(&key)?; + graph.execution_order.push(order_key.clone()); + if order_key.kind != MODEL_RESOURCE { + continue; + } + let model: Bound<'py, PyAny> = models_by_name.call_method1("get", (&order_key.name,))?; + if model.is_none() { + continue; + } + let name: String = model.getattr("name")?.extract()?; + let metadata_json: String = model_version_identity_metadata_json( + &model, + Some(function_hashes.as_any()), + Some(hook_hashes.as_any()), + )?; + let query_fingerprint: String = model_query_hash( + py, + &name, + ( + &model.getattr("query_sql")?.extract::()?, + dialect.as_deref(), + ), + )?; + let local_hash: String = model_local_identity_hash(&query_fingerprint, &metadata_json); + model_metadata.set_item(&name, metadata_json)?; + model_local.set_item(&name, &local_hash)?; + graph.nodes.insert( + key_of(MODEL_RESOURCE, &name), + IdentityNode { + upstream_keys: graph_keys(&model.getattr("deps")?)?, + local_hash: Some(local_hash), + }, + ); + } + let graph_hashes: HashMap> = + expected_graph_identity_hashes(&graph).into_iter().collect(); + let version_hashes: Bound<'py, PyDict> = function_hashes.copy()?; + version_hashes.update(seed_hashes.as_mapping())?; + for name in model_local.keys() { + let name: String = name.extract()?; + if let Some(Some(model_hash)) = graph_hashes.get(&key_of(MODEL_RESOURCE, &name)) { + version_hashes.set_item(&name, model_hash)?; + } + } + call_with( + &imported(py, MODELS_MODULE, "DirectModelVersionIdentities")?, + &[ + ("function_local_hashes", function_hashes.into_any()), + ("seed_version_hashes", seed_hashes.into_any()), + ("seed_metadata_jsons", seed_metadata.into_any()), + ("model_metadata_jsons", model_metadata.into_any()), + ("model_local_hashes", model_local.into_any()), + ("model_version_hashes", version_hashes.into_any()), + ], + ) +} + +/// Planned write hashes replaced by hashes based on upstreams available this run. +#[pyfunction] +#[pyo3(signature = (**options))] +fn with_honest_model_write_hashes<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "with_honest_model_write_hashes", + options, + ( + &["scope", "snapshot", "changes", "version_identities"], + &["available_model_hashes"], + ), + )?; + let scope: Bound<'py, PyAny> = arguments.value("scope")?; + let snapshot: Bound<'py, PyAny> = arguments.value("snapshot")?; + let changes: Bound<'py, PyAny> = arguments.value("changes")?; + let identities: Bound<'py, PyAny> = arguments.value("version_identities")?; + let selected_python: Bound<'py, PyAny> = scope.getattr("selected_keys")?; + let selected: HashSet = graph_keys(&selected_python)?.into_iter().collect(); + let function_hashes: Vec<(String, String)> = + string_pairs(&identities.getattr("function_local_hashes")?)?; + let mut graph = IdentityGraph::default(); + let mut base_hashes: Vec<(GraphKey, String)> = Vec::new(); + let mut function_deps: HashMap> = HashMap::new(); + for item in scope + .getattr("upstream_deps")? + .call_method0("items")? + .try_iter()? + { + let (key, deps): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let key: GraphKey = graph_key(&key)?; + if key.kind == UDF_RESOURCE || key.kind == TABLE_FN_RESOURCE { + function_deps.insert(key, graph_keys(&deps)?); + } + } + for (name, function_hash) in &function_hashes { + for kind in [UDF_RESOURCE, TABLE_FN_RESOURCE] { + let function_key: GraphKey = key_of(kind, name); + graph.nodes.insert( + function_key.clone(), + IdentityNode { + upstream_keys: function_deps + .get(&function_key) + .cloned() + .unwrap_or_default(), + local_hash: Some(function_hash.clone()), + }, + ); + base_hashes.push((function_key, function_hash.clone())); + } + } + let recorded_seeds: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("seeds")?; + for (name, seed_hash) in string_pairs(&identities.getattr("seed_version_hashes")?)? { + let seed_key: GraphKey = key_of(SEED_RESOURCE, &name); + graph.nodes.insert( + seed_key.clone(), + IdentityNode { + upstream_keys: Vec::new(), + local_hash: Some(seed_hash.clone()), + }, + ); + let fingerprint: Bound<'py, PyAny> = recorded_seeds.call_method1("get", (&name,))?; + let write_hash: String = if selected.contains(&seed_key) || fingerprint.is_none() { + seed_hash + } else { + fingerprint.getattr("version_hash")?.extract()? + }; + base_hashes.push((seed_key, write_hash)); + } + let local_hashes: Bound<'py, PyAny> = identities.getattr("model_local_hashes")?; + for model in scope + .getattr("models_by_name")? + .call_method0("values")? + .try_iter()? + { + let model: Bound<'py, PyAny> = model?; + let name: Bound<'py, PyAny> = model.getattr("name")?; + graph.nodes.insert( + key_of(MODEL_RESOURCE, &name.extract::()?), + IdentityNode { + upstream_keys: graph_keys(&model.getattr("deps")?)?, + local_hash: local_hashes.call_method1("get", (&name,))?.extract()?, + }, + ); + } + let recorded_models: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("models")?; + for item in recorded_models.call_method0("items")?.try_iter()? { + let (name, fingerprint): (String, Bound<'py, PyAny>) = item?.extract()?; + let version_hash: Bound<'py, PyAny> = fingerprint.getattr("version_hash")?; + if version_hash.is_truthy()? { + base_hashes.push((key_of(MODEL_RESOURCE, &name), version_hash.extract()?)); + } + } + let available: Bound<'py, PyAny> = arguments.value("available_model_hashes")?; + if available.is_truthy()? { + for (name, model_hash) in string_pairs(&available)? { + base_hashes.push((key_of(MODEL_RESOURCE, &name), model_hash)); + } + } + graph.execution_order = graph_keys(&scope.getattr("execution_order")?)?; + let write_hashes: HashMap = + graph_write_identity_hashes(&graph, &selected, base_hashes) + .into_iter() + .collect(); + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let model_changes: Bound<'py, PyDict> = PyDict::new(py); + model_changes.update( + changes + .getattr("models")? + .downcast::()? + .as_mapping(), + )?; + let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; + for key in scope.getattr("execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + let order_key: GraphKey = graph_key(&key)?; + if order_key.kind != MODEL_RESOURCE || !selected.contains(&order_key) { + continue; + } + let model: Bound<'py, PyAny> = models_by_name.call_method1("get", (&order_key.name,))?; + if model.is_none() { + continue; + } + let name: Bound<'py, PyAny> = model.getattr("name")?; + if local_hashes.call_method1("get", (&name,))?.is_none() { + continue; + } + let Some(write_hash) = + write_hashes.get(&key_of(MODEL_RESOURCE, &name.extract::()?)) + else { + continue; + }; + let Some(change) = model_changes.get_item(&name)? else { + continue; + }; + model_changes.set_item( + &name, + replace_with( + &replace, + &change, + ( + "fingerprint_version_hash", + PyString::new(py, write_hash).into_any(), + ), + )?, + )?; + } + replace_with(&replace, &changes, ("models", model_changes.into_any())) +} + +/// Resolved actions with recomputed model changes patched in where available. +#[pyfunction] +#[pyo3(signature = (*, resolved_actions, changes))] +fn merge_recomputed_model_changes<'py>( + resolved_actions: &Bound<'py, PyAny>, + changes: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = resolved_actions.py(); + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let recomputed: Bound<'py, PyAny> = changes.getattr("models")?; + let merged: Bound<'py, PyDict> = PyDict::new(py); + for item in resolved_actions + .getattr("models")? + .call_method0("items")? + .try_iter()? + { + let (name, resolved): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let mut change: Bound<'py, PyAny> = recomputed.call_method1("get", (&name,))?; + if change.is_none() { + change = resolved.getattr("change")?; + } + merged.set_item(name, replace_with(&replace, &resolved, ("change", change))?)?; + } + replace_with(&replace, resolved_actions, ("models", merged.into_any())) +} + +fn replace_with<'py>( + replace: &Bound<'py, PyAny>, + instance: &Bound<'py, PyAny>, + (field, value): (&str, Bound<'py, PyAny>), +) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(replace.py()); + keywords.set_item(field, value)?; + replace.call((instance,), Some(&keywords)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_direct_model_version_identities, + module + )?)?; + module.add_function(wrap_pyfunction!(with_honest_model_write_hashes, module)?)?; + module.add_function(wrap_pyfunction!(merge_recomputed_model_changes, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs new file mode 100644 index 0000000000..382d078218 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/json_values.rs @@ -0,0 +1,150 @@ +//! Python values as `json.dumps(value, sort_keys=True, ...)` writes them, with or without `default=str`. + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, + PyTypeMethods, +}; + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +const BUILTINS_MODULE: &str = "builtins"; + +/// Compact `(",", ":")` separators. +pub(crate) const COMPACT: (&str, &str) = (",", ":"); +/// Python's default `(", ", ": ")` separators. +pub(crate) const SPACED: (&str, &str) = (", ", ": "); + +/// `json.dumps(value, sort_keys=True, separators=separators)`, with `default=str` when `default_str`. +pub(crate) fn sorted_dumps( + value: &Bound<'_, PyAny>, + separators: (&str, &str), + default_str: bool, +) -> PyResult { + let encoded: JsonValue = Encoder::new(value.py(), default_str)?.encode(value)?; + let dialect = JsonDialect::Stdlib( + StdlibJsonOptions::new(None).with_separators(separators.0, separators.1), + ); + dumps(&encoded, &dialect).map_err(|error| PyValueError::new_err(format!("{error:?}"))) +} + +/// The C encoder's type dispatch, with dictionaries ordered as `sorted(dict.items())`. +struct Encoder<'py> { + default_str: bool, + int_repr: Bound<'py, PyAny>, + sorted: Bound<'py, PyAny>, +} + +impl<'py> Encoder<'py> { + fn new(py: Python<'py>, default_str: bool) -> PyResult { + let builtins = py.import(BUILTINS_MODULE)?; + Ok(Self { + default_str, + int_repr: builtins.getattr("int")?.getattr("__repr__")?, + sorted: builtins.getattr("sorted")?, + }) + } + + fn encode(&self, value: &Bound<'py, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(JsonValue::String(value.extract()?)); + } + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return Ok(JsonValue::Integer(self.integer(value)?)); + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(JsonValue::Array( + value + .try_iter()? + .map(|item| self.encode(&item?)) + .collect::>()?, + )); + } + if let Ok(mapping) = value.downcast::() { + return self.object(mapping); + } + if self.default_str { + return Ok(JsonValue::String(value.str()?.extract()?)); + } + Err(PyTypeError::new_err(format!( + "Object of type {} is not JSON serializable", + value.get_type().name()? + ))) + } + + fn integer(&self, value: &Bound<'py, PyAny>) -> PyResult { + let decimal: String = self.int_repr.call1((value,))?.extract()?; + JsonInteger::parse(&decimal) + .ok_or_else(|| PyValueError::new_err(format!("invalid integer text {decimal}"))) + } + + fn object(&self, mapping: &Bound<'py, PyDict>) -> PyResult { + let mut keys: Vec> = mapping.keys().into_iter().collect(); + if keys.iter().all(|key| key.is_instance_of::()) { + let mut texts: Vec<(String, Bound<'py, PyAny>)> = keys + .into_iter() + .map(|key| Ok((key.extract::()?, key))) + .collect::>()?; + texts.sort_by(|left, right| left.0.cmp(&right.0)); + keys = texts.into_iter().map(|(_, key)| key).collect(); + } else { + keys = self + .sorted + .call1((mapping.keys(),))? + .try_iter()? + .collect::>()?; + } + let mut entries: Vec<(String, JsonValue)> = Vec::with_capacity(keys.len()); + for key in keys { + let item: Bound<'py, PyAny> = mapping.as_any().get_item(&key)?; + entries.push((self.key_text(&key)?, self.encode(&item)?)); + } + Ok(JsonValue::Object(entries)) + } + + fn key_text(&self, key: &Bound<'py, PyAny>) -> PyResult { + if key.is_instance_of::() { + return key.extract(); + } + if key.is_instance_of::() { + let number: f64 = key.extract()?; + return Ok(if number.is_nan() { + "NaN".to_owned() + } else if number.is_infinite() { + if number.is_sign_positive() { + "Infinity" + } else { + "-Infinity" + } + .to_owned() + } else { + python_float_repr(number) + }); + } + if let Ok(flag) = key.downcast::() { + return Ok(if flag.is_true() { "true" } else { "false" }.to_owned()); + } + if key.is_none() { + return Ok("null".to_owned()); + } + if key.is_instance_of::() { + return Ok(self.integer(key)?.as_str().to_owned()); + } + Err(PyTypeError::new_err(format!( + "keys must be str, int, float, bool or None, not {}", + key.get_type().name()? + ))) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs new file mode 100644 index 0000000000..15dcb30116 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/metadata.rs @@ -0,0 +1,512 @@ +//! Version identity metadata JSON and hashes of models, functions and seeds, byte-identical to the recorded fingerprints. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + CONTRACT_CONFIG_KEY, CURSOR_INPUTS_CONFIG_KEY, ENFORCED_CONTRACT_POLICY, + MODEL_CUSTOM_CONFIG_KEY, MODEL_PLACEHOLDERS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY, + MODEL_PRE_HOOKS_CONFIG_KEY, SQL_HOOK_IDENTITY_FIELDS, TABLE_FN_RESOURCE, UDF_RESOURCE, + VERSION_IDENTITY_CONFIG_KEYS, +}; +use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::function_fingerprint_sql; +use sqlbuild_analysis::planner_identity::main::stable_identity_hash::stable_identity_hash; +use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; + +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords, within}; +use crate::bindings::_helpers::discovery::hook_entries::hook_entries; +use crate::bindings::_helpers::planner_identity::hashing::function_definition_hash; +use crate::bindings::_helpers::planner_identity::json_values::{COMPACT, SPACED, sorted_dumps}; + +const LOCAL_FUNCTION_HASHES_KEY: &str = "local_function_hashes"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const CSV_MODULE: &str = "csv"; +const BUILTINS_MODULE: &str = "builtins"; +const DEFAULT_SEED_ENCODING: &str = "utf-8-sig"; +/// CSV settings passed to `csv.reader` when set. +const SEED_DIALECT_OPTIONS: [&str; 5] = [ + "delimiter", + "quotechar", + "doublequote", + "escapechar", + "skipinitialspace", +]; + +/// The config entries that affect a model's produced version identity. +fn version_identity_config<'py>(config_values: &Bound<'py, PyAny>) -> PyResult> { + let identity: Bound<'py, PyDict> = PyDict::new(config_values.py()); + for key in std::iter::once(CURSOR_INPUTS_CONFIG_KEY).chain(VERSION_IDENTITY_CONFIG_KEYS) { + if config_values.contains(key)? { + identity.set_item(key, config_values.get_item(key)?)?; + } + } + Ok(identity) +} + +/// `value or {}`. +fn or_empty<'py>(value: Option>, py: Python<'py>) -> PyResult> { + match value { + Some(value) if value.is_truthy()? => Ok(value), + _ => Ok(PyDict::new(py).into_any()), + } +} + +/// Deterministic metadata JSON for model version identity. +pub(crate) fn version_identity_metadata_json( + model_name: &Bound<'_, PyAny>, + config_values: &Bound<'_, PyAny>, + (local_function_hashes, execution_signature): ( + Option>, + Option>, + ), +) -> PyResult { + let py: Python<'_> = model_name.py(); + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("config", version_identity_config(config_values)?)?; + payload.set_item("execution_signature", or_empty(execution_signature, py)?)?; + payload.set_item( + LOCAL_FUNCTION_HASHES_KEY, + or_empty(local_function_hashes, py)?, + )?; + payload.set_item("model_name", model_name)?; + sorted_dumps(payload.as_any(), COMPACT, true) +} + +/// Deterministic non-query model fingerprint metadata JSON. +pub(crate) fn model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option<&Bound<'py, PyAny>>, + hook_version_hashes: Option<&Bound<'py, PyAny>>, +) -> PyResult { + let py: Python<'py> = model.py(); + let local_function_hashes: Bound<'py, PyDict> = PyDict::new(py); + if let Some(function_hashes) = function_local_hashes.filter(|hashes| !hashes.is_none()) { + for upstream_key in model.getattr("deps")?.try_iter()? { + let upstream_key: Bound<'py, PyAny> = upstream_key?; + let kind: String = upstream_key.getattr("resource_type")?.str()?.extract()?; + if kind != UDF_RESOURCE && kind != TABLE_FN_RESOURCE { + continue; + } + let name: Bound<'py, PyAny> = upstream_key.getattr("name")?; + let upstream_hash: Bound<'py, PyAny> = function_hashes.call_method1("get", (&name,))?; + if !upstream_hash.is_none() { + local_function_hashes.set_item(name, upstream_hash)?; + } + } + } + let config_values: Bound<'py, PyAny> = model.getattr("config")?.getattr("values")?; + let signature: Bound<'py, PyDict> = + execution_signature(model, &config_values, hook_version_hashes)?; + version_identity_metadata_json( + &model.getattr("name")?, + &config_values, + ( + Some(local_function_hashes.into_any()), + Some(signature.into_any()), + ), + ) +} + +fn execution_signature<'py>( + model: &Bound<'py, PyAny>, + config_values: &Bound<'py, PyAny>, + hook_version_hashes: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let signature: Bound<'py, PyDict> = PyDict::new(py); + if let Some(contract) = contract_output_signature(model, config_values)? { + signature.set_item("contract", contract)?; + } + for (config_key, signature_key) in [ + (MODEL_CUSTOM_CONFIG_KEY, "custom_config"), + (MODEL_PLACEHOLDERS_CONFIG_KEY, "custom_placeholders"), + ] { + if config_values.contains(config_key)? { + signature.set_item(signature_key, config_values.get_item(config_key)?)?; + } + } + let python_hook_fields: Bound<'py, PyDict> = PyDict::new(py); + if let Some(hashes) = hook_version_hashes.filter(|hashes| !hashes.is_none()) { + for item in hashes.call_method0("items")?.try_iter()? { + let (name, version_hash): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("version_hash", version_hash)?; + python_hook_fields.set_item(name, fields)?; + } + } + let sql_fields: Vec = SQL_HOOK_IDENTITY_FIELDS + .iter() + .map(|field| (*field).to_owned()) + .collect(); + for config_key in [MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY] { + if config_values.contains(config_key)? { + signature.set_item( + config_key, + hook_entries( + &config_values.get_item(config_key)?, + &sql_fields, + Some(&python_hook_fields), + )?, + )?; + } + } + Ok(signature) +} + +/// Required model output metadata that participates in execution identity. +fn contract_output_signature<'py>( + model: &Bound<'py, PyAny>, + config_values: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = model.py(); + let schema_entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; + if schema_entry.is_none() { + return Ok(None); + } + let columns: Bound<'py, PyAny> = schema_entry.getattr("columns")?; + let families: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + if !(columns.is_truthy()? || families.is_truthy()?) { + return Ok(None); + } + let enforced: bool = config_values + .call_method1("get", (CONTRACT_CONFIG_KEY,))? + .eq(ENFORCED_CONTRACT_POLICY)?; + if !enforced && schema_entry.getattr("model_schema")?.is_none() { + return Ok(None); + } + let enum_columns: Bound<'py, PyAny> = model.getattr("enum_columns")?; + let rendered: Bound<'py, PyList> = PyList::empty(py); + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let entry: Bound<'py, PyDict> = column_signature(&column)?; + let declaration: Bound<'py, PyAny> = + enum_columns.call_method1("get", (column.getattr("name")?,))?; + if !declaration.is_none() { + let members: Bound<'py, PyList> = PyList::empty(py); + for member in declaration.getattr("members")?.try_iter()? { + let member: Bound<'py, PyAny> = member?; + let rendered_member: Bound<'py, PyDict> = PyDict::new(py); + rendered_member.set_item("name", member.getattr("name")?)?; + rendered_member.set_item("value", member.getattr("value")?)?; + members.append(rendered_member)?; + } + let enum_entry: Bound<'py, PyDict> = PyDict::new(py); + enum_entry.set_item("name", declaration.getattr("name")?)?; + enum_entry.set_item("members", members)?; + entry.set_item("enum", enum_entry)?; + } + rendered.append(entry)?; + } + let signature: Bound<'py, PyDict> = PyDict::new(py); + signature.set_item("enforced", enforced)?; + signature.set_item("columns", rendered)?; + if families.is_truthy()? { + signature.set_item("dynamic_columns", family_signatures(&families)?)?; + } + Ok(Some(signature)) +} + +fn column_signature<'py>(column: &Bound<'py, PyAny>) -> PyResult> { + let entry: Bound<'py, PyDict> = PyDict::new(column.py()); + for field in ["name", "type", "nullable"] { + entry.set_item(field, column.getattr(field)?)?; + } + Ok(entry) +} + +fn family_signatures<'py>(families: &Bound<'py, PyAny>) -> PyResult> { + let rendered: Bound<'py, PyList> = PyList::empty(families.py()); + for family in families.try_iter()? { + let family: Bound<'py, PyAny> = family?; + let entry: Bound<'py, PyDict> = PyDict::new(families.py()); + for field in [ + "name", + "pivot_column", + "value_column", + "aggregate", + "type", + "name_pattern", + ] { + entry.set_item(field, family.getattr(field)?)?; + } + rendered.append(entry)?; + } + Ok(rendered) +} + +/// The hash of a model's own declared columns, column families and contract setting. +pub(crate) fn model_declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + let schema_entry: Bound<'_, PyAny> = model.getattr("schema_entry")?; + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item( + "contract", + model + .getattr("config")? + .getattr("values")? + .call_method1("get", (CONTRACT_CONFIG_KEY,))?, + )?; + let columns: Bound<'_, PyList> = PyList::empty(py); + let mut families: Bound<'_, PyList> = PyList::empty(py); + let mut type_enforcement: Bound<'_, PyAny> = py.None().into_bound(py); + if !schema_entry.is_none() { + for column in schema_entry.getattr("columns")?.try_iter()? { + columns.append(column_signature(&column?)?)?; + } + families = family_signatures(&schema_entry.getattr("dynamic_columns")?)?; + type_enforcement = schema_entry.getattr("type_enforcement")?; + } + payload.set_item("columns", columns)?; + payload.set_item("dynamic_columns", families)?; + payload.set_item("type_enforcement", type_enforcement)?; + Ok(stable_identity_hash(&sorted_dumps( + payload.as_any(), + COMPACT, + true, + )?)) +} + +/// Local-only semantic hashes of functions by name; SQL bodies ignore layout and comments. +pub(crate) fn function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, +) -> PyResult> { + let py: Python<'py> = functions.py(); + let hashes: Bound<'py, PyDict> = PyDict::new(py); + for function in functions.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let name: String = function.getattr("name")?.extract()?; + let language: Bound<'py, PyAny> = function.getattr("language")?.getattr("value")?; + let body_hash: String = function_definition_hash( + py, + &name, + ( + &function.getattr("body_sql")?.extract::()?, + &language.extract::()?, + dialect, + ), + )?; + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item( + "arguments", + name_type_pairs(&function.getattr("arguments")?)?, + )?; + payload.set_item("returns", function.getattr("returns")?)?; + payload.set_item( + "return_columns", + name_type_pairs(&function.getattr("return_columns")?)?, + )?; + payload.set_item("body_sql", body_hash)?; + payload.set_item("language", language)?; + for field in ["runtime_version", "entry_point", "packages"] { + payload.set_item(field, function.getattr(field)?)?; + } + hashes.set_item( + name, + stable_identity_hash(&sorted_dumps(payload.as_any(), SPACED, true)?), + )?; + } + Ok(hashes) +} + +fn name_type_pairs<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + let pairs: Bound<'py, PyList> = PyList::empty(items.py()); + for item in items.try_iter()? { + let item: Bound<'py, PyAny> = item?; + pairs.append(PyTuple::new( + items.py(), + [item.getattr("name")?, item.getattr("type")?], + )?)?; + } + Ok(pairs) +} + +/// `f"{value or ''}"` of an optional attribute. +fn text_or_empty(owner: &Bound<'_, PyAny>, name: &str) -> PyResult { + let value: Bound<'_, PyAny> = match owner.getattr(name) { + Ok(value) => value, + Err(_) => return Ok(String::new()), + }; + if value.is_truthy()? { + value.str()?.extract() + } else { + Ok(String::new()) + } +} + +fn rendered_pairs(items: &Bound<'_, PyAny>) -> PyResult> { + items + .try_iter()? + .map(|item| { + let item: Bound<'_, PyAny> = item?; + Ok((text_or_empty(&item, "name")?, text_or_empty(&item, "type")?)) + }) + .collect() +} + +/// Stable function definition text used for change fingerprints. +pub(crate) fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + Ok(function_fingerprint_sql(&FunctionFingerprintFields { + language: function.getattr("language")?.str()?.extract()?, + arguments: rendered_pairs(&function.getattr("arguments")?)?, + returns: function.getattr("returns")?.str()?.extract()?, + return_columns: rendered_pairs(&function.getattr("return_columns")?)?, + runtime_version: text_or_empty(function, "runtime_version")?, + entry_point: text_or_empty(function, "entry_point")?, + packages: function + .getattr("packages")? + .try_iter()? + .map(|package| package?.extract::()) + .collect::>()?, + body_sql: function.getattr("body_sql")?.extract()?, + })) +} + +/// `(identity_hash, metadata_json)` of one compiled seed from its content and load-affecting config. +pub(crate) fn seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + let py: Python<'_> = seed.py(); + let schema_entry: Bound<'_, PyAny> = seed.getattr("schema_entry")?; + let csv_settings: Bound<'_, PyAny> = schema_entry.getattr("csv_settings")?; + let columns: Bound<'_, PyList> = PyList::empty(py); + for column in schema_entry.getattr("columns")?.try_iter()? { + columns.append(column_signature(&column?)?)?; + } + let rows: Bound<'_, PyAny> = if seed.getattr("external")?.is_truthy()? { + PyList::empty(py).into_any() + } else { + seed_rows( + &seed.getattr("seed_file")?.getattr("file_path")?, + &csv_settings, + )? + }; + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("name", seed.getattr("name")?)?; + payload.set_item( + "csv_settings", + imported(py, DATACLASSES_MODULE, "asdict")?.call1((&csv_settings,))?, + )?; + payload.set_item("columns", columns)?; + payload.set_item("rows", rows)?; + let metadata_json: String = sorted_dumps(payload.as_any(), COMPACT, false)?; + Ok((stable_identity_hash(&metadata_json), metadata_json)) +} + +/// Every row of the seed file as the standard library CSV reader yields it. +fn seed_rows<'py>( + file_path: &Bound<'py, PyAny>, + csv_settings: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = file_path.py(); + let encoding: Bound<'py, PyAny> = match csv_settings.getattr("encoding")? { + value if value.is_truthy()? => value, + _ => PyString::new(py, DEFAULT_SEED_ENCODING).into_any(), + }; + let options: Bound<'py, PyDict> = PyDict::new(py); + for option in SEED_DIALECT_OPTIONS { + let value: Bound<'py, PyAny> = csv_settings.getattr(option)?; + if !value.is_none() { + options.set_item(option, value)?; + } + } + let handle: Bound<'py, PyAny> = imported(py, BUILTINS_MODULE, "open")?.call( + (file_path, "r"), + Some(&keywords( + py, + &[ + ("encoding", encoding), + ("newline", PyString::new(py, "").into_any()), + ], + )?), + )?; + within(&handle, |entered| { + let rows: Bound<'py, PyList> = PyList::empty(py); + let reader: Bound<'py, PyAny> = + imported(py, CSV_MODULE, "reader")?.call((entered,), Some(&options))?; + for row in reader.try_iter()? { + rows.append(PyList::new( + py, + row?.try_iter()?.collect::>>()?, + )?)?; + } + Ok(rows.into_any()) + }) +} + +/// Deterministic metadata JSON for model version identity. +#[pyfunction] +#[pyo3(signature = (*, model_name, config_values, local_function_hashes = None, execution_signature = None))] +fn build_version_identity_metadata_json<'py>( + model_name: &Bound<'py, PyAny>, + config_values: &Bound<'py, PyAny>, + local_function_hashes: Option>, + execution_signature: Option>, +) -> PyResult { + version_identity_metadata_json( + model_name, + config_values, + (local_function_hashes, execution_signature), + ) +} + +/// Deterministic non-query model fingerprint metadata JSON. +#[pyfunction] +#[pyo3(signature = (*, model, function_local_hashes = None, hook_version_hashes = None))] +fn build_model_version_identity_metadata_json<'py>( + model: &Bound<'py, PyAny>, + function_local_hashes: Option>, + hook_version_hashes: Option>, +) -> PyResult { + model_version_identity_metadata_json( + model, + function_local_hashes.as_ref(), + hook_version_hashes.as_ref(), + ) +} + +/// Hash the model's own declared columns, column families and contract setting. +#[pyfunction] +#[pyo3(signature = (*, model))] +fn declared_columns_hash(model: &Bound<'_, PyAny>) -> PyResult { + model_declared_columns_hash(model) +} + +/// Local-only semantic hashes of functions by name. +#[pyfunction] +#[pyo3(signature = (*, functions, dialect))] +fn build_function_local_hashes<'py>( + functions: &Bound<'py, PyAny>, + dialect: Option<&str>, +) -> PyResult> { + function_local_hashes(functions, dialect) +} + +/// Stable function definition text used for change fingerprints. +#[pyfunction] +fn build_compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResult { + compiled_function_fingerprint_sql(function) +} + +/// `(identity_hash, metadata_json)` for one compiled seed. +#[pyfunction] +fn build_seed_identity(seed: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + seed_identity(seed) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!( + build_model_version_identity_metadata_json, + module + )?)?; + module.add_function(wrap_pyfunction!(declared_columns_hash, module)?)?; + module.add_function(wrap_pyfunction!(build_function_local_hashes, module)?)?; + module.add_function(wrap_pyfunction!( + build_compiled_function_fingerprint_sql, + module + )?)?; + module.add_function(wrap_pyfunction!(build_seed_identity, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs new file mode 100644 index 0000000000..638074cc97 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod graph; +pub(crate) mod hashing; +pub(crate) mod identities; +pub(crate) mod json_values; +pub(crate) mod metadata; +pub(crate) mod registration; +pub(crate) mod staleness; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs new file mode 100644 index 0000000000..f9df4e384b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/registration.rs @@ -0,0 +1,15 @@ +//! Register planner identity, pruning and graph-order bindings. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_identity::{ + graph, hashing, identities, metadata, staleness, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + graph::register(module)?; + hashing::register(module)?; + identities::register(module)?; + metadata::register(module)?; + staleness::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs new file mode 100644 index 0000000000..f1a0332f64 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -0,0 +1,290 @@ +//! Warnings for selected models stale through changed upstreams outside the run set. + +use std::collections::{HashMap, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use sqlbuild_analysis::planner_identity::constants::{ + MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, +}; +use sqlbuild_analysis::planner_identity::main::classify_selection_staleness::classify_selection_staleness; +use sqlbuild_analysis::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message; +use sqlbuild_analysis::planner_identity::models::{ + GraphKey, StaleWarning, StalenessGraph, StalenessNames, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; +use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_identity::hashing::graph_key; + +/// Change kinds that mean the model's own identity changed. +const OWN_CHANGE_KINDS: [&str; 5] = [ + "first_run", + "query_changed", + "function_changed", + "config_changed", + "schema_changed", +]; + +fn own_change(change: &Bound<'_, PyAny>) -> PyResult { + let kind: String = change.getattr("change_kind")?.str()?.extract()?; + Ok(OWN_CHANGE_KINDS.contains(&kind.as_str())) +} + +fn names_of_kind(keys: &Bound<'_, PyAny>, kind: &str) -> PyResult> { + let mut names: HashSet = HashSet::new(); + for key in keys.try_iter()? { + let key: GraphKey = graph_key(&key?)?; + if key.kind == kind { + names.insert(key.name); + } + } + Ok(names) +} + +fn upstream_graph(upstream_deps: &Bound<'_, PyAny>) -> PyResult>> { + let mut deps: HashMap> = HashMap::new(); + for item in upstream_deps.call_method0("items")?.try_iter()? { + let (key, upstream): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let upstream_keys: Vec = upstream + .try_iter()? + .map(|upstream_key| graph_key(&upstream_key?)) + .collect::>()?; + deps.insert(graph_key(&key)?, upstream_keys); + } + Ok(deps) +} + +/// Inputs of one stale-warning pass. +struct StalenessInputs<'py> { + original_scope: Bound<'py, PyAny>, + changes: Bound<'py, PyAny>, + snapshot: Bound<'py, PyAny>, + identities: Bound<'py, PyAny>, + model_changes_complete: bool, +} + +fn changed_model_names(inputs: &StalenessInputs<'_>) -> PyResult> { + let py: Python<'_> = inputs.snapshot.py(); + let models: Bound<'_, PyAny> = inputs.changes.getattr("models")?; + let mut changed: HashSet = HashSet::new(); + for item in models.call_method0("items")?.try_iter()? { + let (name, change): (String, Bound<'_, PyAny>) = item?.extract()?; + if own_change(&change)? { + changed.insert(name); + } + } + let version_hashes: Bound<'_, PyAny> = inputs.identities.getattr("model_version_hashes")?; + let metadata_jsons: Bound<'_, PyAny> = inputs.identities.getattr("model_metadata_jsons")?; + let pass = ModelPass { + snapshot: inputs.snapshot.clone(), + query_change_tracking: true, + full_refresh: false, + function_local_hashes: py.None().into_bound(py), + hook_version_hashes: py.None().into_bound(py), + }; + let models_by_name: Bound<'_, PyAny> = inputs.original_scope.getattr("models_by_name")?; + for item in models_by_name.call_method0("items")?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let selected_change: Bound<'_, PyAny> = models.call_method1("get", (&name,))?; + if !selected_change.is_none() + && (inputs.model_changes_complete || own_change(&selected_change)?) + { + if own_change(&selected_change)? { + changed.insert(name); + } + continue; + } + if model.is_none() { + continue; + } + let change: Bound<'_, PyAny> = model_change( + &pass, + &model, + ( + version_hashes.call_method1("get", (&name,))?.extract()?, + metadata_jsons.call_method1("get", (&name,))?.extract()?, + ), + )?; + if own_change(&change)? { + changed.insert(name); + } + } + Ok(changed) +} + +fn changed_seed_names(inputs: &StalenessInputs<'_>) -> PyResult> { + let recorded: Bound<'_, PyAny> = inputs.snapshot.getattr("fingerprints")?.getattr("seeds")?; + let mut changed: HashSet = HashSet::new(); + for item in inputs + .identities + .getattr("seed_version_hashes")? + .call_method0("items")? + .try_iter()? + { + let (name, expected): (String, Bound<'_, PyAny>) = item?.extract()?; + if expected.is_none() { + continue; + } + let fingerprint: Bound<'_, PyAny> = recorded.call_method1("get", (&name,))?; + if fingerprint.is_none() || !fingerprint.getattr("version_hash")?.eq(&expected)? { + changed.insert(name); + } + } + Ok(changed) +} + +fn plan_warnings<'py>( + py: Python<'py>, + warnings: Vec, +) -> PyResult> { + let warning_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "PlanWarning")?; + let severity: Bound<'py, PyAny> = planner_enum(py, "WarningSeverity", "warning")?; + let mut rendered: Vec> = Vec::with_capacity(warnings.len()); + for warning in warnings { + let message: String = stale_upstream_warning_message( + ("selected model", &warning.model_name), + "upstream(s)", + &warning.trigger_names, + ); + rendered.push(call_with( + &warning_class, + &[ + ( + "model_name", + PyString::new(py, &warning.model_name).into_any(), + ), + ("severity", severity.clone()), + ("message", PyString::new(py, &message).into_any()), + ], + )?); + } + PyTuple::new(py, rendered) +} + +/// Warn for selected models stale through changed upstreams outside the run set. +#[pyfunction] +#[pyo3(signature = (**options))] +fn build_stale_out_of_selection_warnings<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_stale_out_of_selection_warnings", + options, + ( + &[ + "original_scope", + "execution_scope", + "changes", + "snapshot", + "version_identities", + "source_freshness", + ], + &["include_sources", "model_changes_complete"], + ), + )?; + let include_sources: bool = + !arguments.passed("include_sources")? || arguments.flag("include_sources")?; + let inputs = StalenessInputs { + original_scope: arguments.value("original_scope")?, + changes: arguments.value("changes")?, + snapshot: arguments.value("snapshot")?, + identities: arguments.value("version_identities")?, + model_changes_complete: arguments.flag("model_changes_complete")?, + }; + let run_keys: Bound<'py, PyAny> = arguments + .value("execution_scope")? + .getattr("selected_keys")?; + let source_freshness: Bound<'py, PyAny> = arguments.value("source_freshness")?; + let mut changed_sources: HashSet = HashSet::new(); + if include_sources && !source_freshness.is_none() { + for identity in source_freshness.getattr("changed_identities")?.try_iter()? { + changed_sources.insert(identity?.getattr("source_name")?.extract()?); + } + } + let graph = StalenessGraph { + upstream_deps: upstream_graph(&inputs.original_scope.getattr("upstream_deps")?)?, + selected_model_names: names_of_kind( + &inputs.original_scope.getattr("selected_keys")?, + MODEL_RESOURCE, + )? + .into_iter() + .collect(), + run: StalenessNames { + models: names_of_kind(&run_keys, MODEL_RESOURCE)?, + seeds: names_of_kind(&run_keys, SEED_RESOURCE)?, + sources: names_of_kind(&run_keys, SOURCE_RESOURCE)?, + }, + changed: StalenessNames { + models: changed_model_names(&inputs)?, + seeds: changed_seed_names(&inputs)?, + sources: changed_sources, + }, + }; + plan_warnings(py, classify_selection_staleness(&graph)) +} + +fn name_set(values: &Bound<'_, PyAny>) -> PyResult> { + values.try_iter()?.map(|value| value?.extract()).collect() +} + +/// Stale warnings for selected models with changed upstreams outside the run set. +#[pyfunction] +fn classify_selection_staleness_warnings<'py>( + graph: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let staleness = StalenessGraph { + upstream_deps: upstream_graph(&graph.getattr("upstream_deps")?)?, + selected_model_names: name_set(&graph.getattr("selected_model_names")?)? + .into_iter() + .collect(), + run: StalenessNames { + models: name_set(&graph.getattr("run_model_names")?)?, + seeds: name_set(&graph.getattr("run_seed_names")?)?, + sources: name_set(&graph.getattr("run_source_names")?)?, + }, + changed: StalenessNames { + models: name_set(&graph.getattr("changed_model_names")?)?, + seeds: name_set(&graph.getattr("changed_seed_names")?)?, + sources: name_set(&graph.getattr("changed_source_names")?)?, + }, + }; + let warning_class: Bound<'py, PyAny> = + imported(py, MODELS_MODULE, "SelectionStalenessWarning")?; + let mut warnings: Vec> = Vec::new(); + for warning in classify_selection_staleness(&staleness) { + warnings.push(call_with( + &warning_class, + &[ + ( + "model_name", + PyString::new(py, &warning.model_name).into_any(), + ), + ( + "trigger_names", + PyTuple::new(py, warning.trigger_names)?.into_any(), + ), + ], + )?); + } + PyTuple::new(py, warnings) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + build_stale_out_of_selection_warnings, + module + )?)?; + module.add_function(wrap_pyfunction!( + classify_selection_staleness_warnings, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 070cb9fca0..e1d55181ec 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,9 +7,9 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, auditing, authored_values, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, - lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_strategy, - project_assembly, refactor, references, render, rules, scopes, semantic_checks, - semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_identity, + planner_strategy, project_assembly, refactor, references, render, rules, scopes, + semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -46,6 +46,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { refactor::registration::register(module)?; plan_output::registration::register(module)?; planner_changes::registration::register(module)?; + planner_identity::registration::register(module)?; planner_strategy::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; diff --git a/python-boundary.toml b/python-boundary.toml index 1df4a81de8..d757220997 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 49845 +remaining_lines = 47826 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5851044964..1d75d18537 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -198,8 +198,10 @@ from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, CursorBounds, DeferralInputs, + DirectModelVersionIdentities, FutureCursorSafetyEvidence, MaximumStartSafetyEvidence, + MissingUpstream, ModelPlanEntry, PlannerChangeResults, PlannerResolvedActions, @@ -210,6 +212,8 @@ from sqlbuild.compiler.planner.models import ( ReferenceRename, SchemaAction, SchemaFinding, + SelectionStalenessGraph, + SelectionStalenessWarning, SqlTestPlanEntry, WarehouseSnapshot, ) @@ -257,7 +261,11 @@ from sqlbuild.compiler.scopes.models import ( VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, JsonValue, ScopeQueryTarget -from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord +from sqlbuild.compiler.source_freshness.models import ( + DirectSourceFreshnessPlanningResult, + SourceFreshnessIdentity, + SourceFreshnessRecord, +) from sqlbuild.compiler.source_freshness.types import ( SourceFreshnessAgeStatus, SourceFreshnessComparableRecord, @@ -3992,6 +4000,92 @@ def origin_reference_names( ) -> dict[str, str] | None: ... def with_declared_columns_hash(*, metadata_json: str, declared_columns_hash: str) -> str: ... def without_declared_columns_hash(metadata_json: str) -> str: ... +def model_definition_hash(*, model_name: str, query_sql: str, dialect: str | None) -> str: ... +def build_version_identity_metadata_json( + *, + model_name: str, + config_values: Mapping[str, object], + local_function_hashes: Mapping[str, str] | None = None, + execution_signature: Mapping[str, object] | None = None, +) -> str: ... +def build_model_version_identity_metadata_json( + *, + model: CompiledModelView, + function_local_hashes: Mapping[str, str] | None = None, + hook_version_hashes: Mapping[str, str] | None = None, +) -> str: ... +def declared_columns_hash(*, model: CompiledModelView) -> str: ... +def build_function_local_hashes( + *, functions: Iterable[CompiledFunctionView], dialect: str | None +) -> dict[str, str]: ... +def build_compiled_function_fingerprint_sql(function: CompiledFunctionView) -> str: ... +def build_seed_identity(seed: CompiledSeedView) -> tuple[str, str]: ... +def build_direct_model_version_identities( + *, + functions: Iterable[CompiledFunctionView], + scope: PlannerScope, + dialect: str | None, + seeds: Iterable[CompiledSeedView] = (), + source_version_hashes: Mapping[str, str] | None = None, + hook_functions: Iterable[DiscoveredHookFunction] = (), +) -> DirectModelVersionIdentities: ... +def with_honest_model_write_hashes( + *, + scope: PlannerScope, + snapshot: WarehouseSnapshot, + changes: PlannerChangeResults, + version_identities: DirectModelVersionIdentities, + available_model_hashes: Mapping[str, str] | None = None, +) -> PlannerChangeResults: ... +def merge_recomputed_model_changes( + *, resolved_actions: PlannerResolvedActions, changes: PlannerChangeResults +) -> PlannerResolvedActions: ... +def build_stale_out_of_selection_warnings( + *, + original_scope: PlannerScope, + execution_scope: PlannerScope, + changes: PlannerChangeResults, + snapshot: WarehouseSnapshot, + version_identities: DirectModelVersionIdentities, + source_freshness: DirectSourceFreshnessPlanningResult | None, + include_sources: bool = True, + model_changes_complete: bool = False, +) -> tuple[PlanWarning, ...]: ... +def classify_selection_staleness_warnings( + graph: SelectionStalenessGraph, +) -> tuple[SelectionStalenessWarning, ...]: ... +def build_execution_upstream_deps( + project: CompiledProjectView, +) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... +def build_execution_edge_origins( + project: CompiledProjectView, +) -> dict[tuple[CompiledObjectKey, CompiledObjectKey], str]: ... +def build_downstream_deps( + upstream: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... +def topologically_order_keys( + *, + upstream: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + injected_edge_origins: Mapping[tuple[CompiledObjectKey, CompiledObjectKey], str] | None = None, +) -> tuple[CompiledObjectKey, ...]: ... +def build_downstream_model_name_closure( + *, + start_keys: Iterable[CompiledObjectKey], + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> frozenset[str]: ... +def check_buildability( + *, + selected_keys: Set[CompiledObjectKey], + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + snapshot: WarehouseSnapshot, + deferred_relations: Mapping[str, RelationInfo] | None = None, + satisfied_keys: Set[CompiledObjectKey] | None = None, +) -> tuple[MissingUpstream, ...]: ... +def missing_upstream_message( + *, + missing: tuple[MissingUpstream, ...], + edge_origins: Mapping[tuple[CompiledObjectKey, CompiledObjectKey], str], +) -> str: ... def model_resource_type(entry: ModelPlanEntry | None) -> str: ... def model_execution_annotation(entry: ModelPlanEntry | None) -> str: ... def materialization_type_display(resource_kind: ExecutionResourceKind) -> str: ... diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index fa198254ff..642a82b8af 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -2,15 +2,15 @@ from __future__ import annotations -from sqlbuild._native import get_materialization_type +from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, + build_seed_identity, + get_materialization_type, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) -from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity from sqlbuild.compiler.planner._helpers.output.plan_entry import ( build_planner_relations_context, extract_seed_columns, diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/buildability.py b/src/sqlbuild/compiler/planner/_helpers/graph/buildability.py deleted file mode 100644 index 7bd34e5d8e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/buildability.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Buildability validation for selected scope against warehouse state.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import MissingUpstream, WarehouseSnapshot - -_MESSAGE_ITEM_LIMIT: int = 5 - - -def check_buildability( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - snapshot: WarehouseSnapshot, - deferred_relations: dict[str, RelationInfo] | None = None, - satisfied_keys: frozenset[CompiledObjectKey] = frozenset(), -) -> tuple[MissingUpstream, ...]: - """Validate that all upstream deps for selected keys exist in scope or warehouse.""" - - missing_map: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - - selected_key: CompiledObjectKey - for selected_key in selected_keys: - dep_keys: tuple[CompiledObjectKey, ...] = upstream_deps.get(selected_key, ()) - dep_key: CompiledObjectKey - for dep_key in dep_keys: - if dep_key.resource_type == CompiledResourceType.SQL_TEST: - continue - if dep_key.resource_type == CompiledResourceType.DBT_REF: - continue - if dep_key.resource_type == CompiledResourceType.SOURCE: - continue - if dep_key in selected_keys: - continue - if dep_key in satisfied_keys: - continue - if dep_key.name in snapshot.existing_relations: - continue - if deferred_relations is not None and dep_key.name in deferred_relations: - continue - missing_map.setdefault(dep_key, []).append(selected_key) - - missing: list[MissingUpstream] = [ - MissingUpstream( - key=key, - required_by=tuple(sorted(dependents, key=lambda k: (k.resource_type, k.name))), - ) - for key, dependents in sorted( - missing_map.items(), - key=lambda item: (-len(item[1]), item[0].resource_type, item[0].name), - ) - ] - - return tuple(missing) - - -def missing_upstream_message( - *, - missing: tuple[MissingUpstream, ...], - edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str], -) -> str: - """Describe missing upstream keys, naming the audit when only an audit read requires one.""" - - lineage_missing: list[MissingUpstream] = [] - audit_reads: dict[str, None] = {} - entry: MissingUpstream - for entry in missing: - origins: tuple[str, ...] = tuple( - edge_origins[(dependent, entry.key)] - for dependent in entry.required_by - if (dependent, entry.key) in edge_origins - ) - if len(origins) < len(entry.required_by): - lineage_missing.append(entry) - continue - audit_reads.update(dict.fromkeys(origins)) - parts: list[str] = [] - if lineage_missing: - names: str = ", ".join(m.key.name for m in lineage_missing[:_MESSAGE_ITEM_LIMIT]) - parts.append(f"{len(lineage_missing)} missing upstream dependencies ({names})") - read_names: tuple[str, ...] = tuple(audit_reads) - if len(read_names) == 1: - parts.append( - f"{read_names[0]}, which is not selected and does not exist in the warehouse; " - "select it or build it first" - ) - elif read_names: - shown: str = "; ".join(read_names[:_MESSAGE_ITEM_LIMIT]) - more: str = "; ..." if len(read_names) > _MESSAGE_ITEM_LIMIT else "" - parts.append( - f"{len(read_names)} audit reads are not selected and do not exist in the warehouse " - f"({shown}{more}); select them or build them first" - ) - return "cannot build selected scope: " + "; ".join(parts) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/core.py b/src/sqlbuild/compiler/planner/_helpers/graph/core.py deleted file mode 100644 index d83d9f744c..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/core.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Graph traversal helpers for planning over compiled project state.""" - -from __future__ import annotations - -import heapq - -from sqlbuild._native import ( - attached_audit_gate_edges, - hook_read_edges, - invert_edges, - native_graph_key, - transitive_closure, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, -) -from sqlbuild.compiler.compile.types import ( - CompiledProjectView, - CompiledResourceType, - SqlTestMode, -) -from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge -from sqlbuild.compiler.planner.exceptions import PlannerInputError - - -def _key_sort_key(key: CompiledObjectKey) -> tuple[str, str]: - return (key.resource_type, key.name) - - -def build_execution_upstream_deps( - project: CompiledProjectView, -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Return execution-order edges from lineage and woven SQL test nodes.""" - - upstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - upstream.update({model.key: list(model.deps) for model in project.models}) - upstream.update({source.key: list(source.deps) for source in project.sources}) - upstream.update({seed.key: list(seed.deps) for seed in project.seeds}) - upstream.update({function.key: list(function.deps) for function in project.functions}) - - test: CompiledSqlTest - for test in project.sql_tests: - if test.mode == SqlTestMode.TABLE_FN: - upstream[test.key] = list(_function_scope_deps_for_test(test=test)) - continue - upstream[test.key] = list(_function_deps_for_test(test=test, upstream=upstream)) - for target_key in test.scope_deps: - if target_key in upstream: - upstream[target_key].append(test.key) - - edge: AttachedAuditGateEdge - for edge in attached_audit_gate_edges(project=project): - gated_upstream: list[CompiledObjectKey] = upstream.setdefault(edge.gated, []) - if edge.read not in gated_upstream: - gated_upstream.append(edge.read) - - hook_edge: HookReadEdge - for hook_edge in hook_read_edges(project=project): - hook_upstream: list[CompiledObjectKey] = upstream.setdefault(hook_edge.gated, []) - if hook_edge.read not in hook_upstream: - hook_upstream.append(hook_edge.read) - - return {k: tuple(v) for k, v in upstream.items()} - - -def build_execution_edge_origins( - project: CompiledProjectView, -) -> dict[tuple[CompiledObjectKey, CompiledObjectKey], str]: - """Return human-readable origins for execution edges injected beyond pure lineage.""" - - origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str] = {} - test: CompiledSqlTest - for test in project.sql_tests: - for target_key in test.scope_deps: - origins[(target_key, test.key)] = ( - f"SQL test '{test.name}' runs before '{target_key.name}'" - ) - edge: AttachedAuditGateEdge - for edge in attached_audit_gate_edges(project=project): - if _is_lineage_edge(project=project, gated=edge.gated, read=edge.read): - continue - origins.setdefault( - (edge.gated, edge.read), - f"audit '{edge.audit_name}' on '{edge.target.name}' reads '{edge.read.name}'", - ) - hook_edge: HookReadEdge - for hook_edge in hook_read_edges(project=project): - if _is_lineage_edge(project=project, gated=hook_edge.gated, read=hook_edge.read): - continue - origins.setdefault( - (hook_edge.gated, hook_edge.read), - f"{hook_edge.label} on '{hook_edge.gated.name}' reads '{hook_edge.read.name}'", - ) - return origins - - -def _is_lineage_edge( - *, project: CompiledProjectView, gated: CompiledObjectKey, read: CompiledObjectKey -) -> bool: - return native_graph_key(read) in project.lineage_graph.upstream(native_graph_key(gated)) - - -def _function_scope_deps_for_test(*, test: CompiledSqlTest) -> tuple[CompiledObjectKey, ...]: - return tuple( - scope_dep - for scope_dep in test.scope_deps - if scope_dep.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN} - ) - - -def _function_deps_for_test( - *, - test: CompiledSqlTest, - upstream: dict[CompiledObjectKey, list[CompiledObjectKey]], -) -> tuple[CompiledObjectKey, ...]: - deps: list[CompiledObjectKey] = [] - seen: set[CompiledObjectKey] = set() - target_key: CompiledObjectKey - for target_key in test.scope_deps: - dep_key: CompiledObjectKey - for dep_key in upstream.get(target_key, ()): - if dep_key.resource_type not in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - continue - if dep_key in seen: - continue - seen.add(dep_key) - deps.append(dep_key) - return tuple(deps) - - -def build_downstream_deps( - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Return downstream edges keyed by upstream object key.""" - - return invert_edges(edges=upstream, sort_key=_key_sort_key) - - -def topologically_order_keys( - *, - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - injected_edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str] | None = None, -) -> tuple[CompiledObjectKey, ...]: - """Return all keys in stable dependency order using Kahn's algorithm.""" - - node_keys: set[CompiledObjectKey] = set(upstream) - dep_keys: tuple[CompiledObjectKey, ...] - for dep_keys in upstream.values(): - dep_key: CompiledObjectKey - for dep_key in dep_keys: - node_keys.add(dep_key) - - indegree: dict[CompiledObjectKey, int] = {key: 0 for key in node_keys} - key: CompiledObjectKey - for key, dep_keys in upstream.items(): - indegree[key] = len(dep_keys) - - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream - ) - ready: list[tuple[tuple[str, str], CompiledObjectKey]] = [ - (_key_sort_key(key), key) for key, count in indegree.items() if count == 0 - ] - heapq.heapify(ready) - ordered: list[CompiledObjectKey] = [] - - while ready: - current: CompiledObjectKey = heapq.heappop(ready)[1] - ordered.append(current) - downstream_key: CompiledObjectKey - for downstream_key in downstream.get(current, ()): - if downstream_key not in indegree: - continue - indegree[downstream_key] -= 1 - if indegree[downstream_key] == 0: - heapq.heappush(ready, (_key_sort_key(downstream_key), downstream_key)) - - if len(ordered) != len(node_keys): - ordered_set: set[CompiledObjectKey] = set(ordered) - cycle_keys: tuple[CompiledObjectKey, ...] = tuple( - sorted( - (key for key in node_keys if key not in ordered_set), - key=_key_sort_key, - ) - ) - cycle_names: str = ", ".join(f"{k.resource_type}:{k.name}" for k in cycle_keys) - origin_notes: str = _cycle_origin_notes( - cycle_keys=cycle_keys, - upstream=upstream, - injected_edge_origins=injected_edge_origins or {}, - ) - raise PlannerInputError(f"Dependency cycle detected involving: {cycle_names}{origin_notes}") - - return tuple(ordered) - - -def _cycle_origin_notes( - *, - cycle_keys: tuple[CompiledObjectKey, ...], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - injected_edge_origins: dict[tuple[CompiledObjectKey, CompiledObjectKey], str], -) -> str: - """Describe injected (non-lineage) edges between cycle keys so users see the closure cause.""" - - if not injected_edge_origins: - return "" - cycle_key_set: set[CompiledObjectKey] = set(cycle_keys) - notes: list[str] = [] - key: CompiledObjectKey - for key in cycle_keys: - dep_key: CompiledObjectKey - for dep_key in upstream.get(key, ()): - if dep_key not in cycle_key_set: - continue - origin: str | None = injected_edge_origins.get((key, dep_key)) - if origin is not None: - notes.append(origin) - if not notes: - return "" - return " (via " + "; ".join(sorted(set(notes))) + ")" - - -def expand_upstream( - *, - key: CompiledObjectKey, - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return all transitive upstream keys reachable from the given key.""" - - return transitive_closure(start=key, edges=upstream) - - -def expand_downstream( - *, - key: CompiledObjectKey, - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return all transitive downstream keys reachable from the given key.""" - - return transitive_closure(start=key, edges=downstream) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py b/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py deleted file mode 100644 index a64410fd46..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Pure model-name closure helpers for planner dependency graphs.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure_many -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType - - -def build_downstream_model_name_closure( - *, - start_keys: tuple[CompiledObjectKey, ...], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - """Return model names reachable downstream from the given keys, including model roots.""" - - return _build_model_name_closure(start_keys=start_keys, deps=downstream_deps) - - -def _build_model_name_closure( - *, - start_keys: tuple[CompiledObjectKey, ...], - deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - return frozenset( - key.name - for key in transitive_closure_many(starts=start_keys, edges=deps, include_starts=True) - if key.resource_type == CompiledResourceType.MODEL - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py index 1bca778617..a2e84bf9be 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py @@ -2,16 +2,16 @@ from __future__ import annotations -from sqlbuild._native import lineage_graph_names -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys -from sqlbuild.compiler.planner._helpers.graph.core import ( +from sqlbuild._native import ( build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, + lineage_graph_names, topologically_order_keys, ) +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys from sqlbuild.compiler.planner._helpers.graph.loader_dag import expand_selected_loader_dependencies from sqlbuild.compiler.planner._helpers.graph.selectors import ( parse_selector, diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py deleted file mode 100644 index 44b861581e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Direct planner model version identity helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledResourceType, - CompiledSeedView, -) -from sqlbuild.compiler.discovery.models import DiscoveredHookFunction -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - build_model_local_identity_hash, - compose_native_graph_identity, - graph_key_for_compiled_resource, - model_definition_hash, -) -from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity -from sqlbuild.compiler.planner.main.identity._graph_identity import ( - build_expected_graph_identity_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) -from sqlbuild.compiler.planner.models import ( - DirectModelVersionIdentities, - GraphIdentityNode, - GraphNodeKey, - PlannerScope, -) -from sqlbuild.compiler.planner.types import GraphResourceKind -from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities - - -def build_direct_model_version_identities( - *, - functions: tuple[CompiledFunctionView, ...], - seeds: tuple[CompiledSeedView, ...] = (), - scope: PlannerScope, - source_version_hashes: dict[str, str] | None = None, - hook_functions: tuple[DiscoveredHookFunction, ...] = (), - dialect: str | None, -) -> DirectModelVersionIdentities: - """Compute current direct model identities from code and upstream identities.""" - - function_local_hashes: dict[str, str] = build_function_local_hashes( - functions=functions, dialect=dialect - ) - hook_version_hashes: dict[str, str] = { - name: identity.version_hash - for name, identity in build_hook_identities(hook_functions).items() - } - seed_version_hashes: dict[str, str] = {} - seed_metadata_jsons: dict[str, str] = {} - seed: CompiledSeedView - for seed in seeds: - seed_hash: str - seed_metadata_json: str - seed_hash, seed_metadata_json = build_seed_identity(seed) - seed_version_hashes[seed.name] = seed_hash - seed_metadata_jsons[seed.name] = seed_metadata_json - model_metadata_jsons: dict[str, str] = {} - model_local_hashes: dict[str, str] = {} - graph_nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - function: CompiledFunctionView - for function in functions: - function_hash: str | None = function_local_hashes.get(function.name) - if function_hash is None: - continue - function_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=function.key.resource_type, - name=function.name, - ) - graph_nodes[function_key] = GraphIdentityNode( - key=function_key, - resource_kind=GraphResourceKind.FUNCTION, - upstream_keys=tuple( - graph_key_for_compiled_resource( - resource_type=dep.resource_type, - name=dep.name, - ) - for dep in function.deps - ), - local_hash=function_hash, - ) - seed_name: str - for seed_name, seed_hash in seed_version_hashes.items(): - seed_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - graph_nodes[seed_key] = GraphIdentityNode( - key=seed_key, - resource_kind=GraphResourceKind.SEED, - upstream_keys=(), - local_hash=seed_hash, - ) - source_hashes: dict[str, str] = source_version_hashes or {} - source_name: str - source_hash: str - for source_name, source_hash in source_hashes.items(): - source_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SOURCE, - name=source_name, - ) - graph_nodes[source_key] = GraphIdentityNode( - key=source_key, - resource_kind=GraphResourceKind.SOURCE, - upstream_keys=(), - local_hash=source_hash, - ) - - key: object - for key in scope.execution_order: - if not hasattr(key, "resource_type") or key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = scope.models_by_name.get(key.name) - if model is None: - continue - metadata_json: str = build_model_version_identity_metadata_json( - model=model, - function_local_hashes=function_local_hashes, - hook_version_hashes=hook_version_hashes, - ) - model_metadata_jsons[model.name] = metadata_json - local_hash: str = build_model_local_identity_hash( - query_fingerprint=model_definition_hash( - model_name=model.name, query_sql=model.query_sql, dialect=dialect - ), - metadata_json=metadata_json, - ) - model_local_hashes[model.name] = local_hash - model_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model.name, - ) - graph_nodes[model_key] = GraphIdentityNode( - key=model_key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=tuple( - graph_key_for_compiled_resource(resource_type=dep.resource_type, name=dep.name) - for dep in model.deps - ), - local_hash=local_hash, - ) - - graph_hashes: dict[GraphNodeKey, str | None] = build_expected_graph_identity_hashes( - nodes=graph_nodes, - execution_order=tuple( - graph_key_for_compiled_resource(resource_type=key.resource_type, name=key.name) - for key in scope.execution_order - if hasattr(key, "resource_type") - ), - compose_identity=compose_native_graph_identity, - ) - model_version_hashes: dict[str, str] = {**function_local_hashes, **seed_version_hashes} - model_name: str - for model_name in model_local_hashes: - model_hash: str | None = graph_hashes.get( - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model_name, - ) - ) - if model_hash is not None: - model_version_hashes[model_name] = model_hash - - return DirectModelVersionIdentities( - function_local_hashes=function_local_hashes, - seed_version_hashes=seed_version_hashes, - seed_metadata_jsons=seed_metadata_jsons, - model_metadata_jsons=model_metadata_jsons, - model_local_hashes=model_local_hashes, - model_version_hashes=model_version_hashes, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py deleted file mode 100644 index ac2bb6b279..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Function fingerprint source helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - FunctionArgument, - FunctionReturnColumn, -) -from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash -from sqlbuild.compiler.planner.types import PlanReason - - -def build_compiled_function_fingerprint_sql(function: CompiledFunctionView) -> str: - """Build stable function definition text used for change fingerprints.""" - - return build_function_fingerprint_sql( - arguments=function.arguments, - returns=function.returns, - return_columns=function.return_columns, - body_sql=function.body_sql, - language=str(function.language), - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - - -def detect_function_change( - *, - function: CompiledFunctionView, - fingerprint_sql: str, - fingerprint: Fingerprint | None, - query_change_tracking: bool, - full_refresh: bool, - dialect: str | None, -) -> PlanReason: - """Resolve why a function is redeployed; callers decide their own replay.""" - - if full_refresh: - return PlanReason.FULL_REFRESH - if fingerprint is None: - return PlanReason.FIRST_RUN - if query_change_tracking and ( - function_definition_hash( - function_name=function.name, - fingerprint_sql=fingerprint_sql, - language=str(function.language), - dialect=dialect, - ) - != fingerprint.definition_hash - ): - return PlanReason.QUERY_CHANGED - return PlanReason.NO_CHANGE - - -def build_function_fingerprint_sql( - *, - arguments: tuple[FunctionArgument | object, ...], - returns: str, - body_sql: str, - language: str, - runtime_version: str | None, - entry_point: str | None, - packages: tuple[str, ...], - return_columns: tuple[FunctionReturnColumn | object, ...] = (), -) -> str: - """Build stable function definition text used for change fingerprints.""" - - rendered_arguments: str = ",".join(_render_argument(arg) for arg in arguments) - rendered_return_columns: str = ",".join( - _render_return_column(column) for column in return_columns - ) - rendered_packages: str = ",".join(packages) - return "\n".join( - ( - f"language={language}", - f"arguments={rendered_arguments}", - f"returns={returns}", - f"return_columns={rendered_return_columns}", - f"runtime_version={runtime_version or ''}", - f"entry_point={entry_point or ''}", - f"packages={rendered_packages}", - "body=", - body_sql, - ) - ) - - -def _render_argument(argument: FunctionArgument | object) -> str: - name: object | None = getattr(argument, "name", None) - arg_type: object | None = getattr(argument, "type", None) - return f"{name or ''}:{arg_type or ''}" - - -def _render_return_column(column: FunctionReturnColumn | object) -> str: - name: object | None = getattr(column, "name", None) - col_type: object | None = getattr(column, "type", None) - return f"{name or ''}:{col_type or ''}" diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/graph.py b/src/sqlbuild/compiler/planner/_helpers/identity/graph.py deleted file mode 100644 index be520d01e0..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/graph.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Pure graph identity resolution helpers.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner.classes.graph_identity_resolver import GraphIdentityResolver -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - - -def build_expected_graph_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str | None]: - resolver: GraphIdentityResolver = GraphIdentityResolver( - nodes=nodes, compose_identity=compose_identity, base_hashes={}, selected_keys=None - ) - key: GraphNodeKey - for key in execution_order: - _ = resolver.resolve_expected(key) - return resolver.hashes - - -def build_graph_write_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - selected_keys: frozenset[GraphNodeKey], - base_identity_hashes: Mapping[GraphNodeKey, str], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str]: - resolver: GraphIdentityResolver = GraphIdentityResolver( - nodes=nodes, - compose_identity=compose_identity, - base_hashes=base_identity_hashes, - selected_keys=selected_keys, - ) - key: GraphNodeKey - for key in execution_order: - _ = resolver.resolve_write(key) - return {hashed_key: value for hashed_key, value in resolver.hashes.items() if value is not None} diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py b/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py deleted file mode 100644 index 6160a72aac..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Model version identity hashing helpers.""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable -from functools import partial - -from sqlbuild._native import ( - compute_function_definition_hash, - compute_query_hash, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.fingerprints.constants import ( - NODE_TYPE_MODEL, - NODE_TYPE_SEED, - NODE_TYPE_TABLE_FN, - NODE_TYPE_UDF, -) -from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError -from sqlbuild.compiler.planner.constants import QUERY_FINGERPRINT_FAILED -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import GraphNodeKey - -_FINGERPRINT_HELP: str = ( - "SQLBuild fingerprints the dialect token stream of compiled SQL to detect changes. " - "Fix the SQL so the warehouse dialect tokenizer accepts it." -) - - -def stable_version_identity_hash(value: str) -> str: - """Return the stable SHA-256 hash used for local and composed identities.""" - - return hashlib.sha256(value.encode("utf-8")).hexdigest() - - -def model_definition_hash(*, model_name: str, query_sql: str, dialect: str | None) -> str: - """Fingerprint a model query for change detection, naming the model when it cannot.""" - - return _planner_fingerprint( - subject=f"Model '{model_name}' SQL", - compute=partial(compute_query_hash, query_sql=query_sql, dialect=dialect), - ) - - -def function_definition_hash( - *, function_name: str, fingerprint_sql: str, language: str, dialect: str | None -) -> str: - """Fingerprint a function definition for change detection, naming it when it cannot.""" - - return _planner_fingerprint( - subject=f"Function '{function_name}'", - compute=partial( - compute_function_definition_hash, - fingerprint_sql=fingerprint_sql, - language=language, - dialect=dialect, - ), - ) - - -def _planner_fingerprint(*, subject: str, compute: Callable[[], str]) -> str: - try: - return compute() - except QueryFingerprintError as error: - raise PlannerInputError( - f"{subject} cannot be fingerprinted for change detection: {error}", - code=QUERY_FINGERPRINT_FAILED, - help=_FINGERPRINT_HELP, - ) from None - - -def build_model_local_identity_hash(*, query_fingerprint: str, metadata_json: str) -> str: - """Build a model's local identity hash from its query fingerprint and non-query metadata.""" - - return stable_version_identity_hash("\n".join((query_fingerprint, metadata_json))) - - -def graph_key_for_compiled_resource( - *, resource_type: str | CompiledResourceType, name: str -) -> GraphNodeKey: - normalized: CompiledResourceType | None = None - if isinstance(resource_type, CompiledResourceType): - normalized = resource_type - else: - try: - normalized = CompiledResourceType(resource_type) - except ValueError: - normalized = None - node_type: str = ( - resource_type.value if isinstance(resource_type, CompiledResourceType) else resource_type - ) - if normalized == CompiledResourceType.MODEL: - node_type = NODE_TYPE_MODEL - elif normalized == CompiledResourceType.SEED: - node_type = NODE_TYPE_SEED - elif normalized == CompiledResourceType.UDF: - node_type = NODE_TYPE_UDF - elif normalized == CompiledResourceType.TABLE_FN: - node_type = NODE_TYPE_TABLE_FN - return GraphNodeKey(node_type=node_type, node_name=name) - - -def compose_native_graph_identity( - *, local_hash: str, upstream_hashes: tuple[tuple[GraphNodeKey, str], ...] -) -> str: - return stable_version_identity_hash( - "\n".join((local_hash, *(upstream_hash for _, upstream_hash in upstream_hashes))) - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py b/src/sqlbuild/compiler/planner/_helpers/identity/honest.py deleted file mode 100644 index 2e8cb7ee6f..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Build direct model write hashes from actually available upstream identities.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - compose_native_graph_identity, - graph_key_for_compiled_resource, -) -from sqlbuild.compiler.planner.main.identity._graph_write_identity import ( - build_graph_write_identity_hashes, -) -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DirectModelVersionIdentities, - GraphIdentityNode, - GraphNodeKey, - PlannerChangeResults, - PlannerResolvedActions, - PlannerScope, - ResolvedModelAction, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import GraphResourceKind - - -def with_honest_model_write_hashes( - *, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - changes: PlannerChangeResults, - version_identities: DirectModelVersionIdentities, - available_model_hashes: dict[str, str] | None = None, -) -> PlannerChangeResults: - """Replace planned write hashes with hashes based on upstreams available this run.""" - - base_hashes: dict[GraphNodeKey, str] = _base_write_hashes( - scope=scope, - snapshot=snapshot, - version_identities=version_identities, - available_model_hashes=available_model_hashes, - ) - graph_nodes: dict[GraphNodeKey, GraphIdentityNode] = _graph_identity_nodes( - scope=scope, - version_identities=version_identities, - ) - write_hashes: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=graph_nodes, - execution_order=tuple( - graph_key_for_compiled_resource(resource_type=key.resource_type, name=key.name) - for key in scope.execution_order - ), - selected_keys=frozenset( - graph_key_for_compiled_resource(resource_type=key.resource_type, name=key.name) - for key in scope.selected_keys - ), - base_identity_hashes=base_hashes, - compose_identity=compose_native_graph_identity, - ) - models_by_name: dict[str, CompiledModelView] = scope.models_by_name - model_changes: dict[str, ChangeDetectionResult] = dict(changes.models) - key: CompiledObjectKey - for key in scope.execution_order: - if key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModelView | None = models_by_name.get(key.name) - if model is None: - continue - if key not in scope.selected_keys: - continue - local_hash: str | None = version_identities.model_local_hashes.get(model.name) - if local_hash is None: - continue - graph_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model.name, - ) - write_hash: str | None = write_hashes.get(graph_key) - if write_hash is None: - continue - change: ChangeDetectionResult | None = model_changes.get(model.name) - if change is None: - continue - model_changes[model.name] = replace(change, fingerprint_version_hash=write_hash) - return replace(changes, models=model_changes) - - -def merge_recomputed_model_changes( - *, - resolved_actions: PlannerResolvedActions, - changes: PlannerChangeResults, -) -> PlannerResolvedActions: - """Patch recomputed model changes into resolved actions, keeping run-despite-unchanged kinds.""" - - merged_models: dict[str, ResolvedModelAction] = {} - model_name: str - resolved: ResolvedModelAction - for model_name, resolved in resolved_actions.models.items(): - merged_models[model_name] = replace( - resolved, - change=_merged_recomputed_change( - resolved_change=resolved.change, - recomputed_change=changes.models.get(model_name), - ), - ) - return replace(resolved_actions, models=merged_models) - - -def _merged_recomputed_change( - *, - resolved_change: ChangeDetectionResult, - recomputed_change: ChangeDetectionResult | None, -) -> ChangeDetectionResult: - """Prefer a recomputed change when available.""" - - if recomputed_change is None: - return resolved_change - return recomputed_change - - -def _graph_identity_nodes( - *, scope: PlannerScope, version_identities: DirectModelVersionIdentities -) -> dict[GraphNodeKey, GraphIdentityNode]: - nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - function_name: str - function_hash: str - for function_name, function_hash in version_identities.function_local_hashes.items(): - for resource_type in (CompiledResourceType.UDF, CompiledResourceType.TABLE_FN): - function_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=resource_type, - name=function_name, - ) - nodes[function_key] = GraphIdentityNode( - key=function_key, - resource_kind=GraphResourceKind.FUNCTION, - upstream_keys=tuple( - graph_key_for_compiled_resource( - resource_type=dep.resource_type, - name=dep.name, - ) - for dep in scope.upstream_deps.get( - CompiledObjectKey(resource_type=resource_type, name=function_name), - (), - ) - ), - local_hash=function_hash, - ) - seed_name: str - seed_hash: str - for seed_name, seed_hash in version_identities.seed_version_hashes.items(): - seed_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - nodes[seed_key] = GraphIdentityNode( - key=seed_key, - resource_kind=GraphResourceKind.SEED, - upstream_keys=(), - local_hash=seed_hash, - ) - model: CompiledModelView - for model in scope.models_by_name.values(): - model_key: GraphNodeKey = graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model.name, - ) - nodes[model_key] = GraphIdentityNode( - key=model_key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=tuple( - graph_key_for_compiled_resource(resource_type=dep.resource_type, name=dep.name) - for dep in model.deps - ), - local_hash=version_identities.model_local_hashes.get(model.name), - ) - return nodes - - -def _base_write_hashes( - *, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - available_model_hashes: dict[str, str] | None, -) -> dict[GraphNodeKey, str]: - hashes: dict[GraphNodeKey, str] = {} - function_name: str - function_hash: str - for function_name, function_hash in version_identities.function_local_hashes.items(): - for resource_type in (CompiledResourceType.UDF, CompiledResourceType.TABLE_FN): - hashes[ - graph_key_for_compiled_resource(resource_type=resource_type, name=function_name) - ] = function_hash - seed_name: str - seed_hash: str - for seed_name, seed_hash in _seed_write_hashes( - scope=scope, - snapshot=snapshot, - version_identities=version_identities, - ).items(): - hashes[ - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - ] = seed_hash - model_name: str - model_hash: str - for model_name, model_hash in _built_model_hashes(snapshot).items(): - hashes[ - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model_name, - ) - ] = model_hash - for model_name, model_hash in (available_model_hashes or {}).items(): - hashes[ - graph_key_for_compiled_resource( - resource_type=CompiledResourceType.MODEL, - name=model_name, - ) - ] = model_hash - return hashes - - -def _seed_write_hashes( - *, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, -) -> dict[str, str]: - hashes: dict[str, str] = {} - seed_name: str - expected_hash: str - for seed_name, expected_hash in version_identities.seed_version_hashes.items(): - seed_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SEED, name=seed_name - ) - if seed_key in scope.selected_keys: - hashes[seed_name] = expected_hash - continue - fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) - hashes[seed_name] = fingerprint.version_hash if fingerprint is not None else expected_hash - return hashes - - -def _built_model_hashes(snapshot: WarehouseSnapshot) -> dict[str, str]: - return { - model_name: fingerprint.version_hash - for model_name, fingerprint in snapshot.fingerprints.models.items() - if fingerprint.version_hash - } diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py b/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py deleted file mode 100644 index 00ec07931c..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Model contract metadata used by version identity.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.discovery.models import EnumDeclaration -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry - - -def contract_output_signature(*, model: CompiledModelView) -> dict[str, object] | None: - """Build required model output metadata that participates in execution identity.""" - - schema_entry: SchemaModelEntry | None = model.schema_entry - if schema_entry is None or not (schema_entry.columns or schema_entry.dynamic_columns): - return None - enforced: bool = model.config.values.get("contract") == ContractPolicy.ENFORCED - if not enforced and schema_entry.model_schema is None: - return None - signature: dict[str, object] = { - "enforced": enforced, - "columns": [ - _column_output_signature(model=model, column=column) for column in schema_entry.columns - ], - } - if schema_entry.dynamic_columns: - signature["dynamic_columns"] = [ - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - } - for family in schema_entry.dynamic_columns - ] - return signature - - -def _column_output_signature( - *, model: CompiledModelView, column: SchemaColumn -) -> dict[str, object]: - signature: dict[str, object] = { - "name": column.name, - "type": column.type, - "nullable": column.nullable, - } - enum_declaration: EnumDeclaration | None = model.enum_columns.get(column.name) - if enum_declaration is not None: - signature["enum"] = { - "name": enum_declaration.name, - "members": [ - {"name": member.name, "value": member.value} for member in enum_declaration.members - ], - } - return signature - - -def declared_columns_hash(*, model: CompiledModelView) -> str: - """Hash the model's own declared columns, column families, and contract setting.""" - - schema_entry: SchemaModelEntry | None = model.schema_entry - payload: dict[str, object] = { - "contract": model.config.values.get("contract"), - "columns": [ - {"name": column.name, "type": column.type, "nullable": column.nullable} - for column in (schema_entry.columns if schema_entry is not None else ()) - ], - "dynamic_columns": [ - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - } - for family in (schema_entry.dynamic_columns if schema_entry is not None else ()) - ], - "type_enforcement": schema_entry.type_enforcement if schema_entry is not None else None, - } - return hashlib.sha256( - json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") - ).hexdigest() diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/seed.py b/src/sqlbuild/compiler/planner/_helpers/identity/seed.py deleted file mode 100644 index e957ed8126..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/identity/seed.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Stable seed identity helpers for direct changes-only planning.""" - -from __future__ import annotations - -import csv -import json -from dataclasses import asdict -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.planner._helpers.identity.hashing import stable_version_identity_hash -from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings - - -def build_seed_identity(seed: CompiledSeedView) -> tuple[str, str]: - """Return ``(identity_hash, metadata_json)`` for one compiled seed.""" - - metadata_json: str = build_seed_identity_metadata_json(seed) - return stable_version_identity_hash(metadata_json), metadata_json - - -def build_seed_identity_metadata_json(seed: CompiledSeedView) -> str: - """Build deterministic seed identity metadata from content and load-affecting config.""" - - payload: dict[str, object] = { - "name": seed.name, - "csv_settings": _normalize_json_value(asdict(seed.schema_entry.csv_settings)), - "columns": [_column_identity(column) for column in seed.schema_entry.columns], - "rows": [] - if seed.external - else _read_seed_rows( - file_path=seed.seed_file.file_path, - csv_settings=seed.schema_entry.csv_settings, - ), - } - return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) - - -def _read_seed_rows(*, file_path: Path, csv_settings: SeedCsvSettings) -> list[list[str]]: - encoding: str = csv_settings.encoding or "utf-8-sig" - dialect_options: dict[str, Any] = {} - if csv_settings.delimiter is not None: - dialect_options["delimiter"] = csv_settings.delimiter - if csv_settings.quotechar is not None: - dialect_options["quotechar"] = csv_settings.quotechar - if csv_settings.doublequote is not None: - dialect_options["doublequote"] = csv_settings.doublequote - if csv_settings.escapechar is not None: - dialect_options["escapechar"] = csv_settings.escapechar - if csv_settings.skipinitialspace is not None: - dialect_options["skipinitialspace"] = csv_settings.skipinitialspace - - rows: list[list[str]] = [] - with file_path.open("r", encoding=encoding, newline="") as handle: - reader: Any = csv.reader(handle, **dialect_options) - for row in reader: - rows.append(list(row)) - return rows - - -def _column_identity(column: SchemaColumn) -> dict[str, object]: - return { - "name": column.name, - "type": column.type, - "nullable": column.nullable, - } - - -def _normalize_json_value(value: object) -> object: - if isinstance(value, dict): - return {str(k): _normalize_json_value(v) for k, v in sorted(value.items())} - if isinstance(value, (list, tuple)): - return [_normalize_json_value(v) for v in value] - return value diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index dd3f39880d..c4b55e4ef3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from dataclasses import dataclass, replace -from sqlbuild._native import get_config_str +from sqlbuild._native import get_config_str, model_definition_hash from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType @@ -16,7 +16,6 @@ from sqlbuild.compiler.migrations.main.relation_for_location import migration_relation_for_location from sqlbuild.compiler.migrations.models import ColumnMigrationEvent from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDiscovery -from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash from sqlbuild.compiler.planner._helpers.migrations.column_renames import ( identical_renames, rename_hints, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index 5ddc97cc42..d454e09865 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -6,7 +6,11 @@ from collections import defaultdict from typing import Any -from sqlbuild._native import get_config_str +from sqlbuild._native import ( + build_function_local_hashes, + build_model_version_identity_metadata_json, + get_config_str, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -26,12 +30,6 @@ MigrationStateInspection, ) from sqlbuild.compiler.planner.constants import MIGRATION_FINGERPRINT_METADATA_KEY -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import ( ModelMigrationDeclaration, ModelMigrationDiscovery, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py index fc1eb330af..1e6d2a951d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py @@ -2,13 +2,13 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.buildability import ( +from sqlbuild._native import ( + build_execution_edge_origins, check_buildability, missing_upstream_message, ) -from sqlbuild.compiler.planner._helpers.graph.core import build_execution_edge_origins +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.warehouse.source_tables import ( check_selected_source_tables_exist, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py index 1be7c7f043..30bcab34c4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py @@ -4,12 +4,9 @@ from dataclasses import replace -from sqlbuild._native import detect_model_changes_in_scope +from sqlbuild._native import build_direct_model_version_identities, detect_model_changes_in_scope from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.identity.direct import ( - build_direct_model_version_identities, -) from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index 5b8870b3da..b103f79c02 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -4,15 +4,13 @@ from dataclasses import replace +from sqlbuild._native import build_stale_out_of_selection_warnings from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner._helpers.migrations.entry_fingerprints import ( with_migration_fingerprints, ) from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output from sqlbuild.compiler.planner._helpers.planning.retention import plan_retention, plan_table_types -from sqlbuild.compiler.planner._helpers.pruning.selection_staleness import ( - build_stale_out_of_selection_warnings, -) from sqlbuild.compiler.planner._helpers.warehouse.source_freshness_warnings import ( build_source_freshness_unknown_warnings, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py b/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py index 1524814beb..d1070fcc6b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/reconciliation.py @@ -2,10 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner._helpers.identity.honest import ( - merge_recomputed_model_changes, - with_honest_model_write_hashes, -) +from sqlbuild._native import merge_recomputed_model_changes, with_honest_model_write_hashes from sqlbuild.compiler.planner.models import ( PlannerChangeReconciliation, PlannerChangeResults, diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py deleted file mode 100644 index 01e499e278..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_classifier.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Pure selection-aware staleness classification helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - SelectionStalenessGraph, - SelectionStalenessNodeKey, - SelectionStalenessWarning, -) - -_STALE_TRIGGER_DISPLAY_LIMIT: int = 5 - - -def format_stale_upstream_warning_message( - *, - model_label: str, - model_name: str, - trigger_label: str, - trigger_names: tuple[str, ...], -) -> str: - """Build a multi-line stale-selection warning: summary, capped bullet list, single hint.""" - - count: int = len(trigger_names) - displayed: tuple[str, ...] = trigger_names[:_STALE_TRIGGER_DISPLAY_LIMIT] - lines: list[str] = [ - f"{model_label} '{model_name}' will build on {count} stale {trigger_label} " - "not selected for rebuild:" - ] - name: str - for name in displayed: - lines.append(f" - {name}") - if count > _STALE_TRIGGER_DISPLAY_LIMIT: - lines.append(f" +{count - _STALE_TRIGGER_DISPLAY_LIMIT} more") - lines.append(f" rebuild the closure to refresh them: --select +{model_name}") - return "\n".join(lines) - - -def classify_selection_staleness_warnings( - graph: SelectionStalenessGraph, -) -> tuple[SelectionStalenessWarning, ...]: - """Return stale warnings for selected models with changed upstreams outside the run set.""" - - warnings: list[SelectionStalenessWarning] = [] - for model_name in sorted(graph.selected_model_names): - triggers: tuple[str, ...] = _changed_upstream_names( - graph=graph, - model_key=SelectionStalenessNodeKey( - resource_type=CompiledResourceType.MODEL.value, - name=model_name, - ), - ) - if triggers: - warnings.append( - SelectionStalenessWarning(model_name=model_name, trigger_names=triggers) - ) - return tuple(warnings) - - -def _changed_upstream_names( - *, - graph: SelectionStalenessGraph, - model_key: SelectionStalenessNodeKey, -) -> tuple[str, ...]: - names: set[str] = set() - visiting: set[SelectionStalenessNodeKey] = set() - stale_by_key: dict[SelectionStalenessNodeKey, bool] = {} - - def visit( - *, - upstream_key: SelectionStalenessNodeKey, - found_names: set[str], - visiting_keys: set[SelectionStalenessNodeKey], - stale_cache: dict[SelectionStalenessNodeKey, bool], - ) -> tuple[ - bool, set[str], set[SelectionStalenessNodeKey], dict[SelectionStalenessNodeKey, bool] - ]: - cached: bool | None = stale_cache.get(upstream_key) - if cached is not None: - return cached, found_names, visiting_keys, stale_cache - if upstream_key in visiting_keys: - return False, found_names, visiting_keys, stale_cache - visiting_keys = visiting_keys | {upstream_key} - is_stale: bool - if upstream_key.resource_type == CompiledResourceType.MODEL.value: - in_run_set: bool = upstream_key.name in graph.run_model_names - own_changed: bool = upstream_key.name in graph.changed_model_names - if own_changed and not in_run_set: - is_stale = True - found_names = found_names | {upstream_key.name} - else: - ancestor_stale: bool = False - parent_key: SelectionStalenessNodeKey - for parent_key in graph.upstream_deps.get(upstream_key, ()): - parent_stale, found_names, visiting_keys, stale_cache = visit( - upstream_key=parent_key, - found_names=found_names, - visiting_keys=visiting_keys, - stale_cache=stale_cache, - ) - ancestor_stale = parent_stale or ancestor_stale - if _run_parent_changed(graph=graph, parent_key=parent_key): - ancestor_stale = True - is_stale = ancestor_stale - if ancestor_stale and not in_run_set and upstream_key != model_key: - found_names = found_names | {upstream_key.name} - elif upstream_key.resource_type == CompiledResourceType.SEED.value: - changed: bool = upstream_key.name in graph.changed_seed_names - if changed and upstream_key.name not in graph.run_seed_names: - found_names = found_names | {upstream_key.name} - is_stale = changed - elif upstream_key.resource_type == CompiledResourceType.SOURCE.value: - changed = upstream_key.name in graph.changed_source_names - if changed and upstream_key.name not in graph.run_source_names: - found_names = found_names | {upstream_key.name} - is_stale = changed - else: - is_stale = False - visiting_keys = visiting_keys - {upstream_key} - return is_stale, found_names, visiting_keys, {**stale_cache, upstream_key: is_stale} - - upstream_key: SelectionStalenessNodeKey - for upstream_key in graph.upstream_deps.get(model_key, ()): - _, names, visiting, stale_by_key = visit( - upstream_key=upstream_key, - found_names=names, - visiting_keys=visiting, - stale_cache=stale_by_key, - ) - return tuple(sorted(names)) - - -def _run_parent_changed( - *, graph: SelectionStalenessGraph, parent_key: SelectionStalenessNodeKey -) -> bool: - if parent_key.resource_type == CompiledResourceType.MODEL.value: - return ( - parent_key.name in graph.run_model_names - and parent_key.name in graph.changed_model_names - ) - if parent_key.resource_type == CompiledResourceType.SEED.value: - return ( - parent_key.name in graph.run_seed_names and parent_key.name in graph.changed_seed_names - ) - if parent_key.resource_type == CompiledResourceType.SOURCE.value: - return ( - parent_key.name in graph.run_source_names - and parent_key.name in graph.changed_source_names - ) - return False diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py deleted file mode 100644 index 74d5c886b7..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py +++ /dev/null @@ -1,229 +0,0 @@ -"""Selection-aware stale warning helpers for direct planning.""" - -from __future__ import annotations - -from sqlbuild._native import detect_model_changes -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( - classify_selection_staleness_warnings, - format_stale_upstream_warning_message, -) -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DirectModelVersionIdentities, - PlannerChangeResults, - PlannerScope, - PlanWarning, - SelectionStalenessGraph, - SelectionStalenessNodeKey, - SelectionStalenessWarning, - WarehouseSnapshot, -) -from sqlbuild.compiler.planner.types import ChangeKind, WarningSeverity -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult - -_OWN_CHANGE_KINDS: frozenset[ChangeKind] = frozenset( - { - ChangeKind.FIRST_RUN, - ChangeKind.QUERY_CHANGED, - ChangeKind.FUNCTION_CHANGED, - ChangeKind.CONFIG_CHANGED, - ChangeKind.SCHEMA_CHANGED, - } -) - - -def build_stale_out_of_selection_warnings( - *, - original_scope: PlannerScope, - execution_scope: PlannerScope, - changes: PlannerChangeResults, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - source_freshness: DirectSourceFreshnessPlanningResult | None, - include_sources: bool = True, - model_changes_complete: bool = False, -) -> tuple[PlanWarning, ...]: - """Warn for selected models stale through changed upstreams outside the run set.""" - - run_model_names: frozenset[str] = frozenset( - key.name - for key in execution_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - ) - run_seed_names: frozenset[str] = frozenset( - key.name - for key in execution_scope.selected_keys - if key.resource_type == CompiledResourceType.SEED - ) - run_source_names: frozenset[str] = frozenset( - key.name - for key in execution_scope.selected_keys - if key.resource_type == CompiledResourceType.SOURCE - ) - neutral_graph: SelectionStalenessGraph = SelectionStalenessGraph( - upstream_deps=_neutral_upstream_deps(original_scope.upstream_deps), - selected_model_names=frozenset( - key.name - for key in original_scope.selected_keys - if key.resource_type == CompiledResourceType.MODEL - ), - run_model_names=run_model_names, - run_seed_names=run_seed_names, - run_source_names=run_source_names, - changed_model_names=_changed_model_names( - original_scope=original_scope, - changes=changes, - snapshot=snapshot, - version_identities=version_identities, - model_changes_complete=model_changes_complete, - ), - changed_seed_names=_changed_seed_names( - snapshot=snapshot, - version_identities=version_identities, - ), - changed_source_names=( - _changed_source_names(source_freshness) if include_sources else frozenset() - ), - ) - warnings: list[PlanWarning] = [] - warning: SelectionStalenessWarning - for warning in classify_selection_staleness_warnings(neutral_graph): - warnings.append( - PlanWarning( - model_name=warning.model_name, - severity=WarningSeverity.WARNING, - message=format_stale_upstream_warning_message( - model_label="selected model", - model_name=warning.model_name, - trigger_label="upstream(s)", - trigger_names=warning.trigger_names, - ), - ) - ) - return tuple(warnings) - - -def _neutral_upstream_deps( - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]]: - neutral_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]] = {} - for key, upstream_keys in upstream_deps.items(): - neutral_upstreams: list[SelectionStalenessNodeKey] = [] - for upstream_key in upstream_keys: - neutral_upstreams.append(_neutral_key(upstream_key)) - neutral_deps[_neutral_key(key)] = tuple(neutral_upstreams) - return neutral_deps - - -def _neutral_key(key: CompiledObjectKey) -> SelectionStalenessNodeKey: - return SelectionStalenessNodeKey( - resource_type=_neutral_resource_type(key.resource_type), - name=key.name, - ) - - -def _neutral_resource_type(resource_type: str) -> str: - if resource_type == CompiledResourceType.MODEL: - return "model" - if resource_type == CompiledResourceType.SEED: - return "seed" - if resource_type == CompiledResourceType.SOURCE: - return "source" - return str(resource_type) - - -def _changed_model_names( - *, - original_scope: PlannerScope, - changes: PlannerChangeResults, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - model_changes_complete: bool, -) -> frozenset[str]: - changed: set[str] = { - model_name - for model_name, change in changes.models.items() - if change.change_kind in _OWN_CHANGE_KINDS - } - changed.update( - model_name - for model_name in original_scope.models_by_name - if _model_own_identity_changed( - model_name=model_name, - original_scope=original_scope, - changes=changes, - snapshot=snapshot, - version_identities=version_identities, - model_changes_complete=model_changes_complete, - ) - ) - return frozenset(changed) - - -def _changed_seed_names( - *, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, -) -> frozenset[str]: - return frozenset( - seed_name - for seed_name in version_identities.seed_version_hashes - if _seed_identity_changed( - seed_name=seed_name, - snapshot=snapshot, - version_identities=version_identities, - ) - ) - - -def _changed_source_names( - source_freshness: DirectSourceFreshnessPlanningResult | None, -) -> frozenset[str]: - if source_freshness is None: - return frozenset() - return frozenset(identity.source_name for identity in source_freshness.changed_identities) - - -def _model_own_identity_changed( - *, - model_name: str, - original_scope: PlannerScope, - changes: PlannerChangeResults, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, - model_changes_complete: bool, -) -> bool: - selected_change: ChangeDetectionResult | None = changes.models.get(model_name) - if selected_change is not None and ( - model_changes_complete or selected_change.change_kind in _OWN_CHANGE_KINDS - ): - return selected_change.change_kind in _OWN_CHANGE_KINDS - model: CompiledModelView | None = original_scope.models_by_name.get(model_name) - if model is None: - return False - change: ChangeDetectionResult = detect_model_changes( - model=model, - snapshot=snapshot, - sql_analysis_enabled=False, - query_change_tracking=True, - full_refresh=False, - expected_version_hash=version_identities.model_version_hashes.get(model_name), - expected_metadata_json=version_identities.model_metadata_jsons.get(model_name), - ) - return change.change_kind in _OWN_CHANGE_KINDS - - -def _seed_identity_changed( - *, - seed_name: str, - snapshot: WarehouseSnapshot, - version_identities: DirectModelVersionIdentities, -) -> bool: - expected_hash: str | None = version_identities.seed_version_hashes.get(seed_name) - if expected_hash is None: - return False - fingerprint: Fingerprint | None = snapshot.fingerprints.seeds.get(seed_name) - return fingerprint is None or fingerprint.version_hash != expected_hash diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index 17d3d343e8..d48996d26e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -4,7 +4,13 @@ from typing import Any -from sqlbuild._native import get_materialization_type +from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, + build_downstream_deps, + build_execution_upstream_deps, + get_materialization_type, + topologically_order_keys, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -16,15 +22,7 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, - topologically_order_keys, -) from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, build_model_locations, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index 385e1dcffb..9b7e1d46ad 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -7,8 +7,11 @@ from dataclasses import replace from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, + build_execution_upstream_deps, compiled_model_record, render_source_relation, + topologically_order_keys, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo @@ -35,13 +38,6 @@ from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, ) -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_execution_upstream_deps, - topologically_order_keys, -) -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) from sqlbuild.compiler.planner._helpers.output.plan_entry import extract_seed_columns, plan_model from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index ccff286819..ddeaf62240 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -10,6 +10,7 @@ from typing import Any from sqlbuild._native import ( + build_execution_upstream_deps, classify_cursor_sql_type, get_config_cursor_bound, get_config_str, @@ -62,9 +63,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_execution_upstream_deps, -) from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_upstream_intermediate_source_map, ) diff --git a/src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py b/src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py deleted file mode 100644 index 7d2f6e1fcf..0000000000 --- a/src/sqlbuild/compiler/planner/classes/graph_identity_resolver.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Memoized graph identity resolution over one dependency graph.""" - -from __future__ import annotations - -from collections.abc import Callable, Mapping - -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - -type _Resolve = Callable[[GraphNodeKey], str | None] - - -class GraphIdentityResolver: - """Resolve expected or selected-write identity hashes, each node once, in linear time.""" - - def __init__( - self, - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - compose_identity: GraphIdentityComposer, - base_hashes: Mapping[GraphNodeKey, str | None], - selected_keys: frozenset[GraphNodeKey] | None, - ) -> None: - self._nodes: Mapping[GraphNodeKey, GraphIdentityNode] = nodes - self._compose_identity: GraphIdentityComposer = compose_identity - self._selected_keys: frozenset[GraphNodeKey] | None = selected_keys - self._hashes: dict[GraphNodeKey, str | None] = dict(base_hashes) - self._resolved_selected: dict[GraphNodeKey, str] = {} - self._visiting: set[GraphNodeKey] = set() - - @property - def hashes(self) -> dict[GraphNodeKey, str | None]: - """Return every hash recorded so far, in first-recorded order.""" - - return self._hashes - - def resolve_expected(self, key: GraphNodeKey) -> str | None: - """Compose a node's expected hash from its upstream expected hashes.""" - - if key in self._hashes: - return self._hashes[key] - node: GraphIdentityNode | None = self._nodes.get(key) - if node is None or node.local_hash is None: - self._hashes[key] = None - return None - if key in self._visiting: - self._hashes[key] = node.local_hash - return node.local_hash - composed: str = self._compose( - key=key, local_hash=node.local_hash, node=node, resolve=self.resolve_expected - ) - self._hashes[key] = composed - return composed - - def resolve_write(self, key: GraphNodeKey) -> str | None: - """Recompute a selected node's hash from upstream write hashes; others keep theirs.""" - - if self._selected_keys is None or key not in self._selected_keys: - return self._hashes.get(key) - if key in self._resolved_selected: - return self._resolved_selected[key] - node: GraphIdentityNode | None = self._nodes.get(key) - if node is None or node.local_hash is None: - return self._hashes.get(key) - if key in self._visiting: - return node.local_hash - composed: str = self._compose( - key=key, local_hash=node.local_hash, node=node, resolve=self.resolve_write - ) - self._hashes[key] = composed - self._resolved_selected[key] = composed - return composed - - def _compose( - self, *, key: GraphNodeKey, local_hash: str, node: GraphIdentityNode, resolve: _Resolve - ) -> str: - self._visiting.add(key) - upstream_hashes: list[tuple[GraphNodeKey, str]] = [] - upstream_key: GraphNodeKey - for upstream_key in node.upstream_keys: - upstream_hash: str | None = resolve(upstream_key) - if upstream_hash is not None: - upstream_hashes.append((upstream_key, upstream_hash)) - self._visiting.discard(key) - return self._compose_identity(local_hash=local_hash, upstream_hashes=tuple(upstream_hashes)) diff --git a/src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py b/src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py deleted file mode 100644 index 78400f2196..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/_selection_staleness.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public selection-aware staleness classification entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( - classify_selection_staleness_warnings as _classify_selection_staleness_warnings, -) -from sqlbuild.compiler.planner.models import SelectionStalenessGraph, SelectionStalenessWarning - - -def classify_selection_staleness_warnings( - graph: SelectionStalenessGraph, -) -> tuple[SelectionStalenessWarning, ...]: - """Return stale warnings for selected models with changed upstreams outside the run set.""" - - return _classify_selection_staleness_warnings(graph) diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index 549d9a0158..224227c80b 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -2,15 +2,16 @@ from __future__ import annotations -from sqlbuild._native import get_config_str, get_materialization_type +from sqlbuild._native import ( + build_downstream_deps, + build_execution_upstream_deps, + get_config_str, + get_materialization_type, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import resolve_model_full_refresh from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import ( diff --git a/src/sqlbuild/compiler/planner/main/identity/__init__.py b/src/sqlbuild/compiler/planner/main/identity/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/main/identity/_graph_identity.py b/src/sqlbuild/compiler/planner/main/identity/_graph_identity.py deleted file mode 100644 index a60cf1a941..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_graph_identity.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public entrypoints for neutral graph identity resolution.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner._helpers.identity.graph import ( - build_expected_graph_identity_hashes as _build_expected_graph_identity_hashes, -) -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - - -def build_expected_graph_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str | None]: - """Build expected identity hashes for a neutral dependency graph.""" - - return _build_expected_graph_identity_hashes( - nodes=nodes, - execution_order=execution_order, - compose_identity=compose_identity, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py b/src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py deleted file mode 100644 index 775e5c0ae7..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_graph_write_identity.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public entrypoint for neutral graph write identity resolution.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner._helpers.identity.graph import ( - build_graph_write_identity_hashes as _build_graph_write_identity_hashes, -) -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphIdentityComposer - - -def build_graph_write_identity_hashes( - *, - nodes: Mapping[GraphNodeKey, GraphIdentityNode], - execution_order: tuple[GraphNodeKey, ...], - selected_keys: frozenset[GraphNodeKey], - base_identity_hashes: Mapping[GraphNodeKey, str], - compose_identity: GraphIdentityComposer, -) -> dict[GraphNodeKey, str]: - """Build write identity hashes from caller-supplied available upstream hashes.""" - - return _build_graph_write_identity_hashes( - nodes=nodes, - execution_order=execution_order, - selected_keys=selected_keys, - base_identity_hashes=base_identity_hashes, - compose_identity=compose_identity, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py b/src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py deleted file mode 100644 index 9daac10832..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_version_identity_config.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Build model config payloads that participate in version identity.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY - - -def build_version_identity_config(config_values: dict[str, object]) -> dict[str, object]: - """Return config fields that affect produced model version identity.""" - - identity: dict[str, object] = {} - if CURSOR_INPUTS_CONFIG_KEY in config_values: - identity[CURSOR_INPUTS_CONFIG_KEY] = config_values[CURSOR_INPUTS_CONFIG_KEY] - version_identity_config_keys: tuple[str, ...] = ( - "append_cursor_inclusive", - "batch_size", - "check_columns", - "cursor", - "cursor_grain", - "cursor_start", - "cursor_end", - "cursor_start_max_ahead", - "cursor_start_max_action", - "cursor_future_max_distance", - "cursor_future_action", - "cursor_type", - "historical_input", - "incremental_mode", - "microbatch_strategy", - "cursor_watermark_mode", - "max_microbatches", - "microbatch_limit", - "incremental_strategy", - "merge_exclude_columns", - "full_refresh", - "initial_valid_from", - "invalidate_hard_deletes", - "lookback", - "materialized", - "observed_at", - "on_schema_change", - "snapshot_full_refresh", - "snapshot_schema_change", - "snapshot_strategy", - "unique_key", - "updated_at", - "valid_from_column", - "valid_to_column", - ) - identity.update( - {key: config_values[key] for key in version_identity_config_keys if key in config_values} - ) - return identity diff --git a/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py b/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py deleted file mode 100644 index d5628746d8..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/_version_identity_metadata.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Build non-query metadata that participates in model version identity.""" - -from __future__ import annotations - -import json - -from sqlbuild.compiler.planner.constants import LOCAL_FUNCTION_HASHES_METADATA_KEY -from sqlbuild.compiler.planner.main.identity._version_identity_config import ( - build_version_identity_config, -) - - -def build_version_identity_metadata_json( - *, - model_name: str, - config_values: dict[str, object], - local_function_hashes: dict[str, str] | None = None, - execution_signature: dict[str, object] | None = None, -) -> str: - """Return deterministic metadata JSON for model version identity.""" - - return json.dumps( - { - "config": build_version_identity_config(config_values), - "execution_signature": execution_signature or {}, - LOCAL_FUNCTION_HASHES_METADATA_KEY: local_function_hashes or {}, - "model_name": model_name, - }, - sort_keys=True, - separators=(",", ":"), - default=str, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py deleted file mode 100644 index 0a17680710..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Build function hashes that participate in model version identity.""" - -from __future__ import annotations - -import hashlib -import json - -from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash - - -def build_function_local_hashes( - *, - functions: tuple[CompiledFunctionView, ...], - dialect: str | None, -) -> dict[str, str]: - """Derive local-only semantic hashes for functions; SQL bodies ignore layout and comments.""" - - hashes: dict[str, str] = {} - for function in functions: - arguments: list[tuple[str, str]] = [] - for argument in function.arguments: - arguments.append((argument.name, argument.type)) - return_columns: list[tuple[str, str]] = [] - for column in function.return_columns: - return_columns.append((column.name, column.type)) - hashes[function.name] = _stable_hash( - json.dumps( - { - "arguments": arguments, - "returns": function.returns, - "return_columns": return_columns, - "body_sql": function_definition_hash( - function_name=function.name, - fingerprint_sql=function.body_sql, - language=function.language.value, - dialect=dialect, - ), - "language": function.language.value, - "runtime_version": function.runtime_version, - "entry_point": function.entry_point, - "packages": function.packages, - }, - sort_keys=True, - default=str, - ) - ) - return hashes - - -def _stable_hash(value: str) -> str: - return hashlib.sha256(value.encode("utf-8")).hexdigest() diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py deleted file mode 100644 index 0cc9e3b64b..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Build model metadata that participates in version identity.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import serialize_hook_entries -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.discovery.constants import SQL_HOOK_IDENTITY_FIELDS -from sqlbuild.compiler.planner._helpers.identity.model_metadata import contract_output_signature -from sqlbuild.compiler.planner.constants import ( - MODEL_CUSTOM_CONFIG_KEY, - MODEL_PLACEHOLDERS_CONFIG_KEY, - MODEL_POST_HOOKS_CONFIG_KEY, - MODEL_PRE_HOOKS_CONFIG_KEY, -) -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) - - -def build_model_version_identity_metadata_json( - *, - model: CompiledModelView, - function_local_hashes: dict[str, str] | None = None, - hook_version_hashes: dict[str, str] | None = None, -) -> str: - """Build deterministic non-query model fingerprint metadata JSON.""" - - function_hashes: dict[str, str] = function_local_hashes or {} - local_function_hashes: dict[str, str] = {} - upstream_key: Any - for upstream_key in model.deps: - if upstream_key.resource_type not in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - continue - upstream_hash: str | None = function_hashes.get(upstream_key.name) - if upstream_hash is not None: - local_function_hashes[upstream_key.name] = upstream_hash - return build_version_identity_metadata_json( - model_name=model.name, - config_values=model.config.values, - local_function_hashes=local_function_hashes, - execution_signature=_model_execution_signature( - model=model, - hook_version_hashes=hook_version_hashes or {}, - ), - ) - - -def _model_execution_signature( - *, model: CompiledModelView, hook_version_hashes: dict[str, str] -) -> dict[str, object]: - signature: dict[str, object] = {} - contract_signature: dict[str, object] | None = contract_output_signature(model=model) - if contract_signature is not None: - signature["contract"] = contract_signature - if MODEL_CUSTOM_CONFIG_KEY in model.config.values: - signature["custom_config"] = model.config.values[MODEL_CUSTOM_CONFIG_KEY] - if MODEL_PLACEHOLDERS_CONFIG_KEY in model.config.values: - signature["custom_placeholders"] = model.config.values[MODEL_PLACEHOLDERS_CONFIG_KEY] - if MODEL_PRE_HOOKS_CONFIG_KEY in model.config.values: - signature["pre_hooks"] = _hook_execution_signature( - value=model.config.values[MODEL_PRE_HOOKS_CONFIG_KEY], - hook_version_hashes=hook_version_hashes, - ) - if MODEL_POST_HOOKS_CONFIG_KEY in model.config.values: - signature["post_hooks"] = _hook_execution_signature( - value=model.config.values[MODEL_POST_HOOKS_CONFIG_KEY], - hook_version_hashes=hook_version_hashes, - ) - return signature - - -def _hook_execution_signature( - *, value: object, hook_version_hashes: dict[str, str] -) -> list[dict[str, object]]: - return serialize_hook_entries( - value=value, - sql_fields=SQL_HOOK_IDENTITY_FIELDS, - python_hook_fields={ - hook_name: {"version_hash": version_hash} - for hook_name, version_hash in hook_version_hashes.items() - }, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py b/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py deleted file mode 100644 index ce4bd03d3d..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Compiler-internal downstream model-name closure entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.model_closure import ( - build_downstream_model_name_closure, -) - - -def build_downstream_model_names( - *, - start_keys: tuple[CompiledObjectKey, ...], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - return build_downstream_model_name_closure( - start_keys=start_keys, - downstream_deps=downstream_deps, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_upstream.py b/src/sqlbuild/compiler/planner/main/selection/_upstream.py index a03105df3d..d62e6c8789 100644 --- a/src/sqlbuild/compiler/planner/main/selection/_upstream.py +++ b/src/sqlbuild/compiler/planner/main/selection/_upstream.py @@ -2,8 +2,8 @@ from __future__ import annotations +from sqlbuild._native import transitive_closure from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.core import expand_upstream def expand_project_upstream_keys( @@ -13,4 +13,4 @@ def expand_project_upstream_keys( ) -> frozenset[CompiledObjectKey]: """Return the upstream key closure for one compiled graph key.""" - return frozenset(expand_upstream(key=key, upstream=upstream_deps)) + return frozenset(transitive_closure(start=key, edges=upstream_deps)) diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py index da64eb1d77..9368d38745 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py @@ -2,11 +2,9 @@ from __future__ import annotations +from sqlbuild._native import build_downstream_model_name_closure from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.main.selection._model_downstream_closure import ( - build_downstream_model_names, -) from sqlbuild.compiler.planner.models import PlannerScope from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, @@ -65,7 +63,7 @@ def _downstream_model_names(*, source_name: str, scope: PlannerScope) -> frozens resource_type=CompiledResourceType.SOURCE, name=source_name, ) - return build_downstream_model_names( + return build_downstream_model_name_closure( start_keys=(source_key,), downstream_deps=scope.downstream_deps, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 79c0b1d7c9..11294e2004 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -57,7 +57,7 @@ class BuildExecutionPlanTestCase: class PlannerChangeDetectionWorkTestCase: description: str selection_diagnostics: bool - expected_model_calls: dict[str, int] + expected_stale_detection_calls: dict[str, int] expected_warning_count: int query_change_tracking: bool = True expected_identity_builds: int = 1 diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 432303fc0f..bc10eaabad 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -13,6 +13,7 @@ from typing import Any, cast from sqlbuild._native import ( + build_compiled_function_fingerprint_sql, compiled_model_record, compute_query_hash, discover_project_inputs, @@ -51,9 +52,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, -) from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan from sqlbuild.compiler.planner.models import ( DeferralInputs, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index fe34b95e93..8212321405 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -15,7 +15,6 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.identity import model_metadata from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -64,20 +63,20 @@ PlannerChangeDetectionWorkTestCase( description="selection diagnostics disabled", selection_diagnostics=False, - expected_model_calls={"orders": 1}, + expected_stale_detection_calls={}, expected_warning_count=0, ), PlannerChangeDetectionWorkTestCase( description="selection diagnostics enabled", selection_diagnostics=True, - expected_model_calls={"stg_orders": 1, "orders": 1}, + expected_stale_detection_calls={"stg_orders": 1}, expected_warning_count=0, expected_identity_builds=2, ), PlannerChangeDetectionWorkTestCase( description="selection diagnostics preserve independent query tracking", selection_diagnostics=True, - expected_model_calls={"stg_orders": 1, "orders": 2}, + expected_stale_detection_calls={"stg_orders": 1, "orders": 1}, expected_warning_count=0, query_change_tracking=False, expected_identity_builds=2, @@ -119,8 +118,8 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d query_change_tracking=test_case.query_change_tracking, ), ) - model_detector: Mock = Mock(wraps=model_metadata.declared_columns_hash) - monkeypatch.setattr(model_metadata, "declared_columns_hash", model_detector) + stale_detector: Mock = Mock(wraps=identity_planning.detect_model_changes_in_scope) + monkeypatch.setattr(identity_planning, "detect_model_changes_in_scope", stale_detector) identity_builder: Mock = Mock(wraps=identity_planning.build_direct_model_version_identities) monkeypatch.setattr( identity_planning, @@ -136,10 +135,10 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d selection_diagnostics=test_case.selection_diagnostics, ) - model_calls: Counter[str] = Counter( - call.kwargs["model"].name for call in model_detector.call_args_list - ) - assert dict(model_calls) == test_case.expected_model_calls + stale_calls: Counter[str] = Counter() + for call in stale_detector.call_args_list: + stale_calls.update(key.name for key in call.kwargs["scope"].selected_keys) + assert dict(stale_calls) == test_case.expected_stale_detection_calls assert identity_builder.call_count == test_case.expected_identity_builds assert len(plan.warnings) == test_case.expected_warning_count diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 2a730a73fc..f67198973a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_execution_upstream_deps, build_manifest, build_native_declaration_contexts, build_scope_lookup, @@ -62,7 +63,6 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.planner._helpers.graph.core import build_execution_upstream_deps from sqlbuild.compiler.planner._helpers.resolve.refs import resolve_ref_references from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index fd96381c3d..761be7aae8 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_model_version_identity_metadata_json, compute_query_hash, discover_project_inputs, ) @@ -22,9 +23,6 @@ ) from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CompileDeclarationsErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 0cba374c8f..f167adfc1f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( assemble_project, build_compile_inputs, + build_model_version_identity_metadata_json, discover_project_inputs, evaluate_model_contracts, ) @@ -24,9 +25,6 @@ CompileInputsView, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DynamicModelSchemaTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 9348e7321a..93a797f7ac 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -16,11 +16,10 @@ from sqlbuild.compiler.planner.models import ( CursorBounds, EmptyCursorInputDecision, - GraphIdentityNode, - GraphNodeKey, MissingUpstream, ParsedSelector, PathSelector, + PlanWarning, ScenarioArtifactIdentity, ScenarioGraphPlan, ScenarioRelationMap, @@ -66,7 +65,7 @@ class CompleteFixturePlanningTestCase: @dataclass(frozen=True) class CompleteModelChangesWorkTestCase: description: str - expected_detector_calls: int + expected_warnings: tuple[PlanWarning, ...] @dataclass(frozen=True) @@ -193,18 +192,6 @@ class SelectionStalenessWarningTestCase: expected_warning_fragments: tuple[str, ...] -@dataclass(frozen=True) -class StaleWarningMessageTestCase: - description: str - model_label: str - model_name: str - trigger_label: str - trigger_names: tuple[str, ...] - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] - expected_bullet_count: int - - @dataclass(frozen=True) class SelectionStalenessGraphWarningTestCase: description: str @@ -218,32 +205,6 @@ class SelectionStalenessGraphWarningTestCase: unexpected_warning_fragments: tuple[str, ...] = () -@dataclass(frozen=True) -class GraphIdentityExpectedHashesTestCase: - description: str - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - expected_hashes: dict[GraphNodeKey, str | None] - - -@dataclass(frozen=True) -class GraphIdentityWriteHashesTestCase: - description: str - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - selected_keys: frozenset[GraphNodeKey] - base_identity_hashes: dict[GraphNodeKey, str] - expected_hashes: dict[GraphNodeKey, str] - - -@dataclass(frozen=True) -class GraphIdentityWritePerfTestCase: - description: str - layer_count: int - expected_max_seconds: float - layer_width: int = 2 - - @dataclass(frozen=True) class ModelClosureTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index dcfe977822..cece72c0bf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import cast -from sqlbuild._native import compute_query_hash +from sqlbuild._native import build_version_identity_metadata_json, compute_query_hash from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.compiler.compile.models import ( CompiledFunction, @@ -26,9 +26,6 @@ FunctionLanguage, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import PlannerScope, WarehouseFingerprints, WarehouseSnapshot from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index c3c95b77f6..c4099e9a61 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( build_model_warnings, + build_version_identity_metadata_json, compute_query_hash, detect_changes, detect_model_changes, @@ -18,9 +19,6 @@ from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py index fa2680d0f7..c4efc9f391 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py @@ -2,15 +2,15 @@ import pytest -from sqlbuild._native import project_lineage_views +from sqlbuild._native import ( + build_downstream_deps, + build_execution_upstream_deps, + project_lineage_views, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_test_project, model_key, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 4f60c4a0a5..18f2365be1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib from collections.abc import Callable, Iterator from dataclasses import replace from datetime import UTC, datetime @@ -10,11 +9,14 @@ from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Any, ClassVar +from unittest.mock import Mock import sqlbuild._native as _native from sqlbuild._native import ( + build_downstream_deps, compiled_model_record, compute_query_hash, + detect_changes, project_with, scan_sql_reference_calls, ) @@ -64,13 +66,11 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.scenario.artifacts import build_scenario_relation_map from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, - GraphIdentityNode, - GraphNodeKey, + PlannerChangeResults, PlanOutput, ScenarioArtifactIdentity, ScenarioRelationMap, @@ -78,7 +78,7 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - GraphResourceKind, + PlanReason, ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -159,77 +159,6 @@ def close(self, connection: object) -> None: del connection -def compose_readable_identity( - *, local_hash: str, upstream_hashes: tuple[tuple[GraphNodeKey, str], ...] -) -> str: - rendered: str = ",".join( - f"{key.node_type}:{key.node_name}={upstream_hash}" for key, upstream_hash in upstream_hashes - ) - return (f"{local_hash}|{rendered}", local_hash)[not upstream_hashes] - - -def compose_hashed_identity( - *, local_hash: str, upstream_hashes: tuple[tuple[GraphNodeKey, str], ...] -) -> str: - payload: str = ( - local_hash + "|" + ",".join(upstream_hash for _, upstream_hash in upstream_hashes) - ) - return (hashlib.sha256(payload.encode()).hexdigest(), local_hash)[not upstream_hashes] - - -def build_diamond_ladder_identity_nodes( - *, layer_count: int -) -> tuple[dict[GraphNodeKey, GraphIdentityNode], tuple[GraphNodeKey, ...]]: - """Build a dense ladder where each layer depends on both nodes of the prior layer.""" - - def node_key(name: str) -> GraphNodeKey: - return GraphNodeKey(node_type="model", node_name=name) - - nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - order: list[GraphNodeKey] = [] - previous: tuple[GraphNodeKey, ...] = () - layer_index: int - for layer_index in range(layer_count): - current: list[GraphNodeKey] = [] - suffix: str - for suffix in ("a", "b"): - key: GraphNodeKey = node_key(f"L{layer_index}_{suffix}") - nodes[key] = GraphIdentityNode( - key=key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=previous, - local_hash=f"local_{key.node_name}", - ) - order.append(key) - current.append(key) - previous = tuple(current) - return nodes, tuple(order) - - -def build_wide_layered_identity_nodes( - *, layer_count: int, layer_width: int -) -> tuple[dict[GraphNodeKey, GraphIdentityNode], tuple[GraphNodeKey, ...]]: - """Build a wide project graph where each node reads two nodes of the prior layer.""" - - nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - previous: tuple[GraphNodeKey, ...] = () - layer: int - for layer in range(layer_count): - current: list[GraphNodeKey] = [] - column: int - for column in range(layer_width): - key: GraphNodeKey = GraphNodeKey(node_type="model", node_name=f"L{layer}_{column}") - nodes[key] = GraphIdentityNode( - key=key, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=previous[column : column + 2], - local_hash=f"local_{key.node_name}", - ) - current.append(key) - previous = tuple(current) - return nodes, tuple(nodes) - - def _append_relation_info(listed: list[RelationInfo], relation: RelationInfo) -> None: listed.append(relation) @@ -1509,3 +1438,32 @@ def _assertion_ref_targets(assertion_sql: tuple[str, ...]) -> tuple[str, ...]: lambda reference: reference.ref_kind == SqlReferenceKind.REF, references ) return tuple(dict.fromkeys(reference.ref_name for reference in refs)) + + +def build_unreadable_model() -> CompiledModelView: + """Build a model view whose every attribute read fails, so change detection cannot run.""" + + model: CompiledModelView = Mock(spec=()) + return model + + +def detect_function_reason( + *, function: CompiledFunctionView, fingerprints: dict[str, Fingerprint] +) -> PlanReason: + """Detect one selected function's change reason through planner change detection.""" + + project: Mock = Mock( + functions=(function,), + settings=SimpleNamespace(query_change_tracking=True), + sql_analysis_dialect=None, + hook_functions=(), + ) + scope: Mock = Mock( + selected_keys=frozenset({function.key}), execution_order=(), models_by_name={} + ) + snapshot: Mock = Mock(fingerprints=SimpleNamespace(functions=fingerprints), column_dialect=None) + changes: PlannerChangeResults = detect_changes( + project=project, scope=scope, snapshot=snapshot, full_refresh=False + ) + reason: PlanReason = changes.functions[function.name].reason + return reason diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py index 5264136d02..8eba85f9f6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_definition_hashes.py @@ -2,13 +2,13 @@ from __future__ import annotations +from dataclasses import replace + import pytest -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - function_definition_hash, - model_definition_hash, -) +from sqlbuild._native import build_function_local_hashes, model_definition_hash from sqlbuild.compiler.planner.exceptions import PlannerInputError +from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( DefinitionHashFailureTestCase, ) @@ -45,10 +45,10 @@ def test_given_untokenizable_sql_function_when_hashing_then_planning_error_names test_case: DefinitionHashFailureTestCase, ) -> None: with pytest.raises(PlannerInputError, match=test_case.expected_message) as raised: - _ = function_definition_hash( - function_name="is_open", - fingerprint_sql=_UNTOKENIZABLE_SQL, - language="sql", + _ = build_function_local_hashes( + functions=( + replace(build_compiled_function(body_sql=_UNTOKENIZABLE_SQL), name="is_open"), + ), dialect="duckdb", ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index 208b1a2173..0e9b1029fe 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.identity.honest import merge_recomputed_model_changes +from sqlbuild._native import merge_recomputed_model_changes from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py index 88caa414e8..fc42a40697 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py @@ -2,11 +2,8 @@ import pytest +from sqlbuild._native import check_buildability, missing_upstream_message from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.planner._helpers.graph.buildability import ( - check_buildability, - missing_upstream_message, -) from sqlbuild.compiler.planner.models import MissingUpstream, WarehouseSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CheckBuildabilityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index 0a1d58915f..e46a99578a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -5,10 +5,6 @@ import pytest from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.planner._helpers.identity.functions import ( - build_compiled_function_fingerprint_sql, - detect_function_change, -) from sqlbuild.compiler.planner.types import PlanReason from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( DetectFunctionChangeTestCase, @@ -16,6 +12,7 @@ from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_compiled_function, build_fingerprint, + detect_function_reason, ) @@ -69,13 +66,8 @@ def test_given_function_fingerprint_when_detecting_change_then_returns_reason( body_sql=test_case.body_sql, target_schema=test_case.target_schema, ) - reason: PlanReason = detect_function_change( - function=function, - fingerprint_sql=build_compiled_function_fingerprint_sql(function), - fingerprint=test_case.existing_function_fingerprints.get(function.name), - query_change_tracking=True, - full_refresh=False, - dialect=None, + reason: PlanReason = detect_function_reason( + function=function, fingerprints=test_case.existing_function_fingerprints ) assert reason == test_case.expected_reason diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py index f68ec6b046..85a57d11bb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py @@ -2,15 +2,14 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.core import ( +from sqlbuild._native import ( build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, - expand_downstream, - expand_upstream, topologically_order_keys, + transitive_closure, ) +from sqlbuild.compiler.compile.models import CompiledObjectKey from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildDownstreamDepsTestCase, BuildUpstreamDepsTestCase, @@ -296,8 +295,8 @@ def test_given_audit_scope_deps_when_building_edge_origins_then_names_the_audit( def test_given_key_when_expanding_upstream_then_returns_expected_keys( test_case: ExpandUpstreamTestCase, ) -> None: - result: frozenset[CompiledObjectKey] = expand_upstream( - key=test_case.key, upstream=test_case.upstream + result: frozenset[CompiledObjectKey] = transitive_closure( + start=test_case.key, edges=test_case.upstream ) assert result == test_case.expected_keys @@ -331,8 +330,8 @@ def test_given_key_when_expanding_upstream_then_returns_expected_keys( def test_given_key_when_expanding_downstream_then_returns_expected_keys( test_case: ExpandDownstreamTestCase, ) -> None: - result: frozenset[CompiledObjectKey] = expand_downstream( - key=test_case.key, downstream=test_case.downstream + result: frozenset[CompiledObjectKey] = transitive_closure( + start=test_case.key, edges=test_case.downstream ) assert result == test_case.expected_keys diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py deleted file mode 100644 index e1d04af0ef..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph_identity.py +++ /dev/null @@ -1,259 +0,0 @@ -from __future__ import annotations - -import time - -import pytest - -from sqlbuild.compiler.planner.main.identity._graph_identity import ( - build_expected_graph_identity_hashes, -) -from sqlbuild.compiler.planner.main.identity._graph_write_identity import ( - build_graph_write_identity_hashes, -) -from sqlbuild.compiler.planner.models import GraphIdentityNode, GraphNodeKey -from sqlbuild.compiler.planner.types import GraphResourceKind -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - GraphIdentityExpectedHashesTestCase, - GraphIdentityWriteHashesTestCase, - GraphIdentityWritePerfTestCase, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( - build_diamond_ladder_identity_nodes, - build_wide_layered_identity_nodes, - compose_hashed_identity, - compose_readable_identity, -) - -MODEL_A: GraphNodeKey = GraphNodeKey(node_type="model", node_name="a") -MODEL_B: GraphNodeKey = GraphNodeKey(node_type="model", node_name="b") -MODEL_C: GraphNodeKey = GraphNodeKey(node_type="model", node_name="c") -MODEL_D: GraphNodeKey = GraphNodeKey(node_type="model", node_name="d") - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityExpectedHashesTestCase( - description=( - "uses dependency order and named upstream keys while preserving missing nodes" - ), - nodes={ - MODEL_A: GraphIdentityNode( - key=MODEL_A, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_B, MODEL_C), - local_hash="local_a", - ), - MODEL_B: GraphIdentityNode( - key=MODEL_B, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash="local_b", - ), - MODEL_C: GraphIdentityNode( - key=MODEL_C, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash=None, - ), - }, - execution_order=(MODEL_B, MODEL_C, MODEL_A), - expected_hashes={ - MODEL_A: "local_a|model:b=local_b", - MODEL_B: "local_b", - MODEL_C: None, - }, - ) - ], - ids=lambda case: case.description, -) -def test_given_identity_graph_when_building_expected_hashes_then_resolves_from_upstreams( - test_case: GraphIdentityExpectedHashesTestCase, -) -> None: - result: dict[GraphNodeKey, str | None] = build_expected_graph_identity_hashes( - nodes=test_case.nodes, - execution_order=test_case.execution_order, - compose_identity=compose_readable_identity, - ) - - assert result == test_case.expected_hashes - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWriteHashesTestCase( - description="recomputes selected child from caller supplied parent write hash", - nodes={ - MODEL_A: GraphIdentityNode( - key=MODEL_A, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_B,), - local_hash="local_a", - ), - MODEL_B: GraphIdentityNode( - key=MODEL_B, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash="local_b", - ), - }, - execution_order=(MODEL_B, MODEL_A), - selected_keys=frozenset({MODEL_A}), - base_identity_hashes={MODEL_B: "honest_b"}, - expected_hashes={ - MODEL_A: "local_a|model:b=honest_b", - MODEL_B: "honest_b", - }, - ), - GraphIdentityWriteHashesTestCase( - description="composes a fully selected diamond from the shared root once", - nodes={ - MODEL_A: GraphIdentityNode( - key=MODEL_A, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(), - local_hash="local_a", - ), - MODEL_B: GraphIdentityNode( - key=MODEL_B, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_A,), - local_hash="local_b", - ), - MODEL_C: GraphIdentityNode( - key=MODEL_C, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_A,), - local_hash="local_c", - ), - MODEL_D: GraphIdentityNode( - key=MODEL_D, - resource_kind=GraphResourceKind.MODEL, - upstream_keys=(MODEL_B, MODEL_C), - local_hash="local_d", - ), - }, - execution_order=(MODEL_A, MODEL_B, MODEL_C, MODEL_D), - selected_keys=frozenset({MODEL_A, MODEL_B, MODEL_C, MODEL_D}), - base_identity_hashes={}, - expected_hashes={ - MODEL_A: "local_a", - MODEL_B: "local_b|model:a=local_a", - MODEL_C: "local_c|model:a=local_a", - MODEL_D: "local_d|model:b=local_b|model:a=local_a,model:c=local_c|model:a=local_a", - }, - ), - ], - ids=lambda case: case.description, -) -def test_given_available_hashes_when_building_write_hashes_then_uses_caller_supplied_upstreams( - test_case: GraphIdentityWriteHashesTestCase, -) -> None: - result: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=test_case.nodes, - execution_order=test_case.execution_order, - selected_keys=test_case.selected_keys, - base_identity_hashes=test_case.base_identity_hashes, - compose_identity=compose_readable_identity, - ) - - assert result == test_case.expected_hashes - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWritePerfTestCase( - description="fully selected densely connected graph resolves without path blowup", - layer_count=400, - expected_max_seconds=5.0, - ) - ], - ids=lambda case: case.description, -) -def test_given_fully_selected_dense_graph_when_building_write_hashes_then_stays_linear( - test_case: GraphIdentityWritePerfTestCase, -) -> None: - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - nodes, execution_order = build_diamond_ladder_identity_nodes(layer_count=test_case.layer_count) - - start: float = time.monotonic() - result: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=nodes, - execution_order=execution_order, - selected_keys=frozenset(nodes.keys()), - base_identity_hashes={}, - compose_identity=compose_hashed_identity, - ) - elapsed: float = time.monotonic() - start - - assert len(result) == len(nodes) - assert elapsed < test_case.expected_max_seconds - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWritePerfTestCase( - description="twenty thousand node project graph resolves in linear time", - layer_count=40, - layer_width=500, - expected_max_seconds=1.5, - ) - ], - ids=lambda case: case.description, -) -def test_given_wide_project_graph_when_building_expected_hashes_then_stays_linear( - test_case: GraphIdentityWritePerfTestCase, -) -> None: - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - nodes, execution_order = build_wide_layered_identity_nodes( - layer_count=test_case.layer_count, layer_width=test_case.layer_width - ) - - start: float = time.monotonic() - result: dict[GraphNodeKey, str | None] = build_expected_graph_identity_hashes( - nodes=nodes, execution_order=execution_order, compose_identity=compose_hashed_identity - ) - elapsed: float = time.monotonic() - start - - assert len(result) == len(nodes) - assert elapsed < test_case.expected_max_seconds - - -@pytest.mark.parametrize( - "test_case", - [ - GraphIdentityWritePerfTestCase( - description="twenty thousand node project graph resolves in linear time", - layer_count=40, - layer_width=500, - expected_max_seconds=1.5, - ) - ], - ids=lambda case: case.description, -) -def test_given_wide_selected_project_graph_when_building_write_hashes_then_stays_linear( - test_case: GraphIdentityWritePerfTestCase, -) -> None: - nodes: dict[GraphNodeKey, GraphIdentityNode] - execution_order: tuple[GraphNodeKey, ...] - nodes, execution_order = build_wide_layered_identity_nodes( - layer_count=test_case.layer_count, layer_width=test_case.layer_width - ) - - start: float = time.monotonic() - result: dict[GraphNodeKey, str] = build_graph_write_identity_hashes( - nodes=nodes, - execution_order=execution_order, - selected_keys=frozenset(nodes.keys()), - base_identity_hashes={}, - compose_identity=compose_hashed_identity, - ) - elapsed: float = time.monotonic() - start - - assert len(result) == len(nodes) - assert elapsed < test_case.expected_max_seconds diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index cb71d34ebc..923a00170c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -2,11 +2,9 @@ import pytest +from sqlbuild._native import build_downstream_model_name_closure from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.model_closure import ( - build_downstream_model_name_closure, -) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase SOURCE_ORDERS: CompiledObjectKey = CompiledObjectKey( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py index 0be0633d36..3f34e9ed29 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_seed_identity.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.planner._helpers.identity.seed import build_seed_identity +from sqlbuild._native import build_seed_identity from sqlbuild.spec.contracts.models import SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( SeedIdentityCsvConfigTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index a6dbdd897b..b2b494bb98 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -2,11 +2,11 @@ from dataclasses import replace from datetime import UTC, datetime -from unittest.mock import Mock import pytest from sqlbuild._native import ( + build_stale_out_of_selection_warnings, compiled_model_record, ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject @@ -18,13 +18,6 @@ from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes -from sqlbuild.compiler.planner._helpers.pruning import selection_staleness -from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( - format_stale_upstream_warning_message, -) -from sqlbuild.compiler.planner._helpers.pruning.selection_staleness import ( - build_stale_out_of_selection_warnings, -) from sqlbuild.compiler.planner.models import ( ChangeDetectionResult, DirectModelVersionIdentities, @@ -47,10 +40,10 @@ PlannerStaleWarningScopeTestCase, SelectionStalenessGraphWarningTestCase, SelectionStalenessWarningTestCase, - StaleWarningMessageTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_selection_model, + build_unreadable_model, ) MODEL_KEY: CompiledObjectKey = CompiledObjectKey( @@ -64,20 +57,15 @@ [ CompleteModelChangesWorkTestCase( description="complete model changes", - expected_detector_calls=0, + expected_warnings=(), ) ], ids=lambda case: case.description, ) def test_given_complete_change_results_when_building_stale_warnings_then_models_are_not_redetected( test_case: CompleteModelChangesWorkTestCase, - monkeypatch: pytest.MonkeyPatch, ) -> None: - model: CompiledModelView = build_selection_model( - key=MODEL_KEY, - name=MODEL_KEY.name, - materialized="table", - ) + model: CompiledModelView = build_unreadable_model() scope: PlannerScope = PlannerScope( upstream_deps={MODEL_KEY: ()}, downstream_deps={}, @@ -86,11 +74,6 @@ def test_given_complete_change_results_when_building_stale_warnings_then_models_ selected_keys=frozenset({MODEL_KEY}), execution_order=(MODEL_KEY,), ) - model_detector: Mock = Mock( - side_effect=AssertionError("complete change results must not trigger model detection") - ) - monkeypatch.setattr(selection_staleness, "detect_model_changes", model_detector) - warnings: tuple[PlanWarning, ...] = build_stale_out_of_selection_warnings( original_scope=scope, execution_scope=scope, @@ -116,8 +99,7 @@ def test_given_complete_change_results_when_building_stale_warnings_then_models_ model_changes_complete=True, ) - assert warnings == () - assert model_detector.call_count == test_case.expected_detector_calls + assert warnings == test_case.expected_warnings SEED_KEY: CompiledObjectKey = CompiledObjectKey( @@ -561,60 +543,3 @@ def test_given_stale_upstream_graph_when_classifying_then_reports_expected_trigg unexpected_fragment: str for unexpected_fragment in test_case.unexpected_warning_fragments: assert unexpected_fragment not in warning_text - - -@pytest.mark.parametrize( - "test_case", - [ - StaleWarningMessageTestCase( - description="lists every trigger as a bullet when under the cap", - model_label="selected model", - model_name="leaf", - trigger_label="upstream(s)", - trigger_names=("a", "b", "c"), - expected_fragments=( - "selected model 'leaf' will build on 3 stale upstream(s) not selected for rebuild:", - " - a", - " - b", - " - c", - " rebuild the closure to refresh them: --select +leaf", - ), - unexpected_fragments=("more",), - expected_bullet_count=3, - ), - StaleWarningMessageTestCase( - description="caps the bullet list at five and summarizes the remainder", - model_label="selected dbt model", - model_name="leaf", - trigger_label="upstream(s)", - trigger_names=("a", "b", "c", "d", "e", "f", "g"), - expected_fragments=( - "selected dbt model 'leaf' will build on 7 stale upstream(s) not selected for rebuild:", - " - a", - " - e", - " +2 more", - " rebuild the closure to refresh them: --select +leaf", - ), - unexpected_fragments=(" - f", " - g"), - expected_bullet_count=5, - ), - ], - ids=lambda case: case.description, -) -def test_given_trigger_names_when_formatting_stale_warning_then_caps_and_structures_message( - test_case: StaleWarningMessageTestCase, -) -> None: - message: str = format_stale_upstream_warning_message( - model_label=test_case.model_label, - model_name=test_case.model_name, - trigger_label=test_case.trigger_label, - trigger_names=test_case.trigger_names, - ) - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in message - for fragment in test_case.unexpected_fragments: - assert fragment not in message - bullet_count: int = sum(line.strip().startswith("- ") for line in message.splitlines()) - assert bullet_count == test_case.expected_bullet_count diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index eb51f0cd49..467de551a9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -2,12 +2,11 @@ import pytest -from sqlbuild._native import project_lineage_views +from sqlbuild._native import build_downstream_deps, project_lineage_views from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.graph.selectors import ( parse_selector, resolve_selectors, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index d1fb0ad54b..2cc44d4bd6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -5,7 +5,11 @@ from pathlib import Path from typing import Any -from sqlbuild._native import compute_query_hash, write_fingerprint +from sqlbuild._native import ( + build_version_identity_metadata_json, + compute_query_hash, + write_fingerprint, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -21,9 +25,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan -from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( - build_version_identity_metadata_json, -) from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py index 53a0099164..8e10a52242 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py @@ -1,4 +1,6 @@ +from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path @dataclass(frozen=True) @@ -46,3 +48,10 @@ class HookIdentityPayloadTestCase: hooks: dict[str, object] expected_fragments: tuple[str, ...] forbidden_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PersistedIdentityTestCase: + description: str + build: Callable[[Path], object] + expected_identity: object diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index cd00e48e5d..be89a4876a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -1,7 +1,13 @@ +import json from dataclasses import replace from pathlib import Path +from types import SimpleNamespace from sqlbuild._native import ( + build_direct_model_version_identities, + build_downstream_deps, + build_model_version_identity_metadata_json, + build_version_identity_metadata_json, compiled_model_record, ) from sqlbuild.compiler.compile.models import ( @@ -10,20 +16,29 @@ CompiledObjectKey, CompiledRelationLocation, CompileModelConfig, + FunctionArgument, FunctionReturnColumn, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps -from sqlbuild.compiler.planner._helpers.identity.direct import ( - build_direct_model_version_identities, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledResourceType, + FunctionLanguage, ) from sqlbuild.compiler.planner.models import DirectModelVersionIdentities, PlannerScope +from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function +def build_version_identity_config(config_values: dict[str, object]) -> dict[str, object]: + """Return the config entries recorded in a model's version identity metadata.""" + + metadata: dict[str, object] = json.loads( + build_version_identity_metadata_json(model_name="orders", config_values=config_values) + ) + config: dict[str, object] = metadata["config"] + return config + + def build_table_function_graph_identities(*, base_query: str) -> DirectModelVersionIdentities: base_key: CompiledObjectKey = CompiledObjectKey( resource_type=CompiledResourceType.MODEL, @@ -111,3 +126,48 @@ def build_hooked_model_identity_json(*, hook_config: dict[str, object]) -> str: model=model, hook_version_hashes={"notify_orders": "notify-hash-v1"}, ) + + +def build_contract_model() -> SimpleNamespace: + """Build a model view with one enforced declared column.""" + + return SimpleNamespace( + config=SimpleNamespace(values={"contract": "enforced"}), + schema_entry=SimpleNamespace( + columns=(SchemaColumn(name="id", type="int", nullable=False),), + dynamic_columns=(), + type_enforcement=True, + ), + ) + + +def build_identity_function() -> SimpleNamespace: + """Build a SQL scalar function view with one argument.""" + + return SimpleNamespace( + name="add_one", + arguments=(FunctionArgument(name="x", type="int"),), + returns="int", + return_columns=(), + body_sql="x + 1", + language=FunctionLanguage.SQL, + runtime_version=None, + entry_point=None, + packages=(), + ) + + +def build_identity_seed(directory: Path) -> SimpleNamespace: + """Build a seed view over a two-row CSV file with Windows line endings.""" + + file_path: Path = directory / "regions.csv" + file_path.write_text("id,name\r\n1,café\r\n", encoding="utf-8") + return SimpleNamespace( + name="regions", + external=False, + seed_file=SimpleNamespace(file_path=file_path), + schema_entry=SimpleNamespace( + csv_settings=SeedCsvSettings(), + columns=(SchemaColumn(name="id", type="int"),), + ), + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py new file mode 100644 index 0000000000..dbc48e6433 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_persisted_identity_pins.py @@ -0,0 +1,83 @@ +"""Version identities are persisted in warehouse state; these values must never change.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import ( + build_function_local_hashes, + build_seed_identity, + build_version_identity_metadata_json, + declared_columns_hash, +) +from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( + PersistedIdentityTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner.main.identity.helpers import ( + build_contract_model, + build_identity_function, + build_identity_seed, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PersistedIdentityTestCase( + description="version identity metadata", + build=lambda _: build_version_identity_metadata_json( + model_name="orders", + config_values={ + "materialized": "table", + "batch_size": 10, + "lookback": 1.5, + "unique_key": ("id", "café"), + "cursor_inputs": {"b": None, "a": True}, + "ignored": 1, + }, + local_function_hashes={"f": "h"}, + ), + expected_identity=( + '{"config":{"batch_size":10,"cursor_inputs":{"a":true,"b":null},' + '"lookback":1.5,"materialized":"table","unique_key":["id","caf\\u00e9"]},' + '"execution_signature":{},"local_function_hashes":{"f":"h"},"model_name":"orders"}' + ), + ), + PersistedIdentityTestCase( + description="declared columns hash", + build=lambda _: declared_columns_hash(model=build_contract_model()), + expected_identity="fd43639fd4c7a585e25ad4bc134d59e890cb81f34f87c7083bfbb83a7c5a45c9", + ), + PersistedIdentityTestCase( + description="function local hash", + build=lambda _: build_function_local_hashes( + functions=(build_identity_function(),), dialect="duckdb" + ), + expected_identity={ + "add_one": "f0b98ef7f39ec80c918937418d5d1b13c01407d83846e9817ce4402748d74d97" + }, + ), + PersistedIdentityTestCase( + description="seed identity", + build=lambda directory: build_seed_identity(build_identity_seed(directory)), + expected_identity=( + "37b6c31dd600a9c5b2f811338382d211faa6b7278e6e552cec636fdabf243323", + '{"columns":[{"name":"id","nullable":null,"type":"int"}],"csv_settings":' + '{"delimiter":null,"doublequote":null,"encoding":null,"escapechar":null,' + '"keep_default_na":null,"lineterminator":null,"na_values":null,"quotechar":null,' + '"skipinitialspace":null},"name":"regions","rows":[["id","name"],["1","caf\\u00e9"]]}', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_fixed_inputs_when_building_identities_then_matches_pinned_values( + test_case: PersistedIdentityTestCase, tmp_path: Path +) -> None: + assert test_case.build(tmp_path) == test_case.expected_identity + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py index 6446e8d359..580b489c60 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_config.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild.compiler.planner.main.identity._version_identity_config import ( - build_version_identity_config, -) from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( CursorRoleIdentityTestCase, VersionIdentityConfigTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner.main.identity.helpers import ( + build_version_identity_config, +) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index f578be893f..1062a23f69 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -4,15 +4,13 @@ import pytest +from sqlbuild._native import build_function_local_hashes from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledObjectKey, FunctionReturnColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes, -) from sqlbuild.compiler.planner.models import DirectModelVersionIdentities from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py index f0b8a9edb4..47d74f9eb2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.changes._selection_staleness import ( - classify_selection_staleness_warnings, -) +from sqlbuild._native import classify_selection_staleness_warnings from sqlbuild.compiler.planner.models import ( SelectionStalenessGraph, SelectionStalenessNodeKey, From 67e2cdf82949f129bb8e8c74d395756e31865e7b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:32:29 +0100 Subject: [PATCH 07/32] chore: record the dbt init, compile reuse host and console entry stays --- python-boundary.toml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python-boundary.toml b/python-boundary.toml index d757220997..2972b6d3cf 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -329,6 +329,41 @@ path = "cli/commands/_helpers/entry/compute_log_diagnostics.py" reason = "user-code" note = "Logs compute log capture failures through Python logging during capture." +[[stays]] +path = "integrations/dbt/_helpers/profile/init.py" +reason = "user-code" +note = "Initializes a SQLBuild project from the user's dbt project and runs dbt debug as a subprocess." + +[[stays]] +path = "cli/compile_reuse/_helpers/provider_settings.py" +reason = "user-code" +note = "Inspects user provider settings classes for the environment and files they read." + +[[stays]] +path = "cli/compile_reuse/_helpers/runtime_identity.py" +reason = "user-code" +note = "Interpreter and invocation facts of the Python host that user code runs in." + +[[stays]] +path = "cli/compile_reuse/_helpers/startup_inputs.py" +reason = "user-code" +note = "Interpreter and site-path inputs user modules import from." + +[[stays]] +path = "cli/entry/main/entry.py" +reason = "public-api" +note = "The installed sqlbuild-python console entry point." + +[[stays]] +path = "cli/entry/main/launched.py" +reason = "public-api" +note = "The Python entry point the native sqb executable hands invocations to." + +[[stays]] +path = "cli/entry/_helpers/streams.py" +reason = "public-api" +note = "Reconfigures the Python process's standard streams for the console entry points." + [[stays]] path = "compiler/python_nodes/_helpers/identity.py" reason = "user-code" From 15024aea0aa578bad8610dcc5bdfbf2c1d6e2044 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:35:48 +0100 Subject: [PATCH 08/32] refactor: run the plan and rules commands natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 11 +- .../bindings/_helpers/check/check_command.rs | 3 +- .../src/bindings/_helpers/cli/registration.rs | 6 +- .../bindings/_helpers/clone/clone_command.rs | 3 +- .../_helpers/compile_driver/analysis.rs | 4 +- .../_helpers/contract_command/command.rs | 5 +- .../bindings/_helpers/diff_report/command.rs | 3 +- .../_helpers/format_command/selection.rs | 4 +- .../_helpers/freshness/freshness_command.rs | 5 +- .../_helpers/lineage/lineage_command.rs | 5 +- .../bindings/_helpers/load/load_command.rs | 3 +- .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/_helpers/plan_command/mod.rs | 3 + .../bindings/_helpers/plan_command/plan.rs | 300 +++++++++++++++ .../_helpers/plan_command/registration.rs | 10 + .../bindings/_helpers/plan_command/rules.rs | 355 ++++++++++++++++++ .../_helpers/scenario_command/setup.rs | 3 +- src/sqlbuild/_native.pyi | 4 + .../cli/commands/_helpers/plan/__init__.py | 0 .../cli/commands/_helpers/plan/invocation.py | 70 ---- .../cli/commands/_helpers/plan/outputs.py | 43 --- .../cli/commands/_helpers/plan/planning.py | 73 ---- .../cli/commands/_helpers/rules/command.py | 177 --------- .../cli/commands/main/project/_plan.py | 28 -- .../cli/commands/main/project/_rules.py | 10 - src/sqlbuild/cli/commands/models.py | 15 - ...lection.py => _project_graph_selection.py} | 0 ...tors.py => _reject_unit_test_selectors.py} | 0 ...build_catalogue.py => _build_catalogue.py} | 0 .../cli/commands/main/test_load_command.py | 18 +- 30 files changed, 713 insertions(+), 449 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/plan/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/plan/invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/plan/outputs.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/plan/planning.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/rules/command.py delete mode 100644 src/sqlbuild/cli/commands/main/project/_plan.py delete mode 100644 src/sqlbuild/cli/commands/main/project/_rules.py rename src/sqlbuild/compiler/pipeline/main/{project_graph_selection.py => _project_graph_selection.py} (100%) rename src/sqlbuild/compiler/pipeline/main/{reject_unit_test_selectors.py => _reject_unit_test_selectors.py} (100%) rename src/sqlbuild/rule_engine/main/{build_catalogue.py => _build_catalogue.py} (100%) diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 725f0cea3a..24518a3c75 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -143,9 +143,9 @@ const PLAN_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const PLAN_ROUTE: Route = routed( "run_plan", - ("sqlbuild.cli.commands.main.project._plan", "run_plan"), + ("sqlbuild._native", "run_plan_command"), &PLAN_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const BUILD_FIELDS: [Field; 34] = [ @@ -699,12 +699,9 @@ const RULES_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const RULES_ROUTE: Route = routed( "run_rules", - ( - "sqlbuild.cli.commands.main.project._rules", - "run_rules_command", - ), + ("sqlbuild._native", "run_rules_command"), &RULES_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const DBT_INIT_FIELDS: [Field; 9] = [ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index a2917f3156..bc6a986ef0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,8 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 7e12aa65a4..a22b85a7c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -6,8 +6,9 @@ use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, - format_command, freshness, init_command, janitor, load, presentation_text, query_command, - result_output, scenario_command, seed_command, sql_test_command, workspace_commands, + format_command, freshness, init_command, janitor, load, plan_command, presentation_text, + query_command, result_output, scenario_command, seed_command, sql_test_command, + workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -17,6 +18,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { progress_reporters::register(module)?; progress_text::register(module)?; janitor::registration::register(module)?; + plan_command::registration::register(module)?; freshness::registration::register(module)?; clone::registration::register(module)?; load::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index d131b3336b..116419d67a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -28,7 +28,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 31a965b2b0..ab86bde0ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const DISCOVER_MODULE: &str = "sqlbuild._native"; -const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; @@ -24,7 +24,7 @@ const FAST_COLUMN_LINEAGE: &str = "FAST"; const CONTRACTS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; -const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index d81bf43958..618e90158b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -23,10 +23,11 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 7b92d284e8..a4a47d1989 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -30,7 +30,8 @@ use crate::bindings::_helpers::discovery::configuration::discover_project_config use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 01508c0bdf..a170f4e0d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -20,8 +20,8 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const UNIT_TEST_SELECTORS_MODULE: &str = - "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index a6f4a96e63..0073094454 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -21,11 +21,12 @@ use crate::bindings::_helpers::freshness::freshness_observe::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const CONNECTION_MODULE: &str = "sqlbuild._native"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; const UPSTREAM_MODULE: &str = "sqlbuild.compiler.planner.main.selection._upstream"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index fe1361a9e8..92524026be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -42,8 +42,9 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; const SELECTION_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selection"; const SELECTOR_TOKENS_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_tokens"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 54353b4c43..76ec46b982 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -27,7 +27,8 @@ use crate::bindings::_helpers::load::load_selection::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 0588b71cc8..30b363e5b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -43,6 +43,7 @@ pub(crate) mod load; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; +pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; pub(crate) mod planner_identity; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs new file mode 100644 index 0000000000..5e6119a24d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod plan; +pub(crate) mod registration; +pub(crate) mod rules; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs new file mode 100644 index 0000000000..6025a057f7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -0,0 +1,300 @@ +//! `sqb plan`: compile the plan pipeline and print the plan as text or JSON. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::build_command::support::keyword_relay; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const VERBOSE_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; +const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; +const COMMAND_NAME: &str = "plan"; +const DEFAULT_SECTION_ENTRIES: i64 = 50; +const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; +/// Request fields passed to the compile pipeline under the same name. +const PIPELINE_FIELDS: [&str; 11] = [ + "no_sql_validation", + "no_cache", + "defer_to", + "defer_sources_to", + "cursor_overrides", + "full_refresh", + "select", + "exclude", + "cli_vars", + "max_microbatches", + "selection_diagnostics", +]; + +fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +fn print_line(py: Python<'_>, text: &str) -> PyResult<()> { + imported(py, "builtins", "print")?.call1((text,))?; + Ok(()) +} + +/// The discovered project, adapter context, output stream and progress reporters of one plan. +struct PlanInvocation<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + context: Bound<'py, PyAny>, + stream: Bound<'py, PyAny>, + use_color: bool, + connection_progress: Bound<'py, PyAny>, + planning_progress: Bound<'py, PyAny>, + load_sources: Bound<'py, PyAny>, +} + +fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let requested_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = if requested_dir.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested_dir + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?; + let preview: Bound<'py, PyDict> = PyDict::new(py); + preview.set_item("discovered_inputs", &discovered)?; + preview.set_item("as_target", request.getattr("as_target")?)?; + preview.set_item("command_name", COMMAND_NAME)?; + imported(py, NATIVE_MODULE, "validate_preview_target")?.call((), Some(&preview))?; + let context: Bound<'py, PyAny> = call_with( + &imported( + py, + ADAPTER_CONTEXT_MODULE, + "resolve_adapter_connection_context", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("effective_project_dir", project_dir.clone()), + ("selected_target", request.getattr("selected_target")?), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + let json_output: bool = request.getattr("json_output")?.is_truthy()?; + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && !json_output + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if json_output { "stderr" } else { "stdout" })?; + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", context.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", flag(py, use_color)), + ], + )?; + let planning_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", flag(py, use_color)), + ], + )?; + let requested_load: Bound<'py, PyAny> = request.getattr("load_sources")?; + let load_sources: Bound<'py, PyAny> = if requested_load.is_none() { + call_with( + &imported(py, NATIVE_MODULE, "build_effective_settings_config")?, + &[("discovered_inputs", discovered.clone())], + )? + .getattr("auto_load_sources")? + } else { + requested_load + }; + Ok(PlanInvocation { + project_dir, + discovered, + context, + stream, + use_color, + connection_progress, + planning_progress, + load_sources, + }) +} + +/// A `ConnectionHooks` forwarding to the plan's progress reporters. +fn connection_hooks<'py>(invocation: &PlanInvocation<'py>) -> PyResult> { + let py: Python<'py> = invocation.stream.py(); + let connection: &Bound<'py, PyAny> = &invocation.connection_progress; + call_with( + &imported(py, CONTRACT_MODELS, "ConnectionHooks")?, + &[ + ( + "on_progress", + invocation.planning_progress.getattr("on_progress")?, + ), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + keyword_relay( + connection.getattr("on_connection_complete")?, + &CONNECTION_TIMING, + )?, + ), + ( + "on_connection_error", + keyword_relay( + connection.getattr("on_connection_error")?, + &CONNECTION_TIMING, + )?, + ), + ], + ) +} + +fn compile_pipeline<'py>( + request: &Bound<'py, PyAny>, + invocation: &PlanInvocation<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.project_dir.clone()), + ("discovered_inputs", invocation.discovered.clone()), + ], + )?; + let as_target: Bound<'py, PyAny> = request.getattr("as_target")?; + if !as_target.is_none() { + invocation.stream.call_method1( + "write", + (format!( + "Previewing plan as target '{}' through the active target's connection (inspection only).\n", + as_target.str()? + ),), + )?; + invocation.stream.call_method0("flush")?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "selected_target", + if as_target.is_none() { + request.getattr("selected_target")? + } else { + as_target + }, + )?; + for field in PIPELINE_FIELDS { + options.set_item(field, request.getattr(field)?)?; + } + options.set_item("auto_load_sources", &invocation.load_sources)?; + options.set_item( + "connection_config", + invocation.context.getattr("connection_config")?, + )?; + options.set_item("external_sql_reference_resolver", resolver)?; + options.set_item( + "resolve_python_run_selectors", + request.getattr("include_python")?, + )?; + options.set_item("record_migration_fingerprints", false)?; + let pipeline_options: Bound<'py, PyAny> = + imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?.call((), Some(&options))?; + call_with( + &imported(py, NATIVE_MODULE, "run_compile_pipeline")?, + &[ + ("discovered_inputs", invocation.discovered.clone()), + ("adapter", invocation.context.getattr("adapter")?), + ("options", pipeline_options), + ("hooks", connection_hooks(invocation)?), + ], + ) +} + +fn write_output( + request: &Bound<'_, PyAny>, + invocation: &PlanInvocation<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = request.py(); + let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; + let python_entries: Bound<'_, PyAny> = pipeline.getattr("python_plan_entries")?; + if request.getattr("json_output")?.is_truthy()? { + let text: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_plan_json")?, + &[("plan", plan), ("python_plan_entries", python_entries)], + )?; + imported(py, "builtins", "print")?.call1((text,))?; + return Ok(()); + } + let display: Bound<'_, PyDict> = PyDict::new(py); + display.set_item( + "max_entries_per_section", + if request.getattr("verbose")?.is_truthy()? { + None + } else { + Some(DEFAULT_SECTION_ENTRIES) + }, + )?; + let text: String = call_with( + &imported(py, NATIVE_MODULE, "format_plan")?, + &[ + ("plan", plan), + ("full_refresh", request.getattr("full_refresh")?), + ("use_color", flag(py, invocation.use_color)), + ( + "display_options", + imported(py, PRESENTATION_MODELS, "DisplayOptions")?.call((), Some(&display))?, + ), + ("python_plan_entries", python_entries), + ("include_direct_freshness_diagnostics", flag(py, false)), + ("include_completion", flag(py, true)), + ], + )? + .extract()?; + print_line(py, &format!("\n{text}")) +} + +/// Run one `sqb plan` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_plan_command(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let invocation: PlanInvocation<'_> = resolve_invocation(request)?; + if !request.getattr("json_output")?.is_truthy()? { + invocation.stream.call_method1("write", ("\n",))?; + invocation.stream.call_method0("flush")?; + } + let verbose: Bound<'_, PyDict> = PyDict::new(py); + verbose.set_item("enabled", request.getattr("verbose")?)?; + let pipeline: Bound<'_, PyAny> = within( + &imported(py, VERBOSE_MODULE, "verbose_inspection_output")?.call((), Some(&verbose))?, + |_| compile_pipeline(request, &invocation), + )?; + write_output(request, &invocation, &pipeline)?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_plan_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs new file mode 100644 index 0000000000..f9933699b4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb plan` and `sqb rules` commands. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::plan_command::{plan, rules}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan::register(module)?; + rules::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs new file mode 100644 index 0000000000..fae218bd04 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -0,0 +1,355 @@ +//! `sqb rules`: list, show, install skills for, or run a selection of compiler rules. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const LOAD_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; +const CATALOGUE_MODULE: &str = "sqlbuild.rule_engine.main._build_catalogue"; +const SKILLS_MODULE: &str = "sqlbuild.rule_engine.main.skills"; +const RENDER_RULE_MODULE: &str = "sqlbuild.rule_engine.main.render_rule"; +const RENDER_RESULT_MODULE: &str = "sqlbuild.rule_engine.main.render_result"; +const SKIPPED_RULES_MODULE: &str = "sqlbuild.rule_engine.main.render_skipped_rules"; +const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; +const LIST_ACTION: &str = "list"; +const SHOW_ACTION: &str = "show"; +const SKILLS_ACTION: &str = "skills"; +const GENERIC_DIALECT: &str = "generic"; +const USAGE_EXIT_CODE: i32 = 2; +const RESULT_FIELDS: [&str; 5] = [ + "findings", + "evaluated_models", + "unevaluated_resources", + "cache_hits", + "cache_misses", +]; + +/// `print(text)`, or `print(text, file=sys.stderr)`. +fn print_line(py: Python<'_>, text: &str, stderr: bool) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + if stderr { + options.set_item("file", py.import("sys")?.getattr("stderr")?)?; + } + imported(py, "builtins", "print")?.call((text,), Some(&options))?; + Ok(()) +} + +fn rule_codes(catalogue: &Bound<'_, PyAny>) -> PyResult> { + let mut codes: Vec = Vec::new(); + for rule in catalogue.try_iter()? { + codes.push(rule?.getattr("code")?.extract()?); + } + Ok(codes) +} + +fn run_skills( + request: &Bound<'_, PyAny>, + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = request.py(); + let check: bool = request.getattr("skills_check")?.is_truthy()?; + let fresh: bool = call_with( + &imported(py, SKILLS_MODULE, "install_skills")?, + &[ + ("config", config.clone()), + ("project_dir", project_dir.clone()), + ("check", request.getattr("skills_check")?), + ], + )? + .is_truthy()?; + if !check { + print_line(py, "Installed Rules skills", false)?; + return Ok(0); + } + print_line( + py, + if fresh { + "Rules skills are fresh" + } else { + "Rules skills are stale" + }, + false, + )?; + Ok(if fresh { 0 } else { 1 }) +} + +fn list_rules(catalogue: &Bound<'_, PyAny>, json_output: bool) -> PyResult { + let py: Python<'_> = catalogue.py(); + if !json_output { + let mut lines: Vec = Vec::new(); + for rule in catalogue.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + lines.push(format!( + "{} {}", + rule.getattr("code")?.str()?, + rule.getattr("message")?.str()? + )); + } + print_line(py, &lines.join("\n"), false)?; + return Ok(0); + } + let rules: Bound<'_, PyList> = PyList::empty(py); + for rule in catalogue.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + let item: Bound<'_, PyDict> = PyDict::new(py); + for field in ["code", "family", "message", "custom"] { + item.set_item(field, rule.getattr(field)?)?; + } + let subject: Bound<'_, PyAny> = rule.getattr("subject")?; + item.set_item( + "subject", + if subject.is_none() { + subject + } else { + subject.getattr("value")? + }, + )?; + rules.append(item)?; + } + let payload: Bound<'_, PyDict> = PyDict::new(py); + payload.set_item("rules", rules)?; + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("sort_keys", true)?; + let text: String = imported(py, "json", "dumps")? + .call((payload,), Some(&options))? + .extract()?; + print_line(py, &text, false)?; + Ok(0) +} + +fn show_rule( + selector: &str, + catalogue: &Bound<'_, PyAny>, + config: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = catalogue.py(); + for rule in catalogue.try_iter()? { + let rule: Bound<'_, PyAny> = rule?; + if rule.getattr("code")?.extract::()? == selector { + let text: String = call_with( + &imported(py, RENDER_RULE_MODULE, "format_rule")?, + &[("rule", rule), ("config", config.clone())], + )? + .extract()?; + print_line(py, &text, false)?; + return Ok(0); + } + } + print_line(py, &format!("Unknown rule: {selector}"), false)?; + Ok(USAGE_EXIT_CODE) +} + +/// The discovered project, its adapter, graph and selected keys a rule selection runs over. +struct RuleInputs<'py> { + discovered: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + graph: Bound<'py, PyAny>, + selected: Bound<'py, PyAny>, +} + +fn rule_inputs<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let select: Bound<'py, PyAny> = request.getattr("select")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", select.clone()), + ("exclude", exclude.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )?; + let graph: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_project_graph")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ], + )?; + let selected: Bound<'py, PyAny> = call_with( + &imported(py, GRAPH_SELECTION_MODULE, "select_project_graph")?, + &[ + ("graph", graph.clone()), + ("select", select), + ("exclude", exclude), + ], + )?; + Ok(RuleInputs { + discovered, + adapter, + graph, + selected, + }) +} + +fn run_rule_selection( + request: &Bound<'_, PyAny>, + (selector, catalogue): (&str, &Bound<'_, PyAny>), + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = request.py(); + if !rule_codes(catalogue)? + .iter() + .any(|code| code.starts_with(selector)) + { + print_line(py, &format!("Unknown rule or family: {selector}"), false)?; + return Ok(USAGE_EXIT_CODE); + } + print_line( + py, + &format!("Evaluating rule selection {selector}..."), + true, + )?; + let RuleInputs { + discovered, + adapter, + graph, + selected, + } = rule_inputs(request, project_dir)?; + let narrowed: bool = + request.getattr("select")?.is_truthy()? || request.getattr("exclude")?.is_truthy()?; + let replaced: Bound<'_, PyDict> = PyDict::new(py); + replaced.set_item("select", PyTuple::new(py, [selector])?)?; + replaced.set_item("ignore", PyTuple::empty(py))?; + let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; + let result: Bound<'_, PyAny> = call_with( + &imported(py, RUN_RULES_MODULE, "run_rules")?, + &[ + ("graph", graph), + ("discovered_inputs", discovered), + ( + "config", + imported(py, "dataclasses", "replace")?.call((config,), Some(&replaced))?, + ), + ("project_dir", project_dir.clone()), + ( + "dialect", + if dialect.is_truthy()? { + dialect + } else { + pyo3::types::PyString::new(py, GENERIC_DIALECT).into_any() + }, + ), + ( + "selected_keys", + if narrowed { + selected + } else { + py.None().into_bound(py) + }, + ), + ], + )?; + let note: Bound<'_, PyAny> = call_with( + &imported(py, SKIPPED_RULES_MODULE, "format_skipped_type_proof_rules")?, + &[("codes", result.getattr("skipped_type_proof_rules")?)], + )?; + if !note.is_none() { + print_line(py, &format!("note: {}", note.str()?), true)?; + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + for field in RESULT_FIELDS { + fields.set_item(field, result.getattr(field)?)?; + } + let text: String = call_with( + &imported(py, RENDER_RESULT_MODULE, "format_result")?, + &[ + ( + "result", + imported(py, RULE_MODELS, "RulesResult")?.call((), Some(&fields))?, + ), + ("json_output", request.getattr("json_output")?), + ], + )? + .extract()?; + print_line(py, &text, false)?; + let findings: usize = result.getattr("findings")?.len()?; + if findings > 0 { + print_line( + py, + &format!("Rule selection {selector} failed with {findings} finding(s)."), + true, + )?; + return Ok(1); + } + print_line(py, &format!("Rule selection {selector} passed."), true)?; + Ok(0) +} + +/// List, inspect, execute, or generate guidance for compiler rules. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_rules_command(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let requested_dir: Bound<'_, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'_, PyAny> = if requested_dir.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested_dir + } + .call_method0("resolve")?; + let config: Bound<'_, PyAny> = call_with( + &imported(py, LOAD_CONFIG_MODULE, "load_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + let action: String = request.getattr("action")?.extract()?; + if action == SKILLS_ACTION { + return run_skills(request, &config, &project_dir); + } + let catalogue: Bound<'_, PyAny> = call_with( + &imported(py, CATALOGUE_MODULE, "build_catalogue")?, + &[ + ("config", config.clone()), + ("project_dir", project_dir.clone()), + ], + )?; + if action == LIST_ACTION { + return list_rules(&catalogue, request.getattr("json_output")?.is_truthy()?); + } + let selector: Option = request.getattr("rule_selector")?.extract()?; + let Some(selector) = selector else { + print_line(py, "A rule code or family prefix is required", false)?; + return Ok(USAGE_EXIT_CODE); + }; + if action == SHOW_ACTION { + return show_rule(&selector, &catalogue, &config); + } + run_rule_selection(request, (&selector, &catalogue), &config, &project_dir) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_rules_command, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index cd6fd74b01..8e58cd6110 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -25,7 +25,8 @@ use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = + "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1d75d18537..a74c62182b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -36,10 +36,12 @@ from sqlbuild.cli.commands.models import ( JanitorCommandRequest, LineageCommandRequest, LoadCommandRequest, + PlanCommandRequest, PlanSafetyGates, PlaygroundCommandRequest, QueryDiffPreparation, RefactorCommandRequest, + RulesCommandRequest, ScenarioCaptureCommandRequest, ScenarioTestCommandRequest, ScopeCommandRequest, @@ -3377,6 +3379,8 @@ def resolve_loader_defaults( def write_runtime_target( *, target_dir: Path, plan_output: PlanOutput, result: BuildExecutionResult ) -> None: ... +def run_plan_command(*, request: PlanCommandRequest) -> int: ... +def run_rules_command(*, request: RulesCommandRequest) -> int: ... class TransientStatusReporter: def __init__( diff --git a/src/sqlbuild/cli/commands/_helpers/plan/__init__.py b/src/sqlbuild/cli/commands/_helpers/plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py deleted file mode 100644 index 15aebce087..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Plan command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - build_effective_settings_config, - discover_project_inputs, - validate_preview_target, -) -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.models import ( - AdapterConnectionContext, - PlanCommandRequest, - PlanInvocation, -) -from sqlbuild.cli.progress.main._build_command_progress_reporters import ( - build_command_progress_reporters, -) -from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color - - -def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: - """Resolve discovery, validations, adapter, connection, and reporters for plan.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir, extract_output_column_locations=False - ) - validate_preview_target( - discovered_inputs=discovered_inputs, as_target=request.as_target, command_name="plan" - ) - adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=discovered_inputs, - effective_project_dir=effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - use_color: bool = not request.no_color and not request.json_output and supports_color() - progress_stream: TextIO = sys.stderr if request.json_output else sys.stdout - reporters: CommandProgressReporters = build_command_progress_reporters( - adapter_name=adapter_context.adapter_name, - stream=progress_stream, - use_color=use_color, - ) - should_load_sources: bool = ( - request.load_sources - if request.load_sources is not None - else build_effective_settings_config(discovered_inputs=discovered_inputs).auto_load_sources - ) - return PlanInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter_context.adapter, - connection_config=adapter_context.connection_config, - use_color=use_color, - progress_stream=progress_stream, - connection_progress=reporters.connection, - planning_progress=reporters.planning, - should_load_sources=should_load_sources, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py deleted file mode 100644 index c513c58271..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Plan command output writing phase.""" - -from __future__ import annotations - -from sqlbuild._native import format_plan, format_plan_json -from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions - - -def write_plan_command_output( - *, - request: PlanCommandRequest, - invocation: PlanInvocation, - pipeline_result: CompilePipelineResult, -) -> None: - """Write the plan output as JSON or formatted text.""" - - plan_output: PlanOutput = pipeline_result.plan_output - if request.json_output: - print( - format_plan_json( - plan=plan_output, - python_plan_entries=pipeline_result.python_plan_entries, - ) - ) - return - display_options: DisplayOptions = DisplayOptions( - max_entries_per_section=None if request.verbose else 50 - ) - print( - "\n" - + format_plan( - plan=plan_output, - full_refresh=request.full_refresh, - use_color=invocation.use_color, - display_options=display_options, - python_plan_entries=pipeline_result.python_plan_entries, - include_direct_freshness_diagnostics=(False), - include_completion=True, - ) - ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py deleted file mode 100644 index b52075934b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Plan command pipeline compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import run_compile_pipeline -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, -) -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def compile_plan_pipeline( - *, - request: PlanCommandRequest, - invocation: PlanInvocation, -) -> CompilePipelineResult: - """Compile the plan pipeline.""" - - external_sql_reference_resolver: object | None = resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ) - if request.as_target is not None: - invocation.progress_stream.write( - f"Previewing plan as target '{request.as_target}' through the active target's " - "connection (inspection only).\n" - ) - invocation.progress_stream.flush() - result: CompilePipelineResult = run_compile_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=CompilePipelineOptions( - selected_target=( - request.as_target if request.as_target is not None else request.selected_target - ), - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - defer_to=request.defer_to, - defer_sources_to=request.defer_sources_to, - cursor_overrides=request.cursor_overrides, - full_refresh=request.full_refresh, - auto_load_sources=invocation.should_load_sources, - select=request.select, - exclude=request.exclude, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - resolve_python_run_selectors=request.include_python, - max_microbatches=request.max_microbatches, - selection_diagnostics=request.selection_diagnostics, - record_migration_fingerprints=False, - ), - hooks=ConnectionHooks( - on_progress=invocation.planning_progress.on_progress, - on_connection_start=invocation.connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - return result diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py deleted file mode 100644 index d158cfa71a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Focused compiler-rule command implementation.""" - -from __future__ import annotations - -import json -import sys -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - build_project_graph, - discover_project_inputs, - resolve_effective_adapter_name, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.constants import ( - RULES_LIST_ACTION, - RULES_SHOW_ACTION, - RULES_SKILLS_ACTION, -) -from sqlbuild.cli.commands.models import RulesCommandRequest -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine.main.build_catalogue import build_catalogue -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.render_result import format_result -from sqlbuild.rule_engine.main.render_rule import format_rule -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -from sqlbuild.rule_engine.main.run_rules import run_rules -from sqlbuild.rule_engine.main.skills import install_skills -from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult, RulesRunResult - - -def run_rules_command(request: RulesCommandRequest) -> int: - """List, inspect, execute, or generate guidance for compiler rules.""" - - project_dir: Path = ( - Path.cwd() if request.project_dir is None else request.project_dir - ).resolve() - config: RulesConfig = load_rules_config(project_dir=project_dir) - if request.action == RULES_SKILLS_ACTION: - return _run_skills(request=request, config=config, project_dir=project_dir) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) - if request.action == RULES_LIST_ACTION: - return _list_rules(catalogue=catalogue, json_output=request.json_output) - return _run_catalogue_action( - request=request, - catalogue=catalogue, - config=config, - project_dir=project_dir, - ) - - -def _run_skills(*, request: RulesCommandRequest, config: RulesConfig, project_dir: Path) -> int: - fresh: bool = install_skills(config=config, project_dir=project_dir, check=request.skills_check) - if request.skills_check: - print("Rules skills are fresh" if fresh else "Rules skills are stale") - return 0 if fresh else 1 - print("Installed Rules skills") - return 0 - - -def _list_rules(*, catalogue: tuple[Rule, ...], json_output: bool) -> int: - if json_output: - payload: dict[str, object] = { - "rules": [ - { - "code": rule.code, - "family": rule.family, - "message": rule.message, - "custom": rule.custom, - "subject": rule.subject.value if rule.subject is not None else None, - } - for rule in catalogue - ] - } - print(json.dumps(payload, sort_keys=True)) - else: - print("\n".join(f"{rule.code} {rule.message}" for rule in catalogue)) - return 0 - - -def _run_catalogue_action( - *, - request: RulesCommandRequest, - catalogue: tuple[Rule, ...], - config: RulesConfig, - project_dir: Path, -) -> int: - selector: str | None = request.rule_selector - if selector is None: - print("A rule code or family prefix is required") - return 2 - if request.action == RULES_SHOW_ACTION: - return _show_rule(selector=selector, catalogue=catalogue, config=config) - return _run_rule_selection( - request=request, - selector=selector, - catalogue=catalogue, - config=config, - project_dir=project_dir, - ) - - -def _show_rule(*, selector: str, catalogue: tuple[Rule, ...], config: RulesConfig) -> int: - matching: tuple[Rule, ...] = tuple(rule for rule in catalogue if rule.code == selector) - if not matching: - print(f"Unknown rule: {selector}") - return 2 - print(format_rule(rule=matching[0], config=config)) - return 0 - - -def _run_rule_selection( - *, - request: RulesCommandRequest, - selector: str, - catalogue: tuple[Rule, ...], - config: RulesConfig, - project_dir: Path, -) -> int: - if not any(rule.code.startswith(selector) for rule in catalogue): - print(f"Unknown rule or family: {selector}") - return 2 - print(f"Evaluating rule selection {selector}...", file=sys.stderr) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered.project_config, - local_config=discovered.local_config, - ), - project_dir=project_dir, - ) - graph: ProjectGraph = build_project_graph(discovered_inputs=discovered, adapter=adapter) - selected_keys: frozenset[CompiledObjectKey] = select_project_graph( - graph=graph, select=request.select, exclude=request.exclude - ) - result: RulesRunResult = run_rules( - graph=graph, - discovered_inputs=discovered, - config=replace(config, select=(selector,), ignore=()), - project_dir=project_dir, - dialect=adapter.sql_analysis_dialect() or "generic", - selected_keys=selected_keys if request.select or request.exclude else None, - ) - skipped_rules_note: str | None = format_skipped_type_proof_rules( - codes=result.skipped_type_proof_rules - ) - if skipped_rules_note is not None: - print(f"note: {skipped_rules_note}", file=sys.stderr) - print( - format_result( - result=RulesResult( - findings=result.findings, - evaluated_models=result.evaluated_models, - unevaluated_resources=result.unevaluated_resources, - cache_hits=result.cache_hits, - cache_misses=result.cache_misses, - ), - json_output=request.json_output, - ) - ) - if result.findings: - print( - f"Rule selection {selector} failed with {len(result.findings)} finding(s).", - file=sys.stderr, - ) - return 1 - print(f"Rule selection {selector} passed.", file=sys.stderr) - return 0 diff --git a/src/sqlbuild/cli/commands/main/project/_plan.py b/src/sqlbuild/cli/commands/main/project/_plan.py deleted file mode 100644 index 6f47ec724c..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_plan.py +++ /dev/null @@ -1,28 +0,0 @@ -"""CLI plan command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.plan.invocation import resolve_plan_invocation -from sqlbuild.cli.commands._helpers.plan.outputs import write_plan_command_output -from sqlbuild.cli.commands._helpers.plan.planning import compile_plan_pipeline -from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output -from sqlbuild.compiler.pipeline.models import CompilePipelineResult - - -def run_plan(request: PlanCommandRequest) -> int: - """Execute the plan command.""" - - invocation: PlanInvocation = resolve_plan_invocation(request=request) - if not request.json_output: - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - pipeline_result: CompilePipelineResult - with verbose_inspection_output(enabled=request.verbose): - pipeline_result = compile_plan_pipeline(request=request, invocation=invocation) - write_plan_command_output( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - return 0 diff --git a/src/sqlbuild/cli/commands/main/project/_rules.py b/src/sqlbuild/cli/commands/main/project/_rules.py deleted file mode 100644 index 63a6589f08..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_rules.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Focused compiler-rule CLI entrypoint.""" - -from sqlbuild.cli.commands._helpers.rules.command import run_rules_command as execute_rules_command -from sqlbuild.cli.commands.models import RulesCommandRequest - - -def run_rules_command(request: RulesCommandRequest) -> int: - """Execute one focused Rules command.""" - - return execute_rules_command(request) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 6d010b1750..ef924758ec 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -425,21 +425,6 @@ class PlanCommandRequest: selection_diagnostics: bool = False -@dataclass(frozen=True) -class PlanInvocation: - """Resolved project, adapter, and reporter context for the plan command.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - connection_progress: ConnectionProgressReporter - planning_progress: PlanningProgressReporter - should_load_sources: bool - - @dataclass(frozen=True) class PlaygroundCommandRequest: """CLI inputs for one playground command invocation.""" diff --git a/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py b/src/sqlbuild/compiler/pipeline/main/_project_graph_selection.py similarity index 100% rename from src/sqlbuild/compiler/pipeline/main/project_graph_selection.py rename to src/sqlbuild/compiler/pipeline/main/_project_graph_selection.py diff --git a/src/sqlbuild/compiler/pipeline/main/reject_unit_test_selectors.py b/src/sqlbuild/compiler/pipeline/main/_reject_unit_test_selectors.py similarity index 100% rename from src/sqlbuild/compiler/pipeline/main/reject_unit_test_selectors.py rename to src/sqlbuild/compiler/pipeline/main/_reject_unit_test_selectors.py diff --git a/src/sqlbuild/rule_engine/main/build_catalogue.py b/src/sqlbuild/rule_engine/main/_build_catalogue.py similarity index 100% rename from src/sqlbuild/rule_engine/main/build_catalogue.py rename to src/sqlbuild/rule_engine/main/_build_catalogue.py diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 4a59de8ac9..db6c1fc58e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -19,6 +19,7 @@ run_audit_command, run_build_command, run_load_command, + run_plan_command, run_scenario_test_command, run_test_command, select_load_entries, @@ -26,7 +27,6 @@ from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.main.project._plan import run_plan from sqlbuild.cli.commands.models import ( AuditCommandRequest, BuildCommandRequest, @@ -1065,8 +1065,8 @@ def test_given_plan_source_deferral_override_when_planning_then_succeeds( ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_plan( - PlanCommandRequest( + exit_code: int = run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -1280,8 +1280,8 @@ def test_given_invalid_managed_source_target_config_when_planning_then_errors( write_repo_files(tmp_path, test_case.project_files) with pytest.raises(PlannerInputError) as exc_info: - run_plan( - PlanCommandRequest( + run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, select=test_case.select, @@ -1722,8 +1722,8 @@ def test_given_plan_load_controls_when_running_plan_then_formats_source_load_sec ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_plan( - PlanCommandRequest( + exit_code: int = run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, select=("stg_orders",), @@ -1795,8 +1795,8 @@ def test_given_plan_json_output_when_source_auto_loads_then_includes_source_load ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_plan( - PlanCommandRequest( + exit_code: int = run_plan_command( + request=PlanCommandRequest( project_dir=tmp_path, no_color=True, json_output=True, From 825ee0443fbbf33c7bd18322da1bd76232d07863 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:58:08 +0100 Subject: [PATCH 09/32] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 2972b6d3cf..e7c5996967 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 47826 +remaining_lines = 46953 [[stays]] path = "executor" From 461d8caf070e7216ac990133bb94a3a0facb66ab Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:57:11 +0100 Subject: [PATCH 10/32] refactor: build the rule catalogue and render rule guidance natively --- .../_helpers/rules/builtin_catalogue.rs | 5 +- .../src/bindings/_helpers/rules/catalogue.rs | 413 ++++++++++++++++++ .../_helpers/rules/custom_rule_evidence.rs | 2 +- .../src/bindings/_helpers/rules/guidance.rs | 227 ++++++++++ .../src/bindings/_helpers/rules/mod.rs | 2 + .../bindings/_helpers/rules/registration.rs | 9 +- .../src/bindings/_helpers/rules/selection.rs | 2 +- .../src/bindings/_helpers/rules/skills.rs | 25 +- crates/sqlbuild-rules/src/errors.rs | 15 + crates/sqlbuild-rules/src/lib.rs | 1 + .../src/rule_catalogue/_helpers/mod.rs | 4 + .../src/rule_catalogue/_helpers/options.rs | 46 ++ .../rule_catalogue/_helpers/python_repr.rs | 41 ++ .../src/rule_catalogue/_helpers/staging.rs | 221 ++++++++++ .../src/rule_catalogue/_helpers/thresholds.rs | 37 ++ .../src/rule_catalogue/constants.rs | 135 ++++++ .../rule_catalogue/main/format_rule_text.rs | 54 +++ .../main/install_rendered_skills.rs | 29 ++ .../src/rule_catalogue/main/mod.rs | 6 + .../rule_catalogue/main/render_skills_body.rs | 155 +++++++ .../main/ruleset_fingerprint.rs | 55 +++ .../rule_catalogue/main/skill_lock_name.rs | 8 + .../main/sorted_catalogue_order.rs | 157 +++++++ .../sqlbuild-rules/src/rule_catalogue/mod.rs | 9 + .../src/rule_catalogue/models.rs | 87 ++++ .../src/rule_catalogue/tests.rs | 6 + .../src/rule_catalogue/tests/helpers.rs | 101 +++++ .../tests/test_rule_catalogue.rs | 217 +++++++++ .../src/rule_catalogue/tests/test_types.rs | 32 ++ python-boundary.toml | 5 + .../_helpers/varied_benchmark.py | 8 +- src/sqlbuild/_native.pyi | 19 +- .../_helpers/engine/_custom_catalogue.py | 76 ++++ .../rule_engine/_helpers/engine/catalogue.py | 237 ---------- .../rule_engine/_helpers/engine/config.py | 52 --- .../rule_engine/_helpers/engine/ruleset.py | 133 ------ .../_helpers/guidance/presentation.py | 40 -- .../_helpers/guidance/skill_installation.py | 137 ------ .../rule_engine/_helpers/guidance/skills.py | 152 ------- .../_helpers/guidance/thresholds.py | 19 - .../rule_engine/_helpers/host/custom_host.py | 6 +- .../rule_engine/_helpers/run/rules.py | 9 +- .../rule_engine/main/_build_catalogue.py | 12 - src/sqlbuild/rule_engine/main/_evaluate.py | 26 +- src/sqlbuild/rule_engine/main/render_rule.py | 10 - src/sqlbuild/rule_engine/main/skills.py | 48 -- .../compile/test_dense_compile_performance.py | 8 +- .../rule_engine/_helpers/engine/helpers.py | 29 +- .../_helpers/engine/test_catalogue.py | 38 +- .../engine/test_custom_rule_evidence.py | 4 +- .../rule_engine/_helpers/host/helpers.py | 12 +- .../rule_engine/main/skills/conftest.py | 140 ------ .../rule_engine/main/skills/test_skills.py | 315 +------------ 53 files changed, 2269 insertions(+), 1367 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/constants.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/mod.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/models.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/tests.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs create mode 100644 src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/engine/catalogue.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/engine/config.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/engine/ruleset.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/guidance/presentation.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/guidance/skills.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py delete mode 100644 src/sqlbuild/rule_engine/main/_build_catalogue.py delete mode 100644 src/sqlbuild/rule_engine/main/render_rule.py delete mode 100644 src/sqlbuild/rule_engine/main/skills.py delete mode 100644 tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index edbec57e0a..abeb1410a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -115,9 +115,7 @@ fn rule<'py>( } /// Every built-in rule as a `Rule` whose `check` is the given placeholder. -#[pyfunction] -#[pyo3(signature = (*, check))] -fn builtin_rules<'py>( +pub(crate) fn builtin_rules<'py>( py: Python<'py>, check: Bound<'py, PyAny>, ) -> PyResult>> { @@ -137,7 +135,6 @@ fn selected_sql_rule_codes(select: Vec, ignore: Vec) -> PyResult } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(builtin_rules, module)?)?; module.add_function(wrap_pyfunction!(selected_sql_rule_codes, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs new file mode 100644 index 0000000000..8d9fc09f6d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -0,0 +1,413 @@ +//! The validated rule catalogue, the active rules it selects and resolved ignore selectors. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::models::RulesConfig; +use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::models::{CatalogueRule, OptionKind, OptionSpec, OptionValue}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::builtin_catalogue::builtin_rules; +use crate::bindings::_helpers::rules::selection::{rules_error, selected_rule_codes}; + +const CUSTOM_CATALOGUE_MODULE: &str = "sqlbuild.rule_engine._helpers.engine._custom_catalogue"; +const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; +const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; +const PLANNER_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.scope"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const JSON_MODULE: &str = "json"; +const BUILTINS_MODULE: &str = "builtins"; +const RULES_DIRECTORY: &str = "rules"; +const CUSTOM_RULE_PREFIX: &str = "XSQBR"; +const BUILTIN_CHECK_MESSAGE: &str = "built-in rules execute only through the native engine"; + +/// The Python rules configuration as the native configuration model. +pub(crate) fn native_rules_config(config: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = config.py(); + let fields: Bound<'_, PyAny> = imported(py, DATACLASSES_MODULE, "asdict")?.call1((config,))?; + let encoded: String = imported(py, JSON_MODULE, "dumps")? + .call( + (fields,), + Some(&keywords( + py, + &[("default", imported(py, BUILTINS_MODULE, "str")?)], + )?), + )? + .extract()?; + serde_json::from_str(&encoded) + .map_err(|error| rules_error(py, &format!("invalid rules configuration: {error}"), None)) +} + +/// Whether any selector, option, exception or ignore of `config` names a custom rule. +pub(crate) fn references_custom_rules(config: &Bound<'_, PyAny>) -> PyResult { + let mut selectors: Vec = Vec::new(); + for field in ["select", "ignore", "rule_options"] { + for selector in config.getattr(field)?.try_iter()? { + selectors.push(selector?.extract()?); + } + } + for entry in config.getattr("rule_exceptions")?.try_iter()? { + selectors.push(entry?.getattr("rule")?.extract()?); + } + for entry in config.getattr("rule_ignores")?.try_iter()? { + for rule in entry?.getattr("rules")?.try_iter()? { + selectors.push(rule?.extract()?); + } + } + Ok(selectors + .iter() + .any(|selector| selector.starts_with(CUSTOM_RULE_PREFIX))) +} + +fn option_kind(value_type: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = value_type.py(); + let kinds: [(Bound<'_, PyAny>, OptionKind); 4] = [ + (py.get_type::().into_any(), OptionKind::Bool), + (py.get_type::().into_any(), OptionKind::Int), + (py.get_type::().into_any(), OptionKind::Str), + (py.get_type::().into_any(), OptionKind::Tuple), + ]; + for (class, kind) in kinds { + if value_type.is(&class) { + return Ok(kind); + } + } + let name: String = match value_type.getattr("__name__") { + Ok(name) => name.extract()?, + Err(_) => value_type.repr()?.extract()?, + }; + Ok(OptionKind::Other(name)) +} + +fn option_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(OptionValue::Bool(value.extract()?)); + } + if value.is_instance_of::() { + return Ok(value + .extract::() + .map_or_else(|_| OptionValue::Other("int".into()), OptionValue::Int)); + } + if value.is_instance_of::() { + return Ok(OptionValue::Str(value.extract()?)); + } + if let Ok(items) = value.cast::() { + return Ok(OptionValue::Tuple( + items + .iter() + .map(|item| option_value(&item)) + .collect::>()?, + )); + } + Ok(OptionValue::Other( + value.get_type().getattr("__name__")?.extract()?, + )) +} + +fn optional_integer(option: &Bound<'_, PyAny>, field: &str) -> PyResult> { + option.getattr(field)?.extract() +} + +fn option_spec(option: &Bound<'_, PyAny>) -> PyResult { + let item_type: Bound<'_, PyAny> = option.getattr("item_type")?; + Ok(OptionSpec { + name: option.getattr("name")?.extract()?, + description: option.getattr("description")?.extract()?, + value_type: option_kind(&option.getattr("value_type")?)?, + default: option_value(&option.getattr("default")?)?, + choices: option + .getattr("choices")? + .try_iter()? + .map(|choice| option_value(&choice?)) + .collect::>()?, + minimum: optional_integer(option, "minimum")?, + maximum: optional_integer(option, "maximum")?, + minimum_items: optional_integer(option, "minimum_items")?, + item_type: if item_type.is_none() { + None + } else { + Some(option_kind(&item_type)?) + }, + }) +} + +fn catalogue_rule(rule: &Bound<'_, PyAny>) -> PyResult { + Ok(CatalogueRule { + code: rule.getattr("code")?.extract()?, + options: rule + .getattr("options")? + .try_iter()? + .map(|option| option_spec(&option?)) + .collect::>()?, + }) +} + +fn configured_options( + config: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut configured: BTreeMap> = BTreeMap::new(); + let options: Bound<'_, PyDict> = config.getattr("rule_options")?.cast_into::()?; + for (code, values) in options.iter() { + let values: Bound<'_, PyDict> = values.cast_into::()?; + let mut entries: BTreeMap = BTreeMap::new(); + for (name, value) in values.iter() { + entries.insert(name.extract()?, option_value(&value)?); + } + configured.insert(code.extract()?, entries); + } + Ok(configured) +} + +fn custom_rules<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult>> { + let py: Python<'py> = project_dir.py(); + let root: PathBuf = project_dir.extract()?; + let directory: PathBuf = root.join(RULES_DIRECTORY); + if !directory.is_dir() { + return Ok(Vec::new()); + } + let escapes = |directory: &Path| -> bool { + match (directory.canonicalize(), root.canonicalize()) { + (Ok(resolved), Ok(root)) => !resolved.starts_with(root), + _ => true, + } + }; + if escapes(&directory) { + return Err(rules_error( + py, + &format!("rule path escapes the project: {RULES_DIRECTORY}"), + None, + )); + } + call_with( + &imported(py, CUSTOM_CATALOGUE_MODULE, "load_custom_rules")?, + &[("project_dir", project_dir.clone())], + )? + .try_iter()? + .collect() +} + +/// The validated built-in and, when requested, custom catalogue sorted by rule code. +pub(crate) fn rule_catalogue<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + include_custom: bool, +) -> PyResult>> { + let py: Python<'py> = config.py(); + let check: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + |arguments: &Bound<'_, PyTuple>, _| -> PyResult<()> { + Err(rules_error(arguments.py(), BUILTIN_CHECK_MESSAGE, None)) + }, + )?; + let mut rules: Vec> = builtin_rules(py, check.into_any())?; + if include_custom { + rules.extend(custom_rules(project_dir)?); + } + let views: Vec = rules.iter().map(catalogue_rule).collect::>()?; + let configured = configured_options(config)?; + let order: Vec = compiler_guard(|| Ok(sorted_catalogue_order(&views, &configured)))? + .map_err(|message| rules_error(py, &message, None))?; + Ok(order + .into_iter() + .map(|index| rules[index].clone()) + .collect()) +} + +/// The catalogue rules the configuration activates, in native selection order. +pub(crate) fn select_rules<'py>( + catalogue: &[Bound<'py, PyAny>], + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + custom_payloads: Option>, +) -> PyResult>> { + let py: Python<'py> = config.py(); + let payloads: Bound<'py, PyAny> = match custom_payloads { + Some(payloads) if !payloads.is_none() => payloads, + _ => call_with( + &imported(py, CUSTOM_PAYLOADS_MODULE, "custom_rule_payloads")?, + &[ + ("catalogue", PyTuple::new(py, catalogue)?.into_any()), + ("project_dir", project_dir.clone()), + ], + )?, + }; + let mut by_code: BTreeMap> = BTreeMap::new(); + for rule in catalogue { + by_code.insert(rule.getattr("code")?.extract()?, rule.clone()); + } + selected_rule_codes(config, &payloads)? + .into_iter() + .map(|code| { + by_code.get(&code).cloned().ok_or_else(|| { + rules_error( + py, + &format!("native compiler rules returned unknown rule {code}"), + None, + ) + }) + }) + .collect() +} + +/// Reject non-hermetic imports in the source closure of every selected custom rule. +pub(crate) fn verify_custom_rules<'py>( + rules: &[Bound<'py, PyAny>], + project_dir: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = project_dir.py(); + let _ = call_with( + &imported(py, HERMETICITY_MODULE, "verify_custom_rules")?, + &[ + ("rules", PyTuple::new(py, rules)?.into_any()), + ("project_dir", project_dir.clone()), + ], + )?; + Ok(()) +} + +/// The validated built-in and custom rule catalogue sorted by code. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, include_custom = true))] +fn build_rule_catalogue<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + include_custom: bool, +) -> PyResult> { + PyTuple::new( + config.py(), + rule_catalogue(config, project_dir, include_custom)?, + ) +} + +/// The catalogue rules the configuration activates; `custom_payloads` describes custom rules. +#[pyfunction] +#[pyo3(signature = (*, catalogue, config, project_dir, custom_payloads = None))] +fn select_catalogue_rules<'py>( + catalogue: &Bound<'py, PyAny>, + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + custom_payloads: Option>, +) -> PyResult> { + let rules: Vec> = catalogue.try_iter()?.collect::>()?; + PyTuple::new( + config.py(), + select_rules(&rules, config, project_dir, custom_payloads)?, + ) +} + +fn resource_paths<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let paths: Bound<'py, PyDict> = PyDict::new(project.py()); + for (collection, file) in [ + ("models", None), + ("functions", None), + ("seeds", Some("seed_file")), + ("sources", Some("source_file")), + ] { + for resource in project.getattr(collection)?.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + let owner: Bound<'py, PyAny> = match file { + Some(file) => resource.getattr(file)?, + None => resource.clone(), + }; + paths.set_item(resource.getattr("key")?, owner.getattr("relative_path")?)?; + } + } + Ok(paths) +} + +/// `config` with every selector-scoped rule ignore resolved into deterministic finding paths. +pub(crate) fn resolved_ignore_selectors<'py>( + config: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = config.py(); + let ignores: Vec> = config + .getattr("rule_ignores")? + .try_iter()? + .collect::>()?; + let mut scoped: bool = false; + for ignore in &ignores { + scoped |= ignore.getattr("selectors")?.is_truthy()?; + } + if !scoped { + return Ok(config.clone()); + } + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let scope: Bound<'py, PyAny> = imported(py, PLANNER_SCOPE_MODULE, "build_planner_scope")?; + let path_by_key: Bound<'py, PyDict> = resource_paths(project)?; + let mut resolved: Vec> = Vec::with_capacity(ignores.len()); + for ignore in ignores { + let selectors: Bound<'py, PyAny> = ignore.getattr("selectors")?; + if !selectors.is_truthy()? { + resolved.push(ignore); + continue; + } + let selected: Bound<'py, PyAny> = call_with( + &scope, + &[("project", project.clone()), ("select", selectors.clone())], + )?; + let mut selector_paths: BTreeSet = BTreeSet::new(); + for key in selected.getattr("selected_keys")?.try_iter()? { + if let Some(path) = path_by_key.get_item(key?)? { + selector_paths.insert(path.call_method0("as_posix")?.extract()?); + } + } + if selector_paths.is_empty() { + let joined: Vec = selectors.extract()?; + return Err(rules_error( + py, + &format!( + "rule ignore selectors resolve no resources with paths: {}", + joined.join(", ") + ), + None, + )); + } + for path in ignore.getattr("paths")?.try_iter()? { + selector_paths.insert(path?.extract()?); + } + resolved.push(replace.call( + (ignore,), + Some(&keywords( + py, + &[ + ("paths", PyTuple::new(py, selector_paths)?.into_any()), + ("selectors", PyTuple::empty(py).into_any()), + ], + )?), + )?); + } + replace.call( + (config,), + Some(&keywords( + py, + &[("rule_ignores", PyTuple::new(py, resolved)?.into_any())], + )?), + ) +} + +/// `config` with selector-scoped rule ignores resolved against the compiled project. +#[pyfunction] +#[pyo3(signature = (*, config, project))] +fn resolve_rule_ignore_selectors<'py>( + config: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + resolved_ignore_selectors(config, project) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_rule_catalogue, module)?)?; + module.add_function(wrap_pyfunction!(select_catalogue_rules, module)?)?; + module.add_function(wrap_pyfunction!(resolve_rule_ignore_selectors, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index 922c6cede2..fe1ab554a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -64,7 +64,7 @@ fn custom_rule_import_closure<'py>( /// One rule's source file and helper closure fingerprint, or its check's source digest. #[pyfunction] #[pyo3(signature = (*, rule, project_dir, import_closure = None))] -fn custom_rule_implementation_fingerprint( +pub(crate) fn custom_rule_implementation_fingerprint( rule: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>, import_closure: Option<&Bound<'_, PyAny>>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs new file mode 100644 index 0000000000..b9a2663c78 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs @@ -0,0 +1,227 @@ +//! Rule inspection text and the generated rules skill, rendered and installed natively. + +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::constants::NATIVE_BUILD_IDENTITY; +use sqlbuild_rules::errors::{SkillInstallError, SkillInstallErrorKind}; +use sqlbuild_rules::models::RulesConfig; +use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; +use sqlbuild_rules::rule_catalogue::main::install_rendered_skills::install_rendered_skills; +use sqlbuild_rules::rule_catalogue::main::render_skills_body::render_skills_body; +use sqlbuild_rules::rule_catalogue::main::ruleset_fingerprint::ruleset_fingerprint; +use sqlbuild_rules::rule_catalogue::main::skill_lock_name::skill_lock_name; +use sqlbuild_rules::rule_catalogue::models::{FingerprintRule, OptionText, RuleText, SkillTarget}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::rules::catalogue::{ + native_rules_config, references_custom_rules, rule_catalogue, select_rules, verify_custom_rules, +}; +use crate::bindings::_helpers::rules::custom_rule_evidence::custom_rule_implementation_fingerprint; +use crate::bindings::_helpers::rules::selection::rules_error; +use crate::bindings::_helpers::rules::skills::{owned_skill, skill_state}; + +const FILELOCK_MODULE: &str = "filelock"; +const SKILL_PATHS: [&str; 3] = [ + ".agents/skills/sqlbuild-rules/SKILL.md", + ".claude/skills/sqlbuild-rules/SKILL.md", + ".opencode/skills/sqlbuild-rules/SKILL.md", +]; +const SKILL_LOCK_DIRECTORY: &str = "sqlbuild-rules-skill-locks"; +const FRESH_SKILL: &str = "fresh"; + +fn text_field(object: &Bound<'_, PyAny>, field: &str) -> PyResult { + object.getattr(field)?.extract() +} + +fn repr_text(value: &Bound<'_, PyAny>) -> PyResult { + value.repr()?.extract() +} + +/// The text view of one Python `Rule`, with effective options under `config` when given. +fn rule_text(rule: &Bound<'_, PyAny>, config: Option<&Bound<'_, PyAny>>) -> PyResult { + let code: String = text_field(rule, "code")?; + let configured: Option> = match config { + Some(config) => config + .getattr("rule_options")? + .cast_into::()? + .get_item(&code)?, + None => None, + }; + let mut options: Vec = Vec::new(); + for option in rule.getattr("options")?.try_iter()? { + let option: Bound<'_, PyAny> = option?; + let name: String = text_field(&option, "name")?; + let default: Bound<'_, PyAny> = option.getattr("default")?; + let effective: Bound<'_, PyAny> = match &configured { + Some(values) => values.call_method1("get", (&name, &default))?, + None => default.clone(), + }; + options.push(OptionText { + name, + description: text_field(&option, "description")?, + default_repr: repr_text(&default)?, + effective_repr: repr_text(&effective)?, + }); + } + let guidance: Bound<'_, PyAny> = rule.getattr("guidance")?; + Ok(RuleText { + code, + family: text_field(rule, "family")?, + slug: text_field(rule, "slug")?, + message: text_field(rule, "message")?, + remediation: text_field(rule, "remediation")?, + enabled_by_default: rule.getattr("enabled_by_default")?.is_truthy()?, + custom: rule.getattr("custom")?.is_truthy()?, + source: rule.getattr("source")?.extract()?, + options, + good_example: if guidance.is_none() { + None + } else { + Some(text_field(&guidance, "good_example")?) + }, + }) +} + +/// `sqb rules show` text for one rule, with effective thresholds when a config is given. +#[pyfunction] +#[pyo3(signature = (*, rule, config = None))] +fn format_rule(rule: &Bound<'_, PyAny>, config: Option>) -> PyResult { + let config: Option> = config.filter(|config| !config.is_none()); + let native: Option = config.as_ref().map(native_rules_config).transpose()?; + let text: RuleText = rule_text(rule, config.as_ref())?; + compiler_guard(|| Ok(format_rule_text(&text, native.as_ref()))) +} + +fn fingerprint_rule( + rule: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let custom: bool = rule.getattr("custom")?.is_truthy()?; + Ok(FingerprintRule { + code: text_field(rule, "code")?, + family: text_field(rule, "family")?, + slug: text_field(rule, "slug")?, + message: text_field(rule, "message")?, + remediation: text_field(rule, "remediation")?, + source: rule.getattr("source")?.extract()?, + implementation_hash: if custom { + custom_rule_implementation_fingerprint(rule, project_dir, None)? + } else { + NATIVE_BUILD_IDENTITY.to_owned() + }, + }) +} + +/// The generated skill content and the ruleset fingerprint it records. +fn render_skills( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult<(String, String)> { + let py: Python<'_> = config.py(); + let native: RulesConfig = native_rules_config(config)?; + let catalogue = rule_catalogue(config, project_dir, references_custom_rules(config)?)?; + let rules = select_rules(&catalogue, config, project_dir, None)?; + let mut custom_selected: bool = false; + for rule in &rules { + custom_selected |= rule.getattr("custom")?.is_truthy()?; + } + if custom_selected { + verify_custom_rules(&rules, project_dir)?; + } + let fingerprints: Vec = rules + .iter() + .map(|rule| fingerprint_rule(rule, project_dir)) + .collect::>()?; + let texts: Vec = rules + .iter() + .map(|rule| rule_text(rule, Some(config))) + .collect::>()?; + let (body, fingerprint): (String, Result) = compiler_guard(|| { + Ok(( + render_skills_body(&texts, &native), + ruleset_fingerprint(&fingerprints, &native), + )) + })?; + let fingerprint: String = fingerprint.map_err(|message| rules_error(py, &message, None))?; + Ok((owned_skill(py, &body, &fingerprint)?, fingerprint)) +} + +fn current_content(path: &Path) -> PyResult> { + if !path.is_file() { + return Ok(None); + } + std::fs::read_to_string(path).map(Some).map_err(PyErr::from) +} + +fn install_error(py: Python<'_>, error: SkillInstallError) -> PyErr { + let raised: PyErr = match error.kind { + SkillInstallErrorKind::Refused => rules_error(py, &error.message, None), + SkillInstallErrorKind::Io => PyOSError::new_err(error.message), + }; + for note in error.notes { + let _ = raised.value(py).call_method1("add_note", (note,)); + } + raised +} + +fn lock_path(project_dir: &Path) -> PyResult { + let canonical: String = project_dir.canonicalize()?.display().to_string(); + let canonical: String = if cfg!(windows) { + canonical.to_lowercase().replace('/', "\\") + } else { + canonical + }; + let directory: PathBuf = std::env::temp_dir().join(SKILL_LOCK_DIRECTORY); + std::fs::create_dir_all(&directory)?; + Ok(directory.join(skill_lock_name(&canonical))) +} + +/// Install every owned rules skill target, or with `check` report whether all are fresh. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, check))] +fn install_rules_skills( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + check: bool, +) -> PyResult { + let py: Python<'_> = config.py(); + let (content, fingerprint): (String, String) = render_skills(config, project_dir)?; + let root: PathBuf = project_dir.extract()?; + let targets = || -> PyResult> { + let mut targets: Vec = Vec::with_capacity(SKILL_PATHS.len()); + for relative in SKILL_PATHS { + let path: PathBuf = root.join(relative); + let state: String = skill_state(current_content(&path)?.as_deref(), &fingerprint); + targets.push(SkillTarget { path, state }); + } + Ok(targets) + }; + if check { + return Ok(targets()?.iter().all(|target| target.state == FRESH_SKILL)); + } + let lock: Bound<'_, PyAny> = + imported(py, FILELOCK_MODULE, "FileLock")?.call1((lock_path(&root)?,))?; + let _ = lock.call_method0("acquire")?; + let installed = (|| -> PyResult<()> { + let targets: Vec = targets()?; + compiler_guard(|| Ok(install_rendered_skills(&content, &targets)))? + .map_err(|error| install_error(py, error)) + })(); + let released = lock.call_method0("release"); + installed?; + let _ = released?; + Ok(true) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_rule, module)?)?; + module.add_function(wrap_pyfunction!(install_rules_skills, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 5bad746aca..ec1a2b0d2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,9 +1,11 @@ pub(crate) mod builtin_catalogue; +pub(crate) mod catalogue; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; pub(crate) mod evaluation; pub(crate) mod findings; +pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; pub(crate) mod lint_sql_bodies; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 1451384b04..0aca5e2a68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,9 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, - lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, - selection, skills, + builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, + guidance, lint, lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, + run_lint, selection, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -19,9 +19,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { rule_cache::register(module)?; custom_rule_cache::register(module)?; builtin_catalogue::register(module)?; + catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; selection::register(module)?; results::register(module)?; - skills::register(module) + guidance::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index f0c25105c0..fc819aaec2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -19,7 +19,7 @@ const BUILTINS_MODULE: &str = "builtins"; /// The selected built-in and custom rule codes; invalid configuration raises `RulesError`. #[pyfunction] #[pyo3(signature = (*, config, custom_payloads))] -fn selected_rule_codes( +pub(crate) fn selected_rule_codes( config: &Bound<'_, PyAny>, custom_payloads: &Bound<'_, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs index f6a103f885..3a6d23956f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/skills.rs @@ -1,7 +1,6 @@ -//! Render and check the generated rules skill. +//! The ownership header and freshness of the generated rules skill. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{PyResult, Python}; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::rules::selection::rules_error; @@ -10,9 +9,11 @@ const SKILL_OWNER: &str = "sqlbuild"; const SKILL_IDENTITY: &str = "sqlbuild-rules"; /// Generated guidance with its ownership header; a policy failure raises `RulesError`. -#[pyfunction] -#[pyo3(signature = (*, content, input_fingerprint))] -fn render_owned_skill(py: Python<'_>, content: &str, input_fingerprint: &str) -> PyResult { +pub(crate) fn owned_skill( + py: Python<'_>, + content: &str, + input_fingerprint: &str, +) -> PyResult { let rendered: Result = compiler_guard(|| { Ok(fensu_policy::render_owned_skill( SKILL_OWNER, @@ -26,10 +27,8 @@ fn render_owned_skill(py: Python<'_>, content: &str, input_fingerprint: &str) -> rendered.map_err(|message| rules_error(py, &message, None)) } -/// Whether generated guidance is current, stale or foreign under the ownership contract. -#[pyfunction] -#[pyo3(signature = (*, content, input_fingerprint))] -fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { +/// Whether generated guidance is current, stale, missing or foreign under the ownership contract. +pub(crate) fn skill_state(content: Option<&str>, input_fingerprint: &str) -> String { let freshness = fensu_policy::skill_freshness( content.map(str::as_bytes), SKILL_OWNER, @@ -38,9 +37,3 @@ fn skill_freshness(content: Option<&str>, input_fingerprint: &str) -> String { ); format!("{freshness:?}").to_lowercase() } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; - module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-rules/src/errors.rs b/crates/sqlbuild-rules/src/errors.rs index 98cae0b073..83fd7ca87e 100644 --- a/crates/sqlbuild-rules/src/errors.rs +++ b/crates/sqlbuild-rules/src/errors.rs @@ -4,3 +4,18 @@ pub enum RowsError { Rules(String), Io(std::io::Error), } + +/// Why generated skills were not installed, with rollback notes for the raised error. +#[derive(Debug)] +pub struct SkillInstallError { + pub kind: SkillInstallErrorKind, + pub message: String, + pub notes: Vec, +} + +/// Whether a skill installation failure is a policy refusal or a filesystem error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkillInstallErrorKind { + Refused, + Io, +} diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index ae2bbcee68..3b9babee52 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -11,6 +11,7 @@ pub mod errors; pub mod lint_text; pub mod models; pub mod rule_cache; +pub mod rule_catalogue; pub mod rule_results; pub mod rules; pub mod scope_metadata; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs new file mode 100644 index 0000000000..3910a88da1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod options; +pub(crate) mod python_repr; +pub(crate) mod staging; +pub(crate) mod thresholds; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs new file mode 100644 index 0000000000..9f91b818b8 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/options.rs @@ -0,0 +1,46 @@ +//! Python `isinstance` and equality semantics over option values. + +use crate::rule_catalogue::constants::OBJECT_TYPE; +use crate::rule_catalogue::models::{OptionKind, OptionValue}; + +/// `isinstance(value, kind)`, where `bool` is a subclass of `int`. +pub(crate) fn is_instance(value: &OptionValue, kind: &OptionKind) -> bool { + match (kind, value) { + (OptionKind::Bool, OptionValue::Bool(_)) + | (OptionKind::Int, OptionValue::Int(_) | OptionValue::Bool(_)) + | (OptionKind::Str, OptionValue::Str(_)) + | (OptionKind::Tuple, OptionValue::Tuple(_)) => true, + (OptionKind::Other(name), _) if name == OBJECT_TYPE => true, + (OptionKind::Other(name), OptionValue::Other(type_name)) => name == type_name, + _ => false, + } +} + +/// Whether `value` has the declared type, where a bool is not an `int` value. +pub(crate) fn has_value_type(value: &OptionValue, kind: &OptionKind) -> bool { + is_instance(value, kind) + && !(matches!(kind, OptionKind::Int) && matches!(value, OptionValue::Bool(_))) +} + +/// Python `==` between two option values, where `True == 1`. +pub(crate) fn python_equal(left: &OptionValue, right: &OptionValue) -> bool { + match (left, right) { + (OptionValue::Bool(a), OptionValue::Bool(b)) => a == b, + (OptionValue::Int(a), OptionValue::Int(b)) => a == b, + (OptionValue::Bool(a), OptionValue::Int(b)) + | (OptionValue::Int(b), OptionValue::Bool(a)) => i64::from(*a) == *b, + (OptionValue::Str(a), OptionValue::Str(b)) => a == b, + (OptionValue::Tuple(a), OptionValue::Tuple(b)) => { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| python_equal(x, y)) + } + _ => false, + } +} + +/// The integer value Python's `isinstance(value, int) and not isinstance(value, bool)` admits. +pub(crate) fn plain_integer(value: &OptionValue) -> Option { + match value { + OptionValue::Int(integer) => Some(*integer), + _ => None, + } +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs new file mode 100644 index 0000000000..3739290e1f --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/python_repr.rs @@ -0,0 +1,41 @@ +//! Python `repr` text of the strings and dictionaries guidance prints. + +use crate::rule_catalogue::constants::{DELETE_CHARACTER, FIRST_PRINTABLE}; + +/// `repr(text)` for a Python `str`. +pub(crate) fn string_repr(text: &str) -> String { + let quote: char = if text.contains('\'') && !text.contains('"') { + '"' + } else { + '\'' + }; + let mut rendered: String = String::with_capacity(text.len() + 2); + rendered.push(quote); + for character in text.chars() { + match character { + '\\' => rendered.push_str("\\\\"), + '\n' => rendered.push_str("\\n"), + '\r' => rendered.push_str("\\r"), + '\t' => rendered.push_str("\\t"), + _ if character == quote => { + rendered.push('\\'); + rendered.push(character); + } + _ if (character as u32) < FIRST_PRINTABLE || character as u32 == DELETE_CHARACTER => { + rendered.push_str(&format!("\\x{:02x}", character as u32)); + } + _ => rendered.push(character), + } + } + rendered.push(quote); + rendered +} + +/// `repr(dict)` of string keys and already-rendered value texts, in the given order. +pub(crate) fn dict_repr<'a>(entries: impl IntoIterator) -> String { + let parts: Vec = entries + .into_iter() + .map(|(key, value)| format!("{}: {value}", string_repr(key))) + .collect(); + format!("{{{}}}", parts.join(", ")) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs new file mode 100644 index 0000000000..2610ae6a27 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/staging.rs @@ -0,0 +1,221 @@ +//! Staged temporary files and backups of generated skill targets, with rollback. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::errors::{SkillInstallError, SkillInstallErrorKind}; + +const DEFAULT_MODE: u32 = 0o644; + +/// One staged replacement: the temporary content, the backup of the old target, the target. +pub(crate) struct Staged { + pub(crate) temporary: PathBuf, + pub(crate) backup: Option, + pub(crate) target: PathBuf, +} + +pub(crate) fn io_error(error: &std::io::Error) -> SkillInstallError { + SkillInstallError { + kind: SkillInstallErrorKind::Io, + message: error.to_string(), + notes: Vec::new(), + } +} + +/// Create a new hidden sibling of `target` holding `content`, and return its path. +fn write_sibling( + target: &Path, + suffix: &str, + content: &[u8], + mode: u32, +) -> std::io::Result { + let name: String = target + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let nanos: u128 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_nanos()) + .unwrap_or_default(); + let mut attempt: u32 = 0; + loop { + let path: PathBuf = target.with_file_name(format!( + ".{name}.{}-{nanos}-{attempt}{suffix}", + std::process::id() + )); + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(mut file) => { + file.write_all(content)?; + set_mode(&path, mode)?; + return Ok(path); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => attempt += 1, + Err(error) => return Err(error), + } + } +} + +#[cfg(unix)] +fn file_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path) + .map(|metadata| metadata.permissions().mode() & 0o7777) + .unwrap_or(DEFAULT_MODE) +} + +#[cfg(not(unix))] +fn file_mode(_path: &Path) -> u32 { + DEFAULT_MODE +} + +#[cfg(unix)] +fn set_mode(path: &Path, mode: u32) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(mode)) +} + +#[cfg(not(unix))] +fn set_mode(_path: &Path, _mode: u32) -> std::io::Result<()> { + Ok(()) +} + +fn read_if_file(path: &Path) -> std::io::Result>> { + if path.is_file() { + return fs::read(path).map(Some); + } + Ok(None) +} + +/// A temporary copy of `content` beside each target and a backup of each existing target. +pub(crate) fn stage(targets: &[PathBuf], content: &[u8]) -> Result, SkillInstallError> { + let mut staged: Vec = Vec::with_capacity(targets.len()); + let staging = |staged: &mut Vec| -> std::io::Result<()> { + for target in targets { + if let Some(parent) = target.parent() { + fs::create_dir_all(parent)?; + } + let mode: u32 = if target.is_file() { + file_mode(target) + } else { + DEFAULT_MODE + }; + let temporary: PathBuf = write_sibling(target, ".tmp", content, mode)?; + staged.push(Staged { + temporary, + backup: None, + target: target.clone(), + }); + if target.is_file() { + let backup: PathBuf = write_sibling(target, ".bak", &fs::read(target)?, mode)?; + if let Some(last) = staged.last_mut() { + last.backup = Some(backup); + } + } + } + Ok(()) + }; + if let Err(error) = staging(&mut staged) { + remove_staged(&staged, &[]); + return Err(io_error(&error)); + } + Ok(staged) +} + +/// Replace each target with its staged content unless it changed since staging. +pub(crate) fn apply(staged: &[Staged], installed: &[u8]) -> Result<(), SkillInstallError> { + let mut replaced: Vec<(Option<&Path>, &Path)> = Vec::new(); + let mut outcome: Result<(), SkillInstallError> = Ok(()); + for entry in staged { + let replacing = || -> Result<(), SkillInstallError> { + let current: Option> = + read_if_file(&entry.target).map_err(|error| io_error(&error))?; + let expected: Option> = match &entry.backup { + Some(backup) => Some(fs::read(backup).map_err(|error| io_error(&error))?), + None => None, + }; + if current != expected { + return Err(SkillInstallError { + kind: SkillInstallErrorKind::Refused, + message: format!( + "refusing to overwrite rules skill changed during installation: {}", + entry.target.display() + ), + notes: Vec::new(), + }); + } + fs::rename(&entry.temporary, &entry.target).map_err(|error| io_error(&error)) + }; + if let Err(error) = replacing() { + outcome = Err(error); + break; + } + replaced.push((entry.backup.as_deref(), &entry.target)); + } + let mut retained: Vec = Vec::new(); + if let Err(error) = &mut outcome { + let (kept, notes) = rollback(&replaced, installed); + retained = kept; + error.notes.extend(notes); + } + remove_staged(staged, &retained); + outcome +} + +/// Restore replaced targets; returns the backups kept for recovery and the rollback notes. +fn rollback(replaced: &[(Option<&Path>, &Path)], installed: &[u8]) -> (Vec, Vec) { + let mut retained: Vec = Vec::new(); + let mut notes: Vec = Vec::new(); + let backup_text = |backup: Option<&Path>| { + backup.map_or_else(|| "None".to_owned(), |path| path.display().to_string()) + }; + for (backup, target) in replaced.iter().rev() { + let restoring = || -> std::io::Result { + if read_if_file(target)?.as_deref() != Some(installed) { + return Ok(false); + } + match backup { + None => match fs::remove_file(target) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(error), + _ => Ok(true), + }, + Some(backup) => fs::rename(backup, target).map(|()| true), + } + }; + match restoring() { + Ok(true) => {} + Ok(false) => { + retained.extend(backup.map(Path::to_path_buf)); + notes.push(format!( + "rollback preserved concurrently changed target {}; recovery backup: {}", + target.display(), + backup_text(*backup) + )); + } + Err(error) => { + retained.extend(backup.map(Path::to_path_buf)); + notes.push(format!( + "rollback failed for {}: {error}; recovery backup: {}", + target.display(), + backup_text(*backup) + )); + } + } + } + (retained, notes) +} + +fn remove_staged(staged: &[Staged], retained: &[PathBuf]) { + for entry in staged { + let _ = fs::remove_file(&entry.temporary); + if let Some(backup) = &entry.backup + && !retained.contains(backup) + { + let _ = fs::remove_file(backup); + } + } +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs new file mode 100644 index 0000000000..16b4bdd7fb --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/_helpers/thresholds.rs @@ -0,0 +1,37 @@ +//! Effective rule test thresholds rendered as guidance lines. + +use std::collections::BTreeMap; + +use crate::models::RulesConfig; +use crate::rule_catalogue::_helpers::python_repr::dict_repr; +use crate::rule_catalogue::constants::THRESHOLD_DEFAULTS; + +/// Global and path-scoped thresholds in deterministic authored order. +pub(crate) fn threshold_lines(config: &RulesConfig) -> Vec { + let mut thresholds: BTreeMap<&str, u32> = THRESHOLD_DEFAULTS.into_iter().collect(); + for (name, value) in &config.thresholds { + thresholds.insert(name, *value); + } + let mut lines: Vec = thresholds + .iter() + .map(|(name, value)| format!("- `{name}` = `{value}`")) + .collect(); + for (index, entry) in config.threshold_overrides.iter().enumerate() { + let values: String = dict_repr( + entry + .thresholds + .iter() + .map(|(name, value)| (name.as_str(), value.to_string())), + ); + lines.push(format!( + "- Override {} at `{}`: `{values}` ({})", + index + 1, + entry.paths.join(","), + entry.reason + )); + } + if !config.threshold_overrides.is_empty() { + lines.push("- Matching overrides apply in authored order; the last match wins.".into()); + } + lines +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs new file mode 100644 index 0000000000..927e472f7c --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -0,0 +1,135 @@ +//! Threshold defaults, guidance wording and the rule codes guidance sections depend on. + +/// Effective threshold defaults, sorted by name as guidance renders them. +pub(crate) const THRESHOLD_DEFAULTS: [(&str, u32); 10] = [ + ("max_constant_container_files", 10), + ("max_enum_container_files", 10), + ("max_macro_container_files", 10), + ("max_role_container_depth", 1), + ("max_subdomain_depth", 1), + ("min_audits_per_model", 1), + ("min_custom_rule_test_cases", 1), + ("min_shared_container_prefix_files", 2), + ("min_shared_owner_prefix_directories", 2), + ("min_tests_per_model", 1), +]; +/// The Python type every option value is an instance of. +pub(crate) const OBJECT_TYPE: &str = "object"; +/// The first printable code point; Python `repr` escapes control characters below it. +pub(crate) const FIRST_PRINTABLE: u32 = 0x20; +/// The DEL control character, which Python `repr` escapes. +pub(crate) const DELETE_CHARACTER: u32 = 0x7f; +pub(crate) const MINIMUM_RULES_CODES: [&str; 2] = ["SQBRTEST201", "SQBRTEST202"]; +pub(crate) const LAYOUT_THRESHOLD_RULE_CODES: [&str; 5] = [ + "SQBRPROJECT203", + "SQBRPROJECT204", + "SQBRDECLARATION303", + "SQBRDECLARATION304", + "SQBRDECLARATION306", +]; +pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARATION3"]; +pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; +pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; +pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; +pub(crate) const RULE_EXAMPLES: [(&str, &str); 25] = [ + ( + "SQBRSQL040", + "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", + ), + ( + "SQBRSQL041", + "WITH orders AS (...), final AS (SELECT order_id FROM orders) SELECT order_id FROM final.", + ), + ( + "SQBRMODEL101", + "WITH upstream AS (SELECT * FROM __ref(\"domain__stg__entity\")), ...", + ), + ( + "SQBRMODEL102", + "SELECT id, status FROM final; use a lone SELECT * only when allowed.", + ), + ( + "SQBRMODEL103", + "A view uses stg_v, int_v, or mart_v; a table uses a non-v layer.", + ), + ( + "SQBRMODEL104", + "MODEL() omits schema; central path defaults provide it.", + ), + ( + "SQBRGRAPH101", + "Internal flow is stg -> int_clean -> int_enriched; marts are published interfaces.", + ), + ( + "SQBRPROJECT101", + "domain__int_clean__entity or domain__mart_v__entity.", + ), + ( + "SQBRPROJECT105", + "A domain__int_enriched__entity model lives under intermediate/enriched.", + ), + ( + "SQBRPROJECT106", + "A domain__stg__entity model resolves to the staging logical schema.", + ), + ( + "SQBRCONTRACT101", + "MODEL (contract enforced, columns (...)); declare authoritative columns.", + ), + ( + "SQBRPROJECT202", + "domain/level/owner contains models or child owners, never both.", + ), + ( + "SQBRPROJECT201", + "models///model.sql resolves one domain root.", + ), + ( + "SQBRPROJECT203", + "domain/level/subdomain/model.sql at the default depth of one.", + ), + ( + "SQBRPROJECT204", + "order_status and order_status_history consolidate under order_status.", + ), + ( + "SQBRDECLARATION302", + "_macros/ is flat, or every file uses one concern bucket.", + ), + ( + "SQBRDECLARATION303", + "_macros/normalisation/names.py uses one default bucket level.", + ), + ( + "SQBRDECLARATION304", + "Keep each flat role or concern bucket within its file cap.", + ), + ( + "SQBRDECLARATION305", + "Use temporal/ or scoring/, not utils/ or common/.", + ), + ( + "SQBRDECLARATION306", + "Group normalise_order.py and normalise_person.py under normalise/.", + ), + ( + "SQBRTEST101", + "Keep unit tests in tests/unit/ and scenarios in tests/scenarios/.", + ), + ( + "SQBRTEST102", + "test_stg_orders__excludes_cancelled.sql and daily_revenue__minimal.sql.", + ), + ( + "SQBRTEST103", + "models/staging/stg_orders.sql maps to tests/unit/staging/.", + ), + ( + "SQBRTEST104", + "TEST (name \"stg_orders__excludes_cancelled_orders\");", + ), + ( + "SQBRTEST201", + "Attach not_null/unique audits to the model key.", + ), +]; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs new file mode 100644 index 0000000000..96b6b05ca4 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/format_rule_text.rs @@ -0,0 +1,54 @@ +//! The inspection text of one rule. + +use crate::models::RulesConfig; +use crate::rule_catalogue::_helpers::thresholds::threshold_lines; +use crate::rule_catalogue::constants::{MINIMUM_RULES_CODES, TEST_RULE_PREFIX}; +use crate::rule_catalogue::models::RuleText; + +/// `sqb rules show` text, with effective thresholds and test paths when a config is given. +pub fn format_rule_text(rule: &RuleText, config: Option<&RulesConfig>) -> String { + let mut lines: Vec = vec![ + format!("{}: {}", rule.code, rule.slug), + format!("Family: {}", rule.family), + format!( + "Enabled by default: {}", + if rule.enabled_by_default { "yes" } else { "no" } + ), + format!("Kind: {}", if rule.custom { "custom" } else { "built-in" }), + String::new(), + rule.message.clone(), + String::new(), + format!("Remediation: {}", rule.remediation), + ]; + if let Some(source) = &rule.source { + lines.extend([String::new(), format!("Source: {source}")]); + } + if !rule.options.is_empty() { + lines.extend([String::new(), "Options:".into()]); + for option in &rule.options { + lines.push(format!( + "- {}: default={} ({})", + option.name, option.default_repr, option.description + )); + } + } + if let Some(config) = config { + if MINIMUM_RULES_CODES.contains(&rule.code.as_str()) { + lines.extend([String::new(), "Effective thresholds:".into()]); + lines.extend(threshold_lines(config)); + } + if rule.code.starts_with(TEST_RULE_PREFIX) { + lines.extend([ + String::new(), + "Effective SQL test paths:".into(), + "- Unit tests: tests/unit/".into(), + "- Scenarios: tests/scenarios/".into(), + format!( + "- Cross-domain pipelines: tests/unit/{}/", + config.sql_tests.pipeline_directory + ), + ]); + } + } + lines.join("\n") +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs new file mode 100644 index 0000000000..7ffa96add2 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/install_rendered_skills.rs @@ -0,0 +1,29 @@ +//! Validate and atomically replace every generated skill target. + +use crate::errors::{SkillInstallError, SkillInstallErrorKind}; +use crate::rule_catalogue::_helpers::staging::{apply, stage}; +use crate::rule_catalogue::constants::REPLACEABLE_SKILL_STATES; +use crate::rule_catalogue::models::SkillTarget; + +/// Install `content` at every target, refusing foreign content and rolling back on failure. +pub fn install_rendered_skills( + content: &str, + targets: &[SkillTarget], +) -> Result<(), SkillInstallError> { + for target in targets { + if !REPLACEABLE_SKILL_STATES.contains(&target.state.as_str()) { + return Err(SkillInstallError { + kind: SkillInstallErrorKind::Refused, + message: format!( + "refusing to overwrite {} rules skill: {}", + target.state, + target.path.display() + ), + notes: Vec::new(), + }); + } + } + let paths: Vec = targets.iter().map(|target| target.path.clone()).collect(); + let staged = stage(&paths, content.as_bytes())?; + apply(&staged, content.as_bytes()) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs new file mode 100644 index 0000000000..618095dd7e --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -0,0 +1,6 @@ +pub mod format_rule_text; +pub mod install_rendered_skills; +pub mod render_skills_body; +pub mod ruleset_fingerprint; +pub mod skill_lock_name; +pub mod sorted_catalogue_order; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs new file mode 100644 index 0000000000..2a44c33eb3 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs @@ -0,0 +1,155 @@ +//! The generated rules skill body for the selected rules. + +use crate::models::RulesConfig; +use crate::rule_catalogue::_helpers::python_repr::dict_repr; +use crate::rule_catalogue::_helpers::thresholds::threshold_lines; +use crate::rule_catalogue::constants::{ + GRAPH_EDGE_RULE_CODE, LAYOUT_RULE_PREFIXES, LAYOUT_THRESHOLD_RULE_CODES, MINIMUM_RULES_CODES, + RULE_EXAMPLES, TEST_RULE_PREFIX, +}; +use crate::rule_catalogue::models::RuleText; + +/// Stable guidance text for `rules` under `config`, ending in one newline. +pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { + let mut body: Vec = [ + "# SQLBuild Rules", + "", + "Rules enforce SQL model architecture and never modifies SQL files.", + "Run `sqb rules run SQBR` before handing off model changes.", + "", + "## Active Rules", + ] + .map(String::from) + .to_vec(); + for rule in rules { + body.extend([ + String::new(), + format!("### {}: {}", rule.code, rule.slug), + String::new(), + rule.message.clone(), + String::new(), + format!("Correct shape: {}", rule_example(rule)), + String::new(), + format!("Remediation: {}", rule.remediation), + ]); + if !rule.options.is_empty() { + let effective: String = dict_repr( + rule.options + .iter() + .map(|option| (option.name.as_str(), option.effective_repr.clone())), + ); + body.extend([String::new(), format!("Effective options: {effective}")]); + } + } + if !config.rule_exceptions.is_empty() + || !config.graph_edge_exceptions.is_empty() + || !config.rule_ignores.is_empty() + || !config.select_star_allow.is_empty() + { + body.extend([String::new(), "## Scoped Deviations".into(), String::new()]); + for entry in &config.rule_exceptions { + body.push(format!( + "- Exception `{}` at `{}`: {}", + entry.rule, entry.path, entry.reason + )); + } + for entry in &config.rule_ignores { + let scopes: Vec = entry + .paths + .iter() + .map(|path| format!("path:{path}")) + .chain( + entry + .selectors + .iter() + .map(|selector| format!("selector:{selector}")), + ) + .collect(); + body.push(format!( + "- Ignore `{}` at `{}`: {}", + entry.rules.join(","), + scopes.join(","), + entry.reason + )); + } + for entry in &config.graph_edge_exceptions { + body.push(format!( + "- `{GRAPH_EDGE_RULE_CODE}` exact edge `{}` -> `{}`: {}", + entry.consumer, entry.dependency, entry.reason + )); + } + for entry in &config.select_star_allow { + body.push(format!( + "- Lone-star allowance `{}`: {}", + entry.paths.join(","), + entry.reason + )); + } + } + let any_rule = |matches: &dyn Fn(&str) -> bool| rules.iter().any(|rule| matches(&rule.code)); + if any_rule(&|code| { + MINIMUM_RULES_CODES.contains(&code) || LAYOUT_THRESHOLD_RULE_CODES.contains(&code) + }) { + body.extend([ + String::new(), + "## Effective Thresholds".into(), + String::new(), + ]); + body.extend(threshold_lines(config)); + } + if any_rule(&|code| { + LAYOUT_RULE_PREFIXES + .iter() + .any(|prefix| code.starts_with(prefix)) + }) { + body.extend([String::new(), "## Owner Layout".into(), String::new()]); + body.push(format!( + "- Configured levels: `{}`", + config.layout.levels.join(", ") + )); + body.push("- Every model owner is a leaf or a branch, never both.".into()); + body.push("- Declaration-role buckets organize files without changing visibility.".into()); + } + if any_rule(&|code| code.starts_with(TEST_RULE_PREFIX)) { + body.extend([String::new(), "## SQL Test Paths".into(), String::new()]); + body.push("- Unit tests: `tests/unit/`".into()); + body.push("- Scenarios: `tests/scenarios/`".into()); + body.push(format!( + "- Cross-domain pipelines: `tests/unit/{}/`", + config.sql_tests.pipeline_directory + )); + } + if !config.domains.is_empty() + || !config.approved_source_tokens.is_empty() + || !config.retired_source_tokens.is_empty() + { + body.extend([String::new(), "## Naming Vocabulary".into(), String::new()]); + body.push(format!("- Domains: `{}`", config.domains.join(", "))); + body.push(format!( + "- Approved source tokens: `{}`", + config.approved_source_tokens.join(", ") + )); + for (retired, replacement) in &config.retired_source_tokens { + body.push(format!( + "- Replace retired `{retired}` with `{replacement}`" + )); + } + } + format!("{}\n", body.join("\n").trim_end()) +} + +fn rule_example(rule: &RuleText) -> String { + if let Some(example) = &rule.good_example { + return example.clone(); + } + RULE_EXAMPLES + .iter() + .find(|(code, _)| *code == rule.code) + .map(|(_, example)| (*example).to_owned()) + .unwrap_or_else(|| { + format!( + "Apply `{}` at the SQL node named by the finding.", + rule.slug + ) + }) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs new file mode 100644 index 0000000000..4a20a55fe4 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/ruleset_fingerprint.rs @@ -0,0 +1,55 @@ +//! The fingerprint of one resolved ruleset that generated guidance records. + +use std::collections::BTreeMap; + +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use crate::constants::NATIVE_BUILD_IDENTITY; +use crate::models::RulesConfig; +use crate::rule_catalogue::models::FingerprintRule; + +/// SHA-256 over the engine build, the selected rules, their sources and the guidance config. +pub fn ruleset_fingerprint( + rules: &[FingerprintRule], + config: &RulesConfig, +) -> Result { + let mut payload: Vec = Vec::with_capacity(rules.len()); + for rule in rules { + let source_hash: Option = match &rule.source { + Some(source) => Some(format!( + "{:x}", + Sha256::digest(std::fs::read(source).map_err(|error| error.to_string())?) + )), + None => None, + }; + payload.push(json!({ + "code": rule.code, + "family": rule.family, + "slug": rule.slug, + "message": rule.message, + "remediation": rule.remediation, + "options": config.rule_options.get(&rule.code).cloned().unwrap_or_else(BTreeMap::new), + "source_hash": source_hash, + "implementation_hash": rule.implementation_hash, + })); + } + let rules_payload: Value = json!({ + "engine": NATIVE_BUILD_IDENTITY, + "rules": payload, + "thresholds": config.thresholds, + "threshold_overrides": config.threshold_overrides, + "select_star_allow": config.select_star_allow, + "domains": config.domains, + "approved_source_tokens": config.approved_source_tokens, + "retired_source_tokens": config.retired_source_tokens, + "sql_tests": config.sql_tests, + "layout": config.layout, + "exceptions": config.rule_exceptions, + "graph_edge_exceptions": config.graph_edge_exceptions, + "ignores": config.rule_ignores, + }); + let encoded: String = + serde_json::to_string(&rules_payload).map_err(|error| error.to_string())?; + Ok(format!("{:x}", Sha256::digest(encoded.as_bytes()))) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs new file mode 100644 index 0000000000..faa2aff130 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/skill_lock_name.rs @@ -0,0 +1,8 @@ +//! The per-project lock file name that serializes generated skill installation. + +use sha2::{Digest, Sha256}; + +/// `.lock`. +pub fn skill_lock_name(canonical_project: &str) -> String { + format!("{:x}.lock", Sha256::digest(canonical_project.as_bytes())) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs new file mode 100644 index 0000000000..c3cdd874af --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/sorted_catalogue_order.rs @@ -0,0 +1,157 @@ +//! The code order of a validated rule catalogue and its configured options. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::rule_catalogue::_helpers::options::{ + has_value_type, is_instance, plain_integer, python_equal, +}; +use crate::rule_catalogue::models::{CatalogueRule, OptionSpec, OptionValue}; + +/// Indices of `rules` sorted by code, or the first `RulesError` message the catalogue raises. +pub fn sorted_catalogue_order( + rules: &[CatalogueRule], + configured: &BTreeMap>, +) -> Result, String> { + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for rule in rules { + *counts.entry(rule.code.as_str()).or_default() += 1; + } + let duplicates: Vec<&str> = counts + .iter() + .filter(|(_, count)| **count > 1) + .map(|(code, _)| *code) + .collect(); + if !duplicates.is_empty() { + return Err(format!("duplicate rule codes: {}", duplicates.join(", "))); + } + let empty: BTreeMap = BTreeMap::new(); + for rule in rules { + validate_options(rule, configured.get(&rule.code).unwrap_or(&empty))?; + } + let unknown: Vec<&str> = configured + .keys() + .map(String::as_str) + .filter(|code| !counts.contains_key(code)) + .collect(); + if !unknown.is_empty() { + return Err(format!( + "rule options target unknown codes: {}", + unknown.join(", ") + )); + } + let mut order: Vec = (0..rules.len()).collect(); + order.sort_by(|left, right| rules[*left].code.cmp(&rules[*right].code)); + Ok(order) +} + +fn validate_options( + rule: &CatalogueRule, + configured: &BTreeMap, +) -> Result<(), String> { + let code: &str = &rule.code; + for option in &rule.options { + validate_declaration(code, option)?; + } + let declared: BTreeMap<&str, &OptionSpec> = rule + .options + .iter() + .map(|option| (option.name.as_str(), option)) + .collect(); + let unknown: BTreeSet<&str> = configured + .keys() + .map(String::as_str) + .filter(|name| !declared.contains_key(name)) + .collect(); + if !unknown.is_empty() { + let names: Vec<&str> = unknown.into_iter().collect(); + return Err(format!( + "rule {code} has unknown options: {}", + names.join(", ") + )); + } + for (name, value) in configured { + let Some(option) = declared.get(name.as_str()) else { + continue; + }; + if !has_value_type(value, &option.value_type) { + return Err(format!("rule {code} option {name} has the wrong type")); + } + if !option.choices.is_empty() && !option.choices.iter().any(|c| python_equal(c, value)) { + return Err(format!( + "rule {code} option {name} is not an allowed choice" + )); + } + if let OptionValue::Tuple(items) = value { + if option + .minimum_items + .is_some_and(|minimum| (items.len() as i64) < minimum) + { + return Err(format!("rule {code} option {name} has too few items")); + } + if let Some(item_type) = &option.item_type + && items.iter().any(|item| !is_instance(item, item_type)) + { + return Err(format!( + "rule {code} option {name} has item values of the wrong type" + )); + } + } + if let Some(integer) = plain_integer(value) { + if option.minimum.is_some_and(|minimum| integer < minimum) { + return Err(format!("rule {code} option {name} is below its minimum")); + } + if option.maximum.is_some_and(|maximum| integer > maximum) { + return Err(format!("rule {code} option {name} exceeds its maximum")); + } + } + } + Ok(()) +} + +fn validate_declaration(code: &str, option: &OptionSpec) -> Result<(), String> { + let name: &str = &option.name; + if option.name.trim().is_empty() || option.description.trim().is_empty() { + return Err(format!("rule {code} declares an incomplete option")); + } + let default: &OptionValue = &option.default; + if !has_value_type(default, &option.value_type) { + return Err(format!( + "rule {code} option {name} has a default of the wrong type" + )); + } + if !option.choices.is_empty() && !option.choices.iter().any(|c| python_equal(c, default)) { + return Err(format!( + "rule {code} option {name} default is not an allowed choice" + )); + } + if let OptionValue::Tuple(items) = default { + if option + .minimum_items + .is_some_and(|minimum| (items.len() as i64) < minimum) + { + return Err(format!( + "rule {code} option {name} default has too few items" + )); + } + if let Some(item_type) = &option.item_type + && items.iter().any(|item| !is_instance(item, item_type)) + { + return Err(format!( + "rule {code} option {name} default has items of the wrong type" + )); + } + } + if let Some(integer) = plain_integer(default) { + if option.minimum.is_some_and(|minimum| integer < minimum) { + return Err(format!( + "rule {code} option {name} default is below its minimum" + )); + } + if option.maximum.is_some_and(|maximum| integer > maximum) { + return Err(format!( + "rule {code} option {name} default exceeds its maximum" + )); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/mod.rs new file mode 100644 index 0000000000..297276f558 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/mod.rs @@ -0,0 +1,9 @@ +//! Rule catalogue validation, rule inspection text, generated guidance and its installation. + +pub(crate) mod _helpers; +mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/models.rs b/crates/sqlbuild-rules/src/rule_catalogue/models.rs new file mode 100644 index 0000000000..2ebd66fc1b --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/models.rs @@ -0,0 +1,87 @@ +//! Typed views of catalogue rules, their declared options and the rule text guidance renders. + +/// The Python type an option declares for its value or its tuple items. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OptionKind { + Bool, + Int, + Str, + Tuple, + /// Any other Python type, by name; `object` accepts every value. + Other(String), +} + +/// A declared or configured option value with Python's scalar and tuple distinctions. +#[derive(Clone, Debug)] +pub enum OptionValue { + Bool(bool), + Int(i64), + Str(String), + Tuple(Vec), + /// A value of another Python type, by type name. + Other(String), +} + +/// One option a rule declares. +#[derive(Clone, Debug)] +pub struct OptionSpec { + pub name: String, + pub description: String, + pub value_type: OptionKind, + pub default: OptionValue, + pub choices: Vec, + pub minimum: Option, + pub maximum: Option, + pub minimum_items: Option, + pub item_type: Option, +} + +/// The code and declared options of one catalogue rule. +#[derive(Clone, Debug)] +pub struct CatalogueRule { + pub code: String, + pub options: Vec, +} + +/// One declared option as inspection and guidance show it, with Python `repr` texts. +#[derive(Clone, Debug, Default)] +pub struct OptionText { + pub name: String, + pub description: String, + pub default_repr: String, + pub effective_repr: String, +} + +/// The metadata of one rule that inspection text and guidance render. +#[derive(Clone, Debug, Default)] +pub struct RuleText { + pub code: String, + pub family: String, + pub slug: String, + pub message: String, + pub remediation: String, + pub enabled_by_default: bool, + pub custom: bool, + pub source: Option, + pub options: Vec, + pub good_example: Option, +} + +/// The inputs of one selected rule that the ruleset fingerprint covers. +#[derive(Clone, Debug, Default)] +pub struct FingerprintRule { + pub code: String, + pub family: String, + pub slug: String, + pub message: String, + pub remediation: String, + pub source: Option, + pub implementation_hash: String, +} + +/// One generated skill target and the ownership state of its current content. +#[derive(Clone, Debug)] +pub struct SkillTarget { + pub path: std::path::PathBuf, + pub state: String, +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests.rs new file mode 100644 index 0000000000..949a29ce14 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_rule_catalogue.rs"] +mod rule_catalogue; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs new file mode 100644 index 0000000000..7af1f8c6df --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests/helpers.rs @@ -0,0 +1,101 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use crate::models::{RulesConfig, ThresholdOverride}; +use crate::rule_catalogue::models::{ + CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleText, SkillTarget, +}; +use crate::rule_catalogue::tests::test_types::InstallTestCase; + +pub(crate) fn integer_option(default: i64) -> OptionSpec { + OptionSpec { + name: "limit".into(), + description: "The limit.".into(), + value_type: OptionKind::Int, + default: OptionValue::Int(default), + choices: vec![], + minimum: Some(1), + maximum: Some(5), + minimum_items: None, + item_type: None, + } +} + +pub(crate) fn catalogue_rule(code: &str, options: Vec) -> CatalogueRule { + CatalogueRule { + code: code.into(), + options, + } +} + +pub(crate) fn configured( + code: &str, + value: OptionValue, +) -> BTreeMap> { + BTreeMap::from([(code.into(), BTreeMap::from([("limit".into(), value)]))]) +} + +pub(crate) fn custom_rule() -> RuleText { + RuleText { + code: "XSQBRTEST201".into(), + family: "XSQBRTEST".into(), + slug: "custom-check".into(), + message: "Check it.".into(), + remediation: "Fix it.".into(), + custom: true, + source: Some("rules/check.py".into()), + options: vec![OptionText { + name: "names".into(), + description: "Names.".into(), + default_repr: "('a',)".into(), + effective_repr: "('b', \"it's\")".into(), + }], + ..RuleText::default() + } +} + +pub(crate) fn overridden_config() -> RulesConfig { + RulesConfig { + thresholds: BTreeMap::from([("min_tests_per_model".into(), 3)]), + threshold_overrides: vec![ThresholdOverride { + paths: vec!["models/a".into()], + thresholds: BTreeMap::from([("min_audits_per_model".into(), 2)]), + reason: "Critical.".into(), + }], + ..RulesConfig::default() + } +} + +/// Two skill targets under `root`, seeded with the case's existing content. +pub(crate) fn skill_targets(root: &Path, test_case: &InstallTestCase) -> [SkillTarget; 2] { + let targets = [root.join("a/SKILL.md"), root.join("b/SKILL.md")].map(|path| SkillTarget { + path, + state: test_case.state.into(), + }); + std::fs::create_dir_all(root.join("a")).expect("a first skill folder"); + let _ = test_case + .existing + .map(|existing| std::fs::write(&targets[0].path, existing).expect("a first skill")); + let _ = test_case + .blocked_second + .then(|| std::fs::write(root.join("b"), "").expect("a blocking file")); + let _ = test_case + .existing + .filter(|_| !test_case.blocked_second) + .map(|existing| { + std::fs::create_dir_all(root.join("b")).expect("a second skill folder"); + std::fs::write(&targets[1].path, existing).expect("a second skill") + }); + targets +} + +/// The number of staged temporary or backup files left beside the targets. +pub(crate) fn staged_leftovers(root: &Path) -> usize { + ["a", "b"] + .iter() + .filter_map(|folder| std::fs::read_dir(root.join(folder)).ok()) + .flatten() + .flatten() + .filter(|entry| entry.file_name().to_string_lossy().starts_with('.')) + .count() +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs new file mode 100644 index 0000000000..69a6f4e799 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_rule_catalogue.rs @@ -0,0 +1,217 @@ +use std::collections::BTreeMap; + +use crate::models::RulesConfig; +use crate::rule_catalogue::main::format_rule_text::format_rule_text; +use crate::rule_catalogue::main::install_rendered_skills::install_rendered_skills; +use crate::rule_catalogue::main::render_skills_body::render_skills_body; +use crate::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use crate::rule_catalogue::models::{OptionValue, RuleText, SkillTarget}; +use crate::rule_catalogue::tests::helpers::{ + catalogue_rule, configured, custom_rule, integer_option, overridden_config, skill_targets, + staged_leftovers, +}; +use crate::rule_catalogue::tests::test_types::{ + CatalogueTestCase, InstallTestCase, RuleTextTestCase, SkillsBodyTestCase, +}; + +#[test] +fn given_catalogue_when_validating_then_orders_or_rejects_like_python() { + let test_cases = [ + CatalogueTestCase { + description: "valid rules sort by code", + rules: vec![ + catalogue_rule("XSQBRB", vec![]), + catalogue_rule("SQBRA", vec![]), + ], + configured: BTreeMap::new(), + expected_result: Ok(vec![1, 0]), + }, + CatalogueTestCase { + description: "duplicate codes are rejected", + rules: vec![ + catalogue_rule("SQBRA", vec![]), + catalogue_rule("SQBRA", vec![]), + ], + configured: BTreeMap::new(), + expected_result: Err("duplicate rule codes: SQBRA"), + }, + CatalogueTestCase { + description: "a bool is not an integer option value", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(2)])], + configured: configured("XSQBRA", OptionValue::Bool(true)), + expected_result: Err("rule XSQBRA option limit has the wrong type"), + }, + CatalogueTestCase { + description: "configured values respect the maximum", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(2)])], + configured: configured("XSQBRA", OptionValue::Int(9)), + expected_result: Err("rule XSQBRA option limit exceeds its maximum"), + }, + CatalogueTestCase { + description: "defaults respect the minimum", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(0)])], + configured: BTreeMap::new(), + expected_result: Err("rule XSQBRA option limit default is below its minimum"), + }, + CatalogueTestCase { + description: "options for unknown codes are rejected", + rules: vec![catalogue_rule("XSQBRA", vec![integer_option(2)])], + configured: configured("XSQBRZ", OptionValue::Int(2)), + expected_result: Err("rule options target unknown codes: XSQBRZ"), + }, + ]; + for test_case in test_cases { + let result = sorted_catalogue_order(&test_case.rules, &test_case.configured); + assert_eq!( + result, + test_case.expected_result.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rule_when_formatting_then_matches_python_inspection_text() { + let test_cases = [ + RuleTextTestCase { + description: "a custom rule with options and no config", + rule: custom_rule(), + with_config: false, + expected_text: "XSQBRTEST201: custom-check\nFamily: XSQBRTEST\nEnabled by default: no\nKind: custom\n\nCheck it.\n\nRemediation: Fix it.\n\nSource: rules/check.py\n\nOptions:\n- names: default=('a',) (Names.)", + }, + RuleTextTestCase { + description: "a test rule with config shows test paths only", + rule: RuleText { + code: "SQBRTEST101".into(), + enabled_by_default: true, + ..RuleText::default() + }, + with_config: true, + expected_text: "SQBRTEST101: \nFamily: \nEnabled by default: yes\nKind: built-in\n\n\n\nRemediation: \n\nEffective SQL test paths:\n- Unit tests: tests/unit/\n- Scenarios: tests/scenarios/\n- Cross-domain pipelines: tests/unit/pipelines/", + }, + RuleTextTestCase { + description: "a minimum rule shows effective thresholds and overrides", + rule: RuleText { + code: "SQBRTEST202".into(), + ..RuleText::default() + }, + with_config: true, + expected_text: "SQBRTEST202: \nFamily: \nEnabled by default: no\nKind: built-in\n\n\n\nRemediation: \n\nEffective thresholds:\n- `max_constant_container_files` = `10`\n- `max_enum_container_files` = `10`\n- `max_macro_container_files` = `10`\n- `max_role_container_depth` = `1`\n- `max_subdomain_depth` = `1`\n- `min_audits_per_model` = `1`\n- `min_custom_rule_test_cases` = `1`\n- `min_shared_container_prefix_files` = `2`\n- `min_shared_owner_prefix_directories` = `2`\n- `min_tests_per_model` = `3`\n- Override 1 at `models/a`: `{'min_audits_per_model': 2}` (Critical.)\n- Matching overrides apply in authored order; the last match wins.\n\nEffective SQL test paths:\n- Unit tests: tests/unit/\n- Scenarios: tests/scenarios/\n- Cross-domain pipelines: tests/unit/pipelines/", + }, + ]; + let config: RulesConfig = overridden_config(); + for test_case in test_cases { + let text: String = + format_rule_text(&test_case.rule, test_case.with_config.then_some(&config)); + assert_eq!(text, test_case.expected_text, "{}", test_case.description); + } +} + +#[test] +fn given_rules_when_rendering_skills_then_lists_rules_and_sections() { + let test_cases = [ + SkillsBodyTestCase { + description: "custom options and test paths", + rules: vec![custom_rule()], + expected_lines: vec![ + "### XSQBRTEST201: custom-check", + "Correct shape: Apply `custom-check` at the SQL node named by the finding.", + "Effective options: {'names': ('b', \"it's\")}", + "## Active Rules", + ], + }, + SkillsBodyTestCase { + description: "a built-in example and the layout section", + rules: vec![RuleText { + code: "SQBRPROJECT203".into(), + slug: "depth".into(), + ..RuleText::default() + }], + expected_lines: vec![ + "Correct shape: domain/level/subdomain/model.sql at the default depth of one.", + "## Effective Thresholds", + "## Owner Layout", + "- Configured levels: `staging, intermediate/clean, intermediate/enriched, mart`", + ], + }, + ]; + for test_case in test_cases { + let body: String = render_skills_body(&test_case.rules, &RulesConfig::default()); + let lines: Vec<&str> = body.lines().collect(); + let found: Vec<&str> = test_case + .expected_lines + .iter() + .copied() + .filter(|line| lines.contains(line)) + .collect(); + assert_eq!(found, test_case.expected_lines, "{}", test_case.description); + } +} + +#[test] +fn given_skill_targets_when_installing_then_replaces_only_owned_content() { + let test_cases = [ + InstallTestCase { + description: "missing targets are created", + existing: None, + state: "missing", + blocked_second: false, + expected_error: None, + expected_contents: [Some("new"), Some("new")], + }, + InstallTestCase { + description: "stale targets are replaced", + existing: Some("old"), + state: "stale", + blocked_second: false, + expected_error: None, + expected_contents: [Some("new"), Some("new")], + }, + InstallTestCase { + description: "foreign content is refused and kept", + existing: Some("mine"), + state: "foreign", + blocked_second: false, + expected_error: Some("refusing to overwrite foreign rules skill: "), + expected_contents: [Some("mine"), Some("mine")], + }, + InstallTestCase { + description: "a later staging failure replaces no target", + existing: Some("old"), + state: "stale", + blocked_second: true, + expected_error: Some(""), + expected_contents: [Some("old"), None], + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("a temporary folder"); + let targets: [SkillTarget; 2] = skill_targets(directory.path(), &test_case); + let error: Option = install_rendered_skills("new", &targets) + .err() + .map(|error| error.message); + assert_eq!( + error.map(|message| message.starts_with(test_case.expected_error.unwrap_or("-"))), + test_case.expected_error.map(|_| true), + "{}", + test_case.description + ); + let contents: Vec> = targets + .iter() + .map(|target| std::fs::read_to_string(&target.path).ok()) + .collect(); + let expected: Vec> = test_case + .expected_contents + .iter() + .map(|content| content.map(str::to_owned)) + .collect(); + assert_eq!(contents, expected, "{}", test_case.description); + assert_eq!( + staged_leftovers(directory.path()), + 0, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs new file mode 100644 index 0000000000..68a40bc6cf --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/tests/test_types.rs @@ -0,0 +1,32 @@ +use std::collections::BTreeMap; + +use crate::rule_catalogue::models::{CatalogueRule, OptionValue, RuleText}; + +pub(crate) struct CatalogueTestCase { + pub(crate) description: &'static str, + pub(crate) rules: Vec, + pub(crate) configured: BTreeMap>, + pub(crate) expected_result: Result, &'static str>, +} + +pub(crate) struct RuleTextTestCase { + pub(crate) description: &'static str, + pub(crate) rule: RuleText, + pub(crate) with_config: bool, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct SkillsBodyTestCase { + pub(crate) description: &'static str, + pub(crate) rules: Vec, + pub(crate) expected_lines: Vec<&'static str>, +} + +pub(crate) struct InstallTestCase { + pub(crate) description: &'static str, + pub(crate) existing: Option<&'static str>, + pub(crate) state: &'static str, + pub(crate) blocked_second: bool, + pub(crate) expected_error: Option<&'static str>, + pub(crate) expected_contents: [Option<&'static str>; 2], +} diff --git a/python-boundary.toml b/python-boundary.toml index e7c5996967..123cef517f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -124,6 +124,11 @@ path = "rule_engine/_helpers/engine/custom_payloads.py" reason = "user-code" note = "Reads user rule functions (source lines, qualified names, import closures) to describe custom rules for native selection." +[[stays]] +path = "rule_engine/_helpers/engine/_custom_catalogue.py" +reason = "user-code" +note = "Imports the project's custom rule modules and collects their @rule functions for the native catalogue." + [[stays]] path = "rule_engine/_helpers/engine/custom_rules.py" reason = "user-code" diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index a76454d163..3cc0fcc722 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -7,7 +7,7 @@ from scripts.cold_compile_performance._helpers.dense_project import dense_model_name from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules +from sqlbuild._native import build_rule_catalogue, select_catalogue_rules from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase @@ -51,8 +51,10 @@ def run_varied_cache_benchmark( def _assert_complete_rules(project_dir: Path) -> None: config: RulesConfig = load_rules_config(project_dir=project_dir) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) - selected: tuple[Rule, ...] = select_rules( + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=True + ) + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=config, project_dir=project_dir ) builtin_codes: set[str] = {rule.code for rule in filterfalse(attrgetter("custom"), catalogue)} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a74c62182b..0ab6974209 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -333,7 +333,6 @@ from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult -from sqlbuild.rule_engine.types import RuleCheck from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.models import LifecycleEvent @@ -394,7 +393,6 @@ class LintPreparationRequest(TypedDict): dialect: str def analyze_column_references_json(request_json: str) -> str: ... -def builtin_rules(*, check: RuleCheck) -> list[Rule]: ... def selected_sql_rule_codes(*, select: list[str], ignore: list[str]) -> list[str]: ... class NativeRulesRequest: @@ -2006,8 +2004,21 @@ def selected_rule_codes( *, config: RulesConfig, custom_payloads: list[dict[str, object]] ) -> list[str]: ... def rule_selected(*, config: RulesConfig, code: str) -> bool: ... -def render_owned_skill(*, content: str, input_fingerprint: str) -> str: ... -def skill_freshness(*, content: str | None, input_fingerprint: str) -> str: ... +def build_rule_catalogue( + *, config: RulesConfig, project_dir: Path, include_custom: bool | None = None +) -> tuple[Rule, ...]: ... +def select_catalogue_rules( + *, + catalogue: tuple[Rule, ...], + config: RulesConfig, + project_dir: Path, + custom_payloads: list[dict[str, object]] | None = None, +) -> tuple[Rule, ...]: ... +def resolve_rule_ignore_selectors( + *, config: RulesConfig, project: CompiledProjectView +) -> RulesConfig: ... +def format_rule(*, rule: Rule, config: RulesConfig | None = None) -> str: ... +def install_rules_skills(*, config: RulesConfig, project_dir: Path, check: bool) -> bool: ... def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ... def parse_model_headers( headers: list[str], diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py new file mode 100644 index 0000000000..c86d8bafe6 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -0,0 +1,76 @@ +"""Import the project's custom rule modules and collect their `@rule` functions.""" + +from __future__ import annotations + +import importlib.util +import sys +from dataclasses import replace +from importlib.machinery import ModuleSpec +from pathlib import Path +from types import ModuleType + +from sqlbuild.rule_engine._helpers.engine.definition import ( + resolve_rule_signature, + rule_from_value, +) +from sqlbuild.rule_engine.constants import INIT_MODULE_NAME, RULE_DECORATOR_TOKEN +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.rule_engine.models import Rule + + +def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: + """Every custom rule declared under the project's `rules/` directory, in file order.""" + + result: list[Rule] = [] + existing_modules: frozenset[str] = frozenset(sys.modules) + repository_path: str = str(project_dir.resolve()) + sys.path.insert(0, repository_path) + try: + for file_path in sorted((project_dir / "rules").resolve().rglob("*.py")): + if RULE_DECORATOR_TOKEN not in file_path.read_text(encoding="utf-8"): + continue + name: str = _rule_module_name(file_path=file_path, project_dir=project_dir) + spec: ModuleSpec | None = importlib.util.spec_from_file_location(name, file_path) + if spec is None or spec.loader is None: + raise RulesError(f"could not load rule file {file_path}") + module: ModuleType = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except Exception as error: + raise RulesError(f"could not import rule file {file_path}: {error}") from error + finally: + _ = sys.modules.pop(name, None) + result.extend(_module_rules(module=module, source=file_path)) + finally: + sys.path.remove(repository_path) + _remove_loaded_project_modules(existing_modules=existing_modules, project_dir=project_dir) + return tuple(result) + + +def _rule_module_name(*, file_path: Path, project_dir: Path) -> str: + relative: Path = file_path.relative_to(project_dir.resolve()).with_suffix("") + parts: tuple[str, ...] = relative.parts + if parts[-1] == INIT_MODULE_NAME: + parts = parts[:-1] + return ".".join(parts) + + +def _module_rules(*, module: ModuleType, source: Path) -> tuple[Rule, ...]: + rules: tuple[Rule, ...] = tuple( + resolve_rule_signature(rule=rule) + for value in vars(module).values() + if getattr(value, "__module__", None) == module.__name__ + and (rule := rule_from_value(value=value)) is not None + ) + return tuple(replace(rule, source=source.as_posix()) for rule in rules) + + +def _remove_loaded_project_modules(*, existing_modules: frozenset[str], project_dir: Path) -> None: + for name in tuple(sys.modules): + if name in existing_modules: + continue + module: object = sys.modules.get(name) + source: object = getattr(module, "__file__", None) + if isinstance(source, str) and Path(source).resolve().is_relative_to(project_dir.resolve()): + _ = sys.modules.pop(name, None) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py deleted file mode 100644 index 512d3f411e..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Built-in and custom rule catalogue construction.""" - -from __future__ import annotations - -import importlib.util -import sys -from collections.abc import Mapping -from dataclasses import replace -from importlib.machinery import ModuleSpec -from pathlib import Path -from types import ModuleType - -from sqlbuild._native import builtin_rules, selected_rule_codes -from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads -from sqlbuild.rule_engine._helpers.engine.definition import ( - resolve_rule_signature, - rule_from_value, -) -from sqlbuild.rule_engine.constants import ( - INIT_MODULE_NAME, - RULE_DECORATOR_TOKEN, -) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import ( - Finding, - Rule, - RuleOption, - RulesConfig, -) - - -def build_catalogue( - *, config: RulesConfig, project_dir: Path, include_custom: bool = True -) -> tuple[Rule, ...]: - """Build and validate the complete configured rule catalogue.""" - - rules_dir: Path = project_dir / "rules" - custom: tuple[Rule, ...] = ( - (_load_paths(paths=("rules",), project_dir=project_dir) if rules_dir.is_dir() else ()) - if include_custom - else () - ) - rules: tuple[Rule, ...] = (*_builtins(), *custom) - codes: tuple[str, ...] = tuple(rule.code for rule in rules) - duplicates: tuple[str, ...] = tuple( - sorted(code for code in set(codes) if codes.count(code) > 1) - ) - if duplicates: - raise RulesError(f"duplicate rule codes: {', '.join(duplicates)}") - for rule in rules: - _validate_options(rule=rule, configured=config.rule_options.get(rule.code, {})) - configured_codes: set[str] = set(config.rule_options) - unknown_option_codes: set[str] = configured_codes - set(codes) - if unknown_option_codes: - raise RulesError( - f"rule options target unknown codes: {', '.join(sorted(unknown_option_codes))}" - ) - return tuple(sorted(rules, key=lambda item: item.code)) - - -def _builtins() -> tuple[Rule, ...]: - return tuple(builtin_rules(check=_native_builtin)) - - -def _native_builtin(*, model: object, ctx: object) -> list[Finding]: - del model, ctx - raise RulesError("built-in rules execute only through the native engine") - - -def select_rules( - *, - catalogue: tuple[Rule, ...], - config: RulesConfig, - project_dir: Path, - custom_payloads: list[dict[str, object]] | None = None, -) -> tuple[Rule, ...]: - """Resolve the active catalogue through the native Fensu rules owner.""" - - by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} - codes: list[str] = selected_rule_codes( - config=config, - custom_payloads=( - custom_rule_payloads(catalogue=catalogue, project_dir=project_dir) - if custom_payloads is None - else custom_payloads - ), - ) - try: - return tuple(by_code[code] for code in codes) - except KeyError as error: - raise RulesError(f"native compiler rules returned unknown rule {error.args[0]}") from error - - -def _load_paths(*, paths: tuple[str, ...], project_dir: Path) -> tuple[Rule, ...]: - result: list[Rule] = [] - existing_modules: frozenset[str] = frozenset(sys.modules) - repository_path: str = str(project_dir.resolve()) - sys.path.insert(0, repository_path) - try: - for configured in paths: - path: Path = (project_dir / configured).resolve() - if not path.is_relative_to(project_dir.resolve()): - raise RulesError(f"rule path escapes the project: {configured}") - configured_directory: bool = path.is_dir() - files: tuple[Path, ...] = ( - tuple(sorted(path.rglob("*.py"))) if path.is_dir() else (path,) - ) - for file_path in files: - if not file_path.is_file(): - raise RulesError(f"rule path does not exist: {file_path}") - if configured_directory and RULE_DECORATOR_TOKEN not in file_path.read_text( - encoding="utf-8" - ): - continue - name: str = _rule_module_name(file_path=file_path, project_dir=project_dir) - spec: ModuleSpec | None = importlib.util.spec_from_file_location(name, file_path) - if spec is None or spec.loader is None: - raise RulesError(f"could not load rule file {file_path}") - module: ModuleType = importlib.util.module_from_spec(spec) - sys.modules[name] = module - try: - spec.loader.exec_module(module) - except Exception as error: - raise RulesError(f"could not import rule file {file_path}: {error}") from error - finally: - _ = sys.modules.pop(name, None) - result.extend( - _module_rules( - module=module, - source=file_path, - require_rules=not configured_directory, - ) - ) - finally: - sys.path.remove(repository_path) - _remove_loaded_project_modules( - existing_modules=existing_modules, - project_dir=project_dir, - ) - return tuple(result) - - -def _rule_module_name(*, file_path: Path, project_dir: Path) -> str: - relative: Path = file_path.relative_to(project_dir.resolve()).with_suffix("") - parts: tuple[str, ...] = relative.parts - if parts[-1] == INIT_MODULE_NAME: - parts = parts[:-1] - return ".".join(parts) - - -def _module_rules( - *, module: ModuleType, source: Path, require_rules: bool = True -) -> tuple[Rule, ...]: - rules: tuple[Rule, ...] = tuple( - resolve_rule_signature(rule=rule) - for value in vars(module).values() - if getattr(value, "__module__", None) == module.__name__ - and (rule := rule_from_value(value=value)) is not None - ) - if not rules and require_rules: - raise RulesError(f"rule source exposes no @rule functions: {source}") - return tuple(replace(rule, source=source.as_posix()) for rule in rules) - - -def _remove_loaded_project_modules(*, existing_modules: frozenset[str], project_dir: Path) -> None: - for name in tuple(sys.modules): - if name in existing_modules: - continue - module: object = sys.modules.get(name) - source: object = getattr(module, "__file__", None) - if isinstance(source, str) and Path(source).resolve().is_relative_to(project_dir.resolve()): - _ = sys.modules.pop(name, None) - - -def _validate_options(*, rule: Rule, configured: Mapping[str, object]) -> None: - declarations: dict[str, RuleOption[object]] = {option.name: option for option in rule.options} - for option in rule.options: - _validate_option_declaration(rule=rule, option=option) - unknown: set[str] = set(configured) - set(declarations) - if unknown: - raise RulesError(f"rule {rule.code} has unknown options: {', '.join(sorted(unknown))}") - for name, value in configured.items(): - option: RuleOption[object] = declarations[name] - expected: type[object] = option.value_type - if expected is tuple: - valid: bool = isinstance(value, tuple) - else: - valid = isinstance(value, expected) and not ( - expected is int and isinstance(value, bool) - ) - if not valid: - raise RulesError(f"rule {rule.code} option {name} has the wrong type") - if option.choices and value not in option.choices: - raise RulesError(f"rule {rule.code} option {name} is not an allowed choice") - if isinstance(value, tuple): - if option.minimum_items is not None and len(value) < option.minimum_items: - raise RulesError(f"rule {rule.code} option {name} has too few items") - expected_item_type: type[object] | None = option.item_type - if expected_item_type is not None and any( - not isinstance(item, expected_item_type) for item in value - ): - raise RulesError( - f"rule {rule.code} option {name} has item values of the wrong type" - ) - if isinstance(value, int) and not isinstance(value, bool): - if option.minimum is not None and value < option.minimum: - raise RulesError(f"rule {rule.code} option {name} is below its minimum") - if option.maximum is not None and value > option.maximum: - raise RulesError(f"rule {rule.code} option {name} exceeds its maximum") - - -def _validate_option_declaration(*, rule: Rule, option: RuleOption[object]) -> None: - if not option.name.strip() or not option.description.strip(): - raise RulesError(f"rule {rule.code} declares an incomplete option") - default: object = option.default - expected: type[object] = option.value_type - valid_default: bool = isinstance(default, expected) and not ( - expected is int and isinstance(default, bool) - ) - if not valid_default: - raise RulesError(f"rule {rule.code} option {option.name} has a default of the wrong type") - if option.choices and default not in option.choices: - raise RulesError(f"rule {rule.code} option {option.name} default is not an allowed choice") - if isinstance(default, tuple): - if option.minimum_items is not None and len(default) < option.minimum_items: - raise RulesError(f"rule {rule.code} option {option.name} default has too few items") - if option.item_type is not None and any( - not isinstance(item, option.item_type) for item in default - ): - raise RulesError( - f"rule {rule.code} option {option.name} default has items of the wrong type" - ) - if isinstance(default, int) and not isinstance(default, bool): - if option.minimum is not None and default < option.minimum: - raise RulesError(f"rule {rule.code} option {option.name} default is below its minimum") - if option.maximum is not None and default > option.maximum: - raise RulesError(f"rule {rule.code} option {option.name} default exceeds its maximum") diff --git a/src/sqlbuild/rule_engine/_helpers/engine/config.py b/src/sqlbuild/rule_engine/_helpers/engine/config.py deleted file mode 100644 index 17c5eea811..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/config.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Resolve rule ignore selectors against the compiled project.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope -from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import RuleIgnore, RulesConfig - - -def resolve_rule_ignore_selectors( - *, config: RulesConfig, project: CompiledProjectView -) -> RulesConfig: - """Resolve configured resource selectors into deterministic finding paths.""" - if not any(ignore.selectors for ignore in config.rule_ignores): - return config - path_by_key: dict[CompiledObjectKey, Path] = _resource_paths(project=project) - resolved: list[RuleIgnore] = [] - for ignore in config.rule_ignores: - if not ignore.selectors: - resolved.append(ignore) - continue - scope: PlannerScope = build_planner_scope(project=project, select=ignore.selectors) - selector_paths: set[str] = { - path_by_key[key].as_posix() for key in scope.selected_keys if key in path_by_key - } - if not selector_paths: - joined: str = ", ".join(ignore.selectors) - raise RulesError(f"rule ignore selectors resolve no resources with paths: {joined}") - resolved.append( - replace( - ignore, - paths=tuple(sorted({*ignore.paths, *selector_paths})), - selectors=(), - ) - ) - return replace(config, rule_ignores=tuple(resolved)) - - -def _resource_paths(*, project: CompiledProjectView) -> dict[CompiledObjectKey, Path]: - paths: dict[CompiledObjectKey, Path] = { - model.key: model.relative_path for model in project.models - } - paths.update({function.key: function.relative_path for function in project.functions}) - paths.update({seed.key: seed.seed_file.relative_path for seed in project.seeds}) - paths.update({source.key: source.source_file.relative_path for source in project.sources}) - return paths diff --git a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py deleted file mode 100644 index 1b1625c074..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Single rules catalogue and active-ruleset resolution.""" - -from __future__ import annotations - -import hashlib -import json -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - custom_rule_implementation_fingerprint, -) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors -from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native -from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig, RulesResult - - -def evaluate_project( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - dialect: str = "generic", - verify_determinism: bool = False, -) -> RulesResult: - """Evaluate selected rules through one native project boundary.""" - - effective_config: RulesConfig = resolve_rule_ignore_selectors(config=config, project=project) - catalogue: tuple[Rule, ...] = build_catalogue( - config=effective_config, - project_dir=project_dir, - include_custom=_config_selects_custom(effective_config), - ) - if any(rule.custom for rule in catalogue): - selected: tuple[Rule, ...] = select_rules( - catalogue=catalogue, config=effective_config, project_dir=project_dir - ) - verify_custom_rules(rules=selected, project_dir=project_dir) - return evaluate_native( - project=project, - config=effective_config, - project_dir=project_dir, - catalogue=catalogue, - dialect=dialect, - verify_determinism=verify_determinism, - ) - - -def resolve_ruleset(*, config: RulesConfig, project_dir: Path) -> ResolvedRuleset: - """Resolve one immutable ruleset for evaluation, inspection, caching, and skills.""" - - catalogue: tuple[Rule, ...] = build_catalogue( - config=config, - project_dir=project_dir, - include_custom=_config_selects_custom(config), - ) - rules: tuple[Rule, ...] = select_rules( - catalogue=catalogue, config=config, project_dir=project_dir - ) - custom_selected: bool = any(rule.custom for rule in rules) - cacheable: bool = True - if custom_selected: - verify_custom_rules(rules=rules, project_dir=project_dir) - payload: list[dict[str, object]] = [] - for rule in rules: - source_hash: str | None = None - if rule.source is not None: - source_hash = hashlib.sha256(Path(rule.source).read_bytes()).hexdigest() - implementation_hash: str = custom_rule_implementation_fingerprint( - rule=rule, project_dir=project_dir - ) - payload.append( - { - "code": rule.code, - "family": rule.family, - "slug": rule.slug, - "message": rule.message, - "remediation": rule.remediation, - "options": config.rule_options.get(rule.code, {}), - "source_hash": source_hash, - "implementation_hash": implementation_hash, - } - ) - rules_payload: dict[str, object] = { - "engine": _engine_fingerprint(), - "rules": payload, - "thresholds": config.thresholds, - "threshold_overrides": config.threshold_overrides, - "select_star_allow": config.select_star_allow, - "domains": config.domains, - "approved_source_tokens": config.approved_source_tokens, - "retired_source_tokens": config.retired_source_tokens, - "sql_tests": config.sql_tests, - "layout": config.layout, - "exceptions": config.rule_exceptions, - "graph_edge_exceptions": config.graph_edge_exceptions, - "ignores": config.rule_ignores, - } - fingerprint: str = hashlib.sha256( - json.dumps(rules_payload, sort_keys=True, default=str).encode() - ).hexdigest() - return ResolvedRuleset( - catalogue=catalogue, - rules=rules, - fingerprint=fingerprint, - cacheable=cacheable, - ) - - -def _engine_fingerprint() -> str: - digest: Any = hashlib.sha256() - engine_root: Path = Path(__file__).parents[2] - for path in sorted(engine_root.rglob("*.py")): - digest.update(path.relative_to(engine_root).as_posix().encode()) - digest.update(path.read_bytes()) - return digest.hexdigest() - - -def _config_selects_custom(config: RulesConfig) -> bool: - ignored_selectors: list[str] = [] - for entry in config.rule_ignores: - ignored_selectors.extend(entry.rules) - selectors: tuple[str, ...] = ( - *config.select, - *config.ignore, - *config.rule_options, - *(entry.rule for entry in config.rule_exceptions), - *ignored_selectors, - ) - return any(selector.startswith("XSQBR") for selector in selectors) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py deleted file mode 100644 index 156d4b925f..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Deterministic rules result and rule rendering.""" - -from __future__ import annotations - -from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines -from sqlbuild.rule_engine.models import Rule, RulesConfig - -_MINIMUM_RULES_CODES: frozenset[str] = frozenset({"SQBRTEST201", "SQBRTEST202"}) - - -def format_rule_text(*, rule: Rule, config: RulesConfig | None = None) -> str: - lines: list[str] = [ - f"{rule.code}: {rule.slug}", - f"Family: {rule.family}", - f"Enabled by default: {'yes' if rule.enabled_by_default else 'no'}", - f"Kind: {'custom' if rule.custom else 'built-in'}", - "", - rule.message, - "", - f"Remediation: {rule.remediation}", - ] - if rule.source is not None: - lines.extend(("", f"Source: {rule.source}")) - if rule.options: - lines.extend(("", "Options:")) - for option in rule.options: - lines.append(f"- {option.name}: default={option.default!r} ({option.description})") - if config is not None and rule.code in _MINIMUM_RULES_CODES: - lines.extend(("", "Effective thresholds:", *format_threshold_lines(config=config))) - if config is not None and rule.code.startswith("SQBRTEST"): - lines.extend( - ( - "", - "Effective SQL test paths:", - "- Unit tests: tests/unit/", - "- Scenarios: tests/scenarios/", - f"- Cross-domain pipelines: tests/unit/{config.sql_tests.pipeline_directory}/", - ) - ) - return "\n".join(lines) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py b/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py deleted file mode 100644 index cbe6fa7817..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skill_installation.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Atomic generated-skill installation phases.""" - -from __future__ import annotations - -from contextlib import ExitStack -from pathlib import Path -from stat import S_IMODE -from tempfile import NamedTemporaryFile - -from sqlbuild._native import skill_freshness -from sqlbuild.rule_engine.constants import REPLACEABLE_SKILL_STATES -from sqlbuild.rule_engine.exceptions import RulesError - - -def install_rendered_skills( - *, content: str, input_fingerprint: str, targets: tuple[Path, ...] -) -> None: - """Validate and atomically replace all generated skill targets.""" - - freshness: tuple[str, ...] = tuple( - skill_freshness( - content=path.read_text(encoding="utf-8") if path.is_file() else None, - input_fingerprint=input_fingerprint, - ) - for path in targets - ) - for path, state in zip(targets, freshness, strict=True): - if state not in REPLACEABLE_SKILL_STATES: - raise RulesError(f"refusing to overwrite {state} rules skill: {path}") - staged: tuple[tuple[Path, Path | None, Path], ...] = _stage_skills( - targets=targets, content=content - ) - _ = _apply_staged_skills(staged=staged, installed_bytes=content.encode()) - - -def _apply_staged_skills( - *, staged: tuple[tuple[Path, Path | None, Path], ...], installed_bytes: bytes -) -> None: - retained_backups: set[Path] = set() - replaced: list[tuple[Path | None, Path]] = [] - try: - for temporary, backup, target in staged: - current: bytes | None = target.read_bytes() if target.is_file() else None - expected: bytes | None = backup.read_bytes() if backup is not None else None - if current != expected: - raise RulesError( - f"refusing to overwrite rules skill changed during installation: {target}" - ) - temporary.replace(target) - replaced.append((backup, target)) - except Exception as error: - retained_backups = _rollback_replaced_skills( - replaced=tuple(replaced), - installed_bytes=installed_bytes, - error=error, - ) - raise - finally: - _remove_staged(staged=staged, retained_backups=retained_backups) - - -def _stage_skills( - *, targets: tuple[Path, ...], content: str -) -> tuple[tuple[Path, Path | None, Path], ...]: - staged: list[tuple[Path, Path | None, Path]] = [] - with ExitStack() as staging_cleanup: - for target in targets: - target.parent.mkdir(parents=True, exist_ok=True) - target_mode: int = S_IMODE(target.stat().st_mode) if target.is_file() else 0o644 - with NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=target.parent, - prefix=f".{target.name}.", - suffix=".tmp", - delete=False, - ) as temporary_file: - temporary: Path = Path(temporary_file.name) - staged.append((temporary, None, target)) - _ = staging_cleanup.callback(temporary.unlink, missing_ok=True) - _ = temporary_file.write(content) - temporary.chmod(target_mode) - if target.is_file(): - with NamedTemporaryFile( - mode="wb", - dir=target.parent, - prefix=f".{target.name}.", - suffix=".bak", - delete=False, - ) as backup_file: - backup: Path = Path(backup_file.name) - staged[-1] = (temporary, backup, target) - _ = staging_cleanup.callback(backup.unlink, missing_ok=True) - _ = backup_file.write(target.read_bytes()) - backup.chmod(target_mode) - _ = staging_cleanup.pop_all() - return tuple(staged) - - -def _rollback_replaced_skills( - *, - replaced: tuple[tuple[Path | None, Path], ...], - installed_bytes: bytes, - error: Exception, -) -> set[Path]: - retained_backups: set[Path] = set() - for backup, target in reversed(replaced): - try: - if not target.is_file() or target.read_bytes() != installed_bytes: - if backup is not None: - retained_backups.add(backup) - error.add_note( - f"rollback preserved concurrently changed target {target}; " - f"recovery backup: {backup}" - ) - elif backup is None: - target.unlink(missing_ok=True) - else: - backup.replace(target) - except Exception as rollback_error: - if backup is not None: - retained_backups.add(backup) - error.add_note( - f"rollback failed for {target}: {rollback_error}; recovery backup: {backup}" - ) - return retained_backups - - -def _remove_staged( - *, - staged: tuple[tuple[Path, Path | None, Path], ...], - retained_backups: set[Path], -) -> None: - for temporary, backup, _target in staged: - temporary.unlink(missing_ok=True) - if backup is not None and backup not in retained_backups: - backup.unlink(missing_ok=True) diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py b/src/sqlbuild/rule_engine/_helpers/guidance/skills.py deleted file mode 100644 index 266b9ee7f3..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/skills.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Rules agent-guidance rendering.""" - -from __future__ import annotations - -from collections.abc import Mapping -from pathlib import Path - -from sqlbuild._native import render_owned_skill -from sqlbuild.rule_engine._helpers.engine.ruleset import resolve_ruleset -from sqlbuild.rule_engine._helpers.guidance.thresholds import format_threshold_lines -from sqlbuild.rule_engine.constants import ( - RULES_LAYOUT_RULE_PREFIXES, - RULES_LAYOUT_THRESHOLD_RULE_CODES, - RULES_THRESHOLD_RULE_CODES, -) -from sqlbuild.rule_engine.models import ResolvedRuleset, Rule, RulesConfig - - -def render_skills(*, config: RulesConfig, project_dir: Path) -> tuple[str, str]: - """Render stable guidance from ordinary rules catalogue resolution.""" - - ruleset: ResolvedRuleset = resolve_ruleset(config=config, project_dir=project_dir) - rules: tuple[Rule, ...] = ruleset.rules - body: list[str] = [ - "# SQLBuild Rules", - "", - "Rules enforce SQL model architecture and never modifies SQL files.", - "Run `sqb rules run SQBR` before handing off model changes.", - "", - "## Active Rules", - ] - for rule in rules: - body.extend( - ( - "", - f"### {rule.code}: {rule.slug}", - "", - rule.message, - "", - f"Correct shape: {_rule_example(rule=rule)}", - "", - f"Remediation: {rule.remediation}", - ) - ) - configured: Mapping[str, object] = config.rule_options.get(rule.code, {}) - effective: dict[str, object] = { - option.name: configured.get(option.name, option.default) for option in rule.options - } - if effective: - body.extend(("", f"Effective options: {effective}")) - if ( - config.rule_exceptions - or config.graph_edge_exceptions - or config.rule_ignores - or config.select_star_allow - ): - body.extend( - ( - "", - "## Scoped Deviations", - "", - ) - ) - for entry in config.rule_exceptions: - body.append(f"- Exception `{entry.rule}` at `{entry.path}`: {entry.reason}") - for entry in config.rule_ignores: - scopes: tuple[str, ...] = ( - *(f"path:{path}" for path in entry.paths), - *(f"selector:{selector}" for selector in entry.selectors), - ) - body.append( - f"- Ignore `{','.join(entry.rules)}` at `{','.join(scopes)}`: {entry.reason}" - ) - for entry in config.graph_edge_exceptions: - body.append( - f"- `SQBRGRAPH101` exact edge `{entry.consumer}` -> " - f"`{entry.dependency}`: {entry.reason}" - ) - for entry in config.select_star_allow: - body.append(f"- Lone-star allowance `{','.join(entry.paths)}`: {entry.reason}") - if any( - rule.code in RULES_THRESHOLD_RULE_CODES or rule.code in RULES_LAYOUT_THRESHOLD_RULE_CODES - for rule in rules - ): - body.extend(("", "## Effective Thresholds", "")) - body.extend(format_threshold_lines(config=config)) - if any(rule.code.startswith(RULES_LAYOUT_RULE_PREFIXES) for rule in rules): - body.extend(("", "## Owner Layout", "")) - body.append(f"- Configured levels: `{', '.join(config.layout.levels)}`") - body.append("- Every model owner is a leaf or a branch, never both.") - body.append("- Declaration-role buckets organize files without changing visibility.") - if any(rule.code.startswith("SQBRTEST") for rule in rules): - body.extend(("", "## SQL Test Paths", "")) - body.append("- Unit tests: `tests/unit/`") - body.append("- Scenarios: `tests/scenarios/`") - body.append( - f"- Cross-domain pipelines: `tests/unit/{config.sql_tests.pipeline_directory}/`" - ) - if config.domains or config.approved_source_tokens or config.retired_source_tokens: - body.extend(("", "## Naming Vocabulary", "")) - body.append(f"- Domains: `{', '.join(config.domains)}`") - body.append(f"- Approved source tokens: `{', '.join(config.approved_source_tokens)}`") - for retired, replacement in sorted(config.retired_source_tokens.items()): - body.append(f"- Replace retired `{retired}` with `{replacement}`") - content: str = "\n".join(body).rstrip() + "\n" - return ( - render_owned_skill(content=content, input_fingerprint=ruleset.fingerprint), - ruleset.fingerprint, - ) - - -def _rule_example(*, rule: Rule) -> str: - examples: dict[str, str] = { - "SQBRSQL040": ( - "Compute category_key in an input CTE, then JOIN categories AS c " - "ON o.category_key = c.category_key." - ), - "SQBRSQL041": ( - "WITH orders AS (...), final AS (SELECT order_id FROM orders) " - "SELECT order_id FROM final." - ), - "SQBRMODEL101": 'WITH upstream AS (SELECT * FROM __ref("domain__stg__entity")), ...', - "SQBRMODEL102": "SELECT id, status FROM final; use a lone SELECT * only when allowed.", - "SQBRMODEL103": "A view uses stg_v, int_v, or mart_v; a table uses a non-v layer.", - "SQBRMODEL104": "MODEL() omits schema; central path defaults provide it.", - "SQBRGRAPH101": ( - "Internal flow is stg -> int_clean -> int_enriched; marts are published interfaces." - ), - "SQBRPROJECT101": "domain__int_clean__entity or domain__mart_v__entity.", - "SQBRPROJECT105": "A domain__int_enriched__entity model lives under intermediate/enriched.", - "SQBRPROJECT106": "A domain__stg__entity model resolves to the staging logical schema.", - "SQBRCONTRACT101": ( - "MODEL (contract enforced, columns (...)); declare authoritative columns." - ), - "SQBRPROJECT202": "domain/level/owner contains models or child owners, never both.", - "SQBRPROJECT201": "models///model.sql resolves one domain root.", - "SQBRPROJECT203": "domain/level/subdomain/model.sql at the default depth of one.", - "SQBRPROJECT204": "order_status and order_status_history consolidate under order_status.", - "SQBRDECLARATION302": "_macros/ is flat, or every file uses one concern bucket.", - "SQBRDECLARATION303": "_macros/normalisation/names.py uses one default bucket level.", - "SQBRDECLARATION304": "Keep each flat role or concern bucket within its file cap.", - "SQBRDECLARATION305": "Use temporal/ or scoring/, not utils/ or common/.", - "SQBRDECLARATION306": "Group normalise_order.py and normalise_person.py under normalise/.", - "SQBRTEST101": "Keep unit tests in tests/unit/ and scenarios in tests/scenarios/.", - "SQBRTEST102": "test_stg_orders__excludes_cancelled.sql and daily_revenue__minimal.sql.", - "SQBRTEST103": "models/staging/stg_orders.sql maps to tests/unit/staging/.", - "SQBRTEST104": 'TEST (name "stg_orders__excludes_cancelled_orders");', - "SQBRTEST201": "Attach not_null/unique audits to the model key.", - } - if rule.guidance is not None: - return rule.guidance.good_example - return examples.get(rule.code, f"Apply `{rule.slug}` at the SQL node named by the finding.") diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py b/src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py deleted file mode 100644 index 522842924c..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/guidance/thresholds.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Shared rendering for effective rule test thresholds.""" - -from sqlbuild.rule_engine.constants import RULES_THRESHOLD_DEFAULTS -from sqlbuild.rule_engine.models import RulesConfig - - -def format_threshold_lines(*, config: RulesConfig) -> tuple[str, ...]: - """Render global and path-scoped thresholds in deterministic authored order.""" - - thresholds: dict[str, int] = {**RULES_THRESHOLD_DEFAULTS, **config.thresholds} - lines: list[str] = [f"- `{name}` = `{value}`" for name, value in sorted(thresholds.items())] - for index, entry in enumerate(config.threshold_overrides, start=1): - lines.append( - f"- Override {index} at `{','.join(entry.paths)}`: " - f"`{dict(sorted(entry.thresholds.items()))}` ({entry.reason})" - ) - if config.threshold_overrides: - lines.append("- Matching overrides apply in authored order; the last match wins.") - return tuple(lines) diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index 4923494533..76d0c2cbf9 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -14,9 +14,9 @@ from pathlib import Path from typing import Any, BinaryIO +from sqlbuild._native import build_rule_catalogue from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard @@ -80,7 +80,9 @@ def main() -> int: guard.guard_filesystem_metadata() messages: io.StringIO = io.StringIO() with contextlib.redirect_stdout(messages): - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=True + ) guard.raise_violation() by_code: dict[str, Rule] = {rule.code: rule for rule in catalogue} selected: tuple[Rule, ...] = tuple(by_code[code] for code in sorted(plan)) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index ccbf7171c9..d89d114471 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( build_lint_expansion_context, + build_rule_catalogue, collect_lint_project_files, compiled_relation_keys, declared_relation_keys, @@ -16,7 +17,9 @@ load_sql_rule_cache, lookup_sql_rule_cache, project_with, + resolve_rule_ignore_selectors, run_lint, + select_catalogue_rules, selected_sql_rule_codes, sql_file_rule_key, sql_model_rule_keys, @@ -49,8 +52,6 @@ from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from sqlbuild.lint.types import RelationKeys -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path @@ -98,7 +99,7 @@ def plan_rule_run( ) resolved_project_dir: Path = project_dir.resolve() include_custom: bool = _config_references_custom_rules(effective_config) - catalogue: tuple[Rule, ...] = build_catalogue( + catalogue: tuple[Rule, ...] = build_rule_catalogue( config=effective_config, project_dir=resolved_project_dir, include_custom=include_custom, @@ -106,7 +107,7 @@ def plan_rule_run( custom_payloads: list[dict[str, object]] = custom_rule_payloads( catalogue=catalogue, project_dir=resolved_project_dir ) - selected: tuple[Rule, ...] = select_rules( + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=effective_config, project_dir=resolved_project_dir, diff --git a/src/sqlbuild/rule_engine/main/_build_catalogue.py b/src/sqlbuild/rule_engine/main/_build_catalogue.py deleted file mode 100644 index 3cf786006a..0000000000 --- a/src/sqlbuild/rule_engine/main/_build_catalogue.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Rules catalogue construction entrypoint.""" - -from pathlib import Path - -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue as _build_catalogue -from sqlbuild.rule_engine.models import Rule, RulesConfig - - -def build_catalogue(*, config: RulesConfig, project_dir: Path) -> tuple[Rule, ...]: - """Build the validated built-in and custom rules catalogue.""" - - return _build_catalogue(config=config, project_dir=project_dir) diff --git a/src/sqlbuild/rule_engine/main/_evaluate.py b/src/sqlbuild/rule_engine/main/_evaluate.py index b239779aa9..0677a2dd9d 100644 --- a/src/sqlbuild/rule_engine/main/_evaluate.py +++ b/src/sqlbuild/rule_engine/main/_evaluate.py @@ -2,9 +2,15 @@ from pathlib import Path +from sqlbuild._native import ( + build_rule_catalogue, + resolve_rule_ignore_selectors, + select_catalogue_rules, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.ruleset import evaluate_project -from sqlbuild.rule_engine.models import RulesConfig, RulesResult +from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native +from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules +from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult def evaluate( @@ -17,10 +23,22 @@ def evaluate( ) -> RulesResult: """Evaluate the selected compiler rules over compiled models.""" - return evaluate_project( + effective_config: RulesConfig = resolve_rule_ignore_selectors(config=config, project=project) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=effective_config, project_dir=project_dir + ) + if any(rule.custom for rule in catalogue): + _ = verify_custom_rules( + rules=select_catalogue_rules( + catalogue=catalogue, config=effective_config, project_dir=project_dir + ), + project_dir=project_dir, + ) + return evaluate_native( project=project, - config=config, + config=effective_config, project_dir=project_dir, + catalogue=catalogue, dialect=dialect, verify_determinism=verify_determinism, ) diff --git a/src/sqlbuild/rule_engine/main/render_rule.py b/src/sqlbuild/rule_engine/main/render_rule.py deleted file mode 100644 index c428ba36f3..0000000000 --- a/src/sqlbuild/rule_engine/main/render_rule.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Rule formatting entrypoint.""" - -from sqlbuild.rule_engine._helpers.guidance.presentation import format_rule_text -from sqlbuild.rule_engine.models import Rule, RulesConfig - - -def format_rule(*, rule: Rule, config: RulesConfig | None = None) -> str: - """Format one rule for inspection.""" - - return format_rule_text(rule=rule, config=config) diff --git a/src/sqlbuild/rule_engine/main/skills.py b/src/sqlbuild/rule_engine/main/skills.py deleted file mode 100644 index 68e6c5846e..0000000000 --- a/src/sqlbuild/rule_engine/main/skills.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Generate deterministic agent guidance from the resolved compiler rules.""" - -from __future__ import annotations - -import hashlib -import os -from pathlib import Path -from tempfile import gettempdir - -from filelock import FileLock - -from sqlbuild._native import skill_freshness -from sqlbuild.rule_engine._helpers.guidance.skill_installation import install_rendered_skills -from sqlbuild.rule_engine._helpers.guidance.skills import render_skills -from sqlbuild.rule_engine.constants import ( - RULES_SKILL_PATHS, - SKILL_FRESH, -) -from sqlbuild.rule_engine.models import RulesConfig - - -def install_skills(*, config: RulesConfig, project_dir: Path, check: bool) -> bool: - """Install all owned skill targets or report whether they are fresh.""" - - content: str - input_fingerprint: str - content, input_fingerprint = render_skills(config=config, project_dir=project_dir) - targets: tuple[Path, ...] = tuple(project_dir / value for value in RULES_SKILL_PATHS) - if check: - return all( - skill_freshness( - content=path.read_text(encoding="utf-8") if path.is_file() else None, - input_fingerprint=input_fingerprint, - ) - == SKILL_FRESH - for path in targets - ) - canonical_project: str = os.path.normcase(str(project_dir.resolve())) - lock_name: str = hashlib.sha256(canonical_project.encode()).hexdigest() - lock_dir: Path = Path(gettempdir()) / "sqlbuild-rules-skill-locks" - lock_dir.mkdir(parents=True, exist_ok=True) - with FileLock(lock_dir / f"{lock_name}.lock"): - _ = install_rendered_skills( - content=content, - input_fingerprint=input_fingerprint, - targets=targets, - ) - return True diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 2cd5640d92..01ce72bf63 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -14,7 +14,7 @@ from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( assert_required_cgroup_memory_limit, ) -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules +from sqlbuild._native import build_rule_catalogue, select_catalogue_rules from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -77,8 +77,10 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) config: RulesConfig = load_rules_config(project_dir=project_dir) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=project_dir) - selected: tuple[Rule, ...] = select_rules( + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=True + ) + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=config, project_dir=project_dir ) builtin_codes: set[str] = {rule.code for rule in filterfalse(attrgetter("custom"), catalogue)} diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 6a5bc8a547..9b054b6833 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -11,6 +11,8 @@ from sqlbuild._native import ( NativeRulesRequest, + build_rule_catalogue, + install_rules_skills, project_with, ) from sqlbuild.compiler.compile.models import ( @@ -25,10 +27,9 @@ from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run import native_rows -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES +from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES, RULES_SKILL_PATHS from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( CustomRulesOutcome, @@ -226,7 +227,9 @@ def write_project_file_rule( def load_custom_rule(*, root: Path, configured_path: Path) -> Rule: del configured_path config: RulesConfig = RulesConfig() - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=root) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=root, include_custom=True + ) custom_rules: tuple[Rule, ...] = tuple(filter(attrgetter("custom"), catalogue)) return custom_rules[0] @@ -267,8 +270,8 @@ def custom_rules_with_imports( path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(source, encoding="utf-8") - catalogue: tuple[Rule, ...] = build_catalogue( - config=RulesConfig(select=("XSQBRT101",)), project_dir=project_dir + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=RulesConfig(select=("XSQBRT101",)), project_dir=project_dir, include_custom=True ) return tuple(filter(lambda rule: rule.custom, catalogue)) @@ -302,7 +305,10 @@ def evaluate_cached_custom_rules( select=("XSQBRT101",), cache=RulesCacheConfig(enabled=cache_enabled) ) rules: tuple[Rule, ...] = tuple( - filter(attrgetter("custom"), build_catalogue(config=config, project_dir=project_dir)) + filter( + attrgetter("custom"), + build_rule_catalogue(config=config, project_dir=project_dir, include_custom=True), + ) ) return evaluate_custom_rules_cached( project=project, config=config, project_dir=project_dir, rules=rules, dialect="duckdb" @@ -376,7 +382,9 @@ def evaluate_incremental_step( project=_incremental_project(step.models), config=config, project_dir=project_dir, - catalogue=build_catalogue(config=config, project_dir=project_dir, include_custom=False), + catalogue=build_rule_catalogue( + config=config, project_dir=project_dir, include_custom=False + ), dialect=step.dialect, initial_findings=tuple( Finding( @@ -449,3 +457,10 @@ def scenario_summary(fact: dict[str, Any]) -> tuple[object, ...]: fact["assertion_target_model_names"], fact["target_model_names"], ) + + +def installed_skill(*, config: RulesConfig, project_dir: Path) -> str: + """Install the generated rules skill and return the first target's content.""" + + _ = install_rules_skills(config=config, project_dir=project_dir, check=False) + return (project_dir / RULES_SKILL_PATHS[0]).read_text(encoding="utf-8") diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 5593ed2d7e..2bce72eb50 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -4,11 +4,13 @@ import pytest -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules -from sqlbuild.rule_engine._helpers.engine.ruleset import resolve_ruleset -from sqlbuild.rule_engine._helpers.guidance.skills import render_skills +from sqlbuild._native import ( + build_rule_catalogue, + format_rule, + install_rules_skills, + select_catalogue_rules, +) from sqlbuild.rule_engine.main._evaluate import evaluate -from sqlbuild.rule_engine.main.render_rule import format_rule from sqlbuild.rule_engine.models import ( Rule, RulesCacheConfig, @@ -22,6 +24,7 @@ PolicySelectionTestCase, SqlTestPolicyGuidanceTestCase, ) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import installed_skill from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project RULES_BUILT_IN_CODES: tuple[str, ...] = ( @@ -167,9 +170,11 @@ def test_given_rule_selectors_when_resolving_then_returns_expected_codes( test_case: PolicySelectionTestCase, ) -> None: config: RulesConfig = RulesConfig(select=test_case.select, ignore=test_case.ignore) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=tmp_path) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=tmp_path, include_custom=True + ) - selected: tuple[Rule, ...] = select_rules( + selected: tuple[Rule, ...] = select_catalogue_rules( catalogue=catalogue, config=config, project_dir=tmp_path ) @@ -195,12 +200,14 @@ def test_given_sql_test_policy_when_inspecting_and_rendering_skills_then_effecti select=("SQBRTEST",), sql_tests=SqlTestRulesConfig(pipeline_directory=test_case.pipeline_directory), ) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=tmp_path) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=tmp_path, include_custom=True + ) rules_by_code: dict[str, Rule] = {item.code: item for item in catalogue} rule: Rule = rules_by_code["SQBRTEST103"] inspection: str = format_rule(rule=rule, config=config) - skill, _ = render_skills(config=config, project_dir=tmp_path) + skill: str = installed_skill(config=config, project_dir=tmp_path) assert test_case.expected_path in inspection assert test_case.expected_path in skill @@ -236,7 +243,9 @@ def test_given_minimum_test_rule_when_rendering_guidance_then_all_surfaces_have_ threshold_overrides=(override,), cache=RulesCacheConfig(enabled=False), ) - catalogue: tuple[Rule, ...] = build_catalogue(config=config, project_dir=tmp_path) + catalogue: tuple[Rule, ...] = build_rule_catalogue( + config=config, project_dir=tmp_path, include_custom=True + ) rules_by_code: dict[str, Rule] = {item.code: item for item in catalogue} rule: Rule = rules_by_code["SQBRTEST202"] result: RulesResult = evaluate( @@ -250,17 +259,16 @@ def test_given_minimum_test_rule_when_rendering_guidance_then_all_surfaces_have_ project_dir=tmp_path, ) inspection: str = format_rule(rule=rule, config=config) - skill, _ = render_skills(config=config, project_dir=tmp_path) - without_override: str = resolve_ruleset( - config=RulesConfig(select=("SQBRTEST202",)), project_dir=tmp_path - ).fingerprint - with_override: str = resolve_ruleset(config=config, project_dir=tmp_path).fingerprint + skill: str = installed_skill(config=config, project_dir=tmp_path) + fresh_without_override: bool = install_rules_skills( + config=RulesConfig(select=("SQBRTEST202",)), project_dir=tmp_path, check=True + ) assert result.findings[0].remediation == rule.remediation assert f"Remediation: {rule.remediation}" in inspection assert f"Remediation: {rule.remediation}" in skill assert "marts require two focused tests" in inspection assert "marts require two focused tests" in skill - assert without_override != with_override + assert not fresh_without_override for snippet in test_case.expected_snippets: assert snippet in result.findings[0].remediation diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index 65d1375bf6..89517c73b8 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -5,11 +5,11 @@ import pytest from sqlbuild._native import ( + build_rule_catalogue, custom_rule_implementation_fingerprint, custom_rule_test_evidence, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import Rule, RulesConfig, RulesResult @@ -401,7 +401,7 @@ def test_given_duplicate_source_codes_when_building_catalogue_then_rejects_dupli config: RulesConfig = RulesConfig() with pytest.raises(RulesError, match=test_case.test_source) as raised: - build_catalogue(config=config, project_dir=tmp_path) + build_rule_catalogue(config=config, project_dir=tmp_path, include_custom=True) assert str(raised.value).count("XSQBRT101") == test_case.expected_count diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index daa5f077eb..9c581503f4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -14,10 +14,10 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + build_rule_catalogue, project_with, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.host import custom_host_pool from sqlbuild.rule_engine._helpers.host.module_state import module_state_token, rule_namespaces @@ -102,7 +102,10 @@ def evaluate_order_rules( select=(_MODEL_RULE, _PROJECT_RULE), cache=RulesCacheConfig(enabled=cache_enabled) ) rules: tuple[Rule, ...] = tuple( - filter(attrgetter("custom"), build_catalogue(config=config, project_dir=root)) + filter( + attrgetter("custom"), + build_rule_catalogue(config=config, project_dir=root, include_custom=True), + ) ) return evaluate_custom_rules_cached( project=project, config=config, project_dir=root, rules=rules, dialect="duckdb" @@ -365,7 +368,10 @@ def load_state_rules(*, root: Path, files: tuple[tuple[str, str], ...]) -> tuple path.parent.mkdir(parents=True, exist_ok=True) path.write_text(contents, encoding="utf-8") return tuple( - filter(attrgetter("custom"), build_catalogue(config=RulesConfig(), project_dir=root)) + filter( + attrgetter("custom"), + build_rule_catalogue(config=RulesConfig(), project_dir=root, include_custom=True), + ) ) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py deleted file mode 100644 index f98746790c..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/conftest.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Process and failure orchestration for policy skill installation tests.""" - -from collections.abc import Callable -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event -from pathlib import Path -from typing import Any -from unittest.mock import patch - -from sqlbuild.rule_engine._helpers.guidance import skill_installation -from sqlbuild.rule_engine.main import skills as skills_module -from sqlbuild.rule_engine.models import RulesConfig - - -class FailingReplacement: - def __init__(self, *, original_replace: Callable[..., Path]) -> None: - self._original_replace = original_replace - self._temporary_replacements = 0 - - def __call__(self, path: Path, target: Path) -> Path: - if path.suffix == ".tmp": - self._temporary_replacements += 1 - if self._temporary_replacements == 2: - raise OSError("replacement failed") - if path.suffix == ".bak": - raise OSError("rollback failed") - return self._original_replace(path, target) - - -class ReplacementFailure: - def __init__(self, *, original_replace: Callable[..., Path]) -> None: - self._original_replace = original_replace - self._temporary_replacements = 0 - - def __call__(self, path: Path, target: Path) -> Path: - if path.suffix == ".tmp": - self._temporary_replacements += 1 - if self._temporary_replacements == 2: - raise OSError("replacement failed") - return self._original_replace(path, target) - - -class CoordinatedReplacementFailure: - def __init__( - self, - *, - original_replace: Callable[..., Path], - failure_reached: Event, - other_writer_attempting: Event, - ) -> None: - self._original_replace = original_replace - self._failure_reached = failure_reached - self._other_writer_attempting = other_writer_attempting - self._temporary_replacements = 0 - - def __call__(self, path: Path, target: Path) -> Path: - if path.suffix == ".tmp": - self._temporary_replacements += 1 - if self._temporary_replacements == 2: - self._failure_reached.set() - if not self._other_writer_attempting.wait(timeout=20): - raise TimeoutError("timed out waiting for the other skill writer") - raise OSError("replacement failed") - return self._original_replace(path, target) - - -class EditAfterBackupRead: - def __init__(self, *, original_read: Callable[..., bytes], target: Path) -> None: - self._original_read = original_read - self._target = target - self._edited = False - - def __call__(self, path: Path) -> bytes: - content: bytes = self._original_read(path) - if path == self._target and not self._edited: - self._edited = True - path.write_text("concurrent local edit\n", encoding="utf-8") - return content - - -def install_skills_in_process( - *, - root: str, - selected_rule: str, - result_queue: Queue[Any], - attempting: Event, - staged: Event | None = None, - release: Event | None = None, -) -> None: - original_stage: Callable[..., tuple[tuple[Path, Path | None, Path], ...]] = ( - skill_installation._stage_skills - ) - - def wait_and_stage( - *, targets: tuple[Path, ...], content: str - ) -> tuple[tuple[Path, Path | None, Path], ...]: - if staged is not None: - staged.set() - if release is not None and not release.wait(timeout=20): - raise TimeoutError("timed out waiting to release staged skill installation") - return original_stage(targets=targets, content=content) - - attempting.set() - try: - with patch.object(skill_installation, "_stage_skills", side_effect=wait_and_stage): - _ = skills_module.install_skills( - config=RulesConfig(select=(selected_rule,)), - project_dir=Path(root), - check=False, - ) - except Exception as error: - result_queue.put((selected_rule, type(error).__name__, str(error))) - else: - result_queue.put((selected_rule, "ok", "")) - - -def install_failing_skills_in_process( - *, - root: str, - selected_rule: str, - result_queue: Queue[Any], - failure_reached: Event, - other_writer_attempting: Event, -) -> None: - replacement: CoordinatedReplacementFailure = CoordinatedReplacementFailure( - original_replace=Path.replace, - failure_reached=failure_reached, - other_writer_attempting=other_writer_attempting, - ) - try: - with patch.object(Path, "replace", autospec=True, side_effect=replacement): - _ = skills_module.install_skills( - config=RulesConfig(select=(selected_rule,)), - project_dir=Path(root), - check=False, - ) - except Exception as error: - result_queue.put((selected_rule, type(error).__name__, str(error))) - else: - result_queue.put((selected_rule, "ok", "")) diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index 209616fbbe..c6df6b64a3 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -1,19 +1,13 @@ """Generated rules skill freshness behavior tests.""" -from multiprocessing import get_context -from multiprocessing.context import SpawnContext -from multiprocessing.process import BaseProcess -from multiprocessing.queues import Queue -from multiprocessing.synchronize import Event from pathlib import Path from stat import S_IMODE -from unittest.mock import patch import pytest +from sqlbuild._native import install_rules_skills as install_skills from sqlbuild.rule_engine.constants import RULES_SKILL_PATHS from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.main.skills import install_skills from sqlbuild.rule_engine.models import RuleExemption, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( SkillDivergenceTestCase, @@ -21,13 +15,6 @@ SkillGuidanceTestCase, SkillInstallationTestCase, ) -from tests.unit.src.sqlbuild.rule_engine.main.skills.conftest import ( - EditAfterBackupRead, - FailingReplacement, - ReplacementFailure, - install_failing_skills_in_process, - install_skills_in_process, -) from tests.unit.src.sqlbuild.rule_engine.main.skills.helpers import write_custom_rule @@ -130,49 +117,6 @@ def test_given_later_target_failure_when_installing_then_all_targets_remain_unch assert unrelated.read_bytes() == b"keep me\n" -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="partial staging failure leaves every target unchanged", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="staging failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_staging_failure_when_installing_then_no_target_is_replaced( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - paths: tuple[Path, ...] = tuple(tmp_path / value for value in RULES_SKILL_PATHS) - expected_contents: tuple[bytes, ...] = tuple(path.read_bytes() for path in paths) - - with ( - patch.object( - Path, - "read_bytes", - side_effect=(expected_contents[0], OSError(test_case.expected_error)), - ), - pytest.raises(OSError, match=test_case.expected_error), - ): - install_skills( - config=RulesConfig(select=(test_case.changed_rule,)), - project_dir=tmp_path, - check=False, - ) - - assert tuple(path.read_bytes() for path in paths) == expected_contents - assert tuple(tmp_path.rglob("*.tmp")) == () - assert tuple(tmp_path.rglob("*.bak")) == () - - @pytest.mark.parametrize( "test_case", [ @@ -324,260 +268,3 @@ def test_given_same_policy_when_reinstalling_then_all_targets_are_byte_identical assert test_case.expected_fragment.encode() in second[0] assert test_case.absent_fragment.encode() not in second[0] assert tuple(S_IMODE(path.stat().st_mode) for path in paths) == (0o640, 0o640, 0o640) - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="different concurrent policies serialize complete installations", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="", - ) - ], - ids=lambda case: case.description, -) -def test_given_different_concurrent_policies_when_installing_then_last_writer_is_atomic( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - context: SpawnContext = get_context("spawn") - result_queue: Queue[object] = context.Queue() - first_attempting: Event = context.Event() - first_staged: Event = context.Event() - release_first: Event = context.Event() - second_attempting: Event = context.Event() - first: BaseProcess = context.Process( - target=install_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.selected_rule, - "result_queue": result_queue, - "attempting": first_attempting, - "staged": first_staged, - "release": release_first, - }, - ) - second: BaseProcess = context.Process( - target=install_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.changed_rule, - "result_queue": result_queue, - "attempting": second_attempting, - }, - ) - - first.start() - assert first_staged.wait(timeout=20) - second.start() - assert second_attempting.wait(timeout=20) - release_first.set() - first.join(timeout=20) - second.join(timeout=20) - - assert first.exitcode == 0 - assert second.exitcode == 0 - assert sorted((result_queue.get(timeout=5), result_queue.get(timeout=5))) == [ - (test_case.selected_rule, "ok", test_case.expected_error), - (test_case.changed_rule, "ok", test_case.expected_error), - ] - contents: tuple[str, ...] = tuple( - (tmp_path / value).read_text(encoding="utf-8") for value in RULES_SKILL_PATHS - ) - assert len(set(contents)) == 1 - assert test_case.changed_rule in contents[0] - assert test_case.selected_rule not in contents[0] - - -@pytest.mark.parametrize( - "test_case", - [ - SkillDivergenceTestCase( - description="target edited after staging", - selected_rule="SQBRMODEL101", - local_edit="concurrent local edit\n", - expected_error="changed during installation", - ) - ], - ids=lambda case: case.description, -) -def test_given_target_edit_between_validation_and_replacement_when_installing_then_edit_is_preserved( - tmp_path: Path, - test_case: SkillDivergenceTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - target: Path = tmp_path / RULES_SKILL_PATHS[0] - edit_after_backup: EditAfterBackupRead = EditAfterBackupRead( - original_read=Path.read_bytes, - target=target, - ) - - with ( - patch.object(Path, "read_bytes", autospec=True, side_effect=edit_after_backup), - pytest.raises(RulesError, match=test_case.expected_error), - ): - install_skills( - config=RulesConfig(select=("SQBRMODEL102",)), - project_dir=tmp_path, - check=False, - ) - - paths: tuple[Path, ...] = tuple(tmp_path / value for value in RULES_SKILL_PATHS) - assert paths[0].read_text(encoding="utf-8") == test_case.local_edit - assert test_case.selected_rule in paths[1].read_text(encoding="utf-8") - assert test_case.selected_rule in paths[2].read_text(encoding="utf-8") - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="rollback failure retains original error and recovery backup", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="replacement failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_rollback_failure_when_replacement_fails_then_original_error_and_backup_are_preserved( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - original_contents: bytes = (tmp_path / RULES_SKILL_PATHS[0]).read_bytes() - replacement: FailingReplacement = FailingReplacement(original_replace=Path.replace) - - with ( - patch.object(Path, "replace", autospec=True, side_effect=replacement), - pytest.raises(OSError, match=test_case.expected_error) as raised, - ): - install_skills( - config=RulesConfig(select=(test_case.changed_rule,)), - project_dir=tmp_path, - check=False, - ) - - backups: tuple[Path, ...] = tuple(tmp_path.rglob("*.bak")) - assert raised.value.__notes__ is not None - assert "rollback failed" in raised.value.__notes__[0] - assert len(backups) == 1 - assert backups[0].read_bytes() == original_contents - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="successful rollback restores original modes", - selected_rule="SQBRMODEL101", - changed_rule="SQBRMODEL102", - expected_error="replacement failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_replacement_failure_when_rolling_back_then_original_modes_are_restored( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=(test_case.selected_rule,)), - project_dir=tmp_path, - check=False, - ) - paths: tuple[Path, ...] = tuple(tmp_path / value for value in RULES_SKILL_PATHS) - for path in paths: - path.chmod(0o640) - replacement: ReplacementFailure = ReplacementFailure(original_replace=Path.replace) - - with ( - patch.object(Path, "replace", autospec=True, side_effect=replacement), - pytest.raises(OSError, match=test_case.expected_error), - ): - install_skills( - config=RulesConfig(select=(test_case.changed_rule,)), - project_dir=tmp_path, - check=False, - ) - - assert tuple(S_IMODE(path.stat().st_mode) for path in paths) == (0o640, 0o640, 0o640) - - -@pytest.mark.parametrize( - "test_case", - [ - SkillInstallationTestCase( - description="failed writer rolls back before successful writer installs", - selected_rule="SQBRMODEL102", - changed_rule="SQBRGRAPH101", - expected_error="replacement failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_failed_and_successful_concurrent_writers_when_installing_then_rollback_cannot_stale_overwrite( - tmp_path: Path, - test_case: SkillInstallationTestCase, -) -> None: - _ = install_skills( - config=RulesConfig(select=("SQBRMODEL101",)), - project_dir=tmp_path, - check=False, - ) - context: SpawnContext = get_context("spawn") - result_queue: Queue[object] = context.Queue() - failure_reached: Event = context.Event() - successful_writer_attempting: Event = context.Event() - failed: BaseProcess = context.Process( - target=install_failing_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.selected_rule, - "result_queue": result_queue, - "failure_reached": failure_reached, - "other_writer_attempting": successful_writer_attempting, - }, - ) - successful: BaseProcess = context.Process( - target=install_skills_in_process, - kwargs={ - "root": str(tmp_path), - "selected_rule": test_case.changed_rule, - "result_queue": result_queue, - "attempting": successful_writer_attempting, - }, - ) - - failed.start() - assert failure_reached.wait(timeout=20) - successful.start() - failed.join(timeout=20) - successful.join(timeout=20) - - assert failed.exitcode == 0 - assert successful.exitcode == 0 - results: set[tuple[str, str, str]] = { - result_queue.get(timeout=5), - result_queue.get(timeout=5), - } - assert results == { - (test_case.selected_rule, "OSError", test_case.expected_error), - (test_case.changed_rule, "ok", ""), - } - contents: tuple[str, ...] = tuple( - (tmp_path / value).read_text(encoding="utf-8") for value in RULES_SKILL_PATHS - ) - assert len(set(contents)) == 1 - assert test_case.changed_rule in contents[0] - assert test_case.selected_rule not in contents[0] From 9c0492ed7f1f5b7e884d85d5144af0cd5ddf5cb7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:16:20 +0100 Subject: [PATCH 11/32] refactor: finish rules runs, cache paths and rule result text natively --- .../_helpers/compile_driver/analysis.rs | 26 +-- .../_helpers/compile_driver/command.rs | 17 +- .../_helpers/compile_driver/pipeline.rs | 41 +---- .../src/bindings/_helpers/rules/mod.rs | 1 + .../bindings/_helpers/rules/registration.rs | 5 +- .../src/bindings/_helpers/rules/results.rs | 38 +++- .../src/bindings/_helpers/rules/rule_run.rs | 172 ++++++++++++++++++ .../bindings/_helpers/rules/rules_config.rs | 19 +- .../src/rule_cache/constants.rs | 3 + .../sqlbuild-rules/src/rule_cache/main/mod.rs | 1 + .../rule_cache/main/rules_bulk_cache_path.rs | 14 ++ .../src/rule_results/constants.rs | 1 + .../main/format_rules_progress.rs | 20 ++ .../rule_results/main/format_skipped_rules.rs | 12 ++ .../rule_results/main/group_unevaluated.rs | 60 ++++++ .../src/rule_results/main/mod.rs | 3 + .../sqlbuild-rules/src/rule_results/tests.rs | 2 + .../src/rule_results/tests/helpers.rs | 14 ++ .../rule_results/tests/test_rule_results.rs | 85 ++++++++- .../src/rule_results/tests/test_types.rs | 17 ++ .../_helpers/varied_benchmark.py | 3 +- src/sqlbuild/_native.pyi | 31 +++- .../_helpers/engine/custom_rules.py | 2 +- .../_helpers/engine/rule_harness.py | 3 +- .../rule_engine/_helpers/run/cache_paths.py | 24 --- .../rule_engine/_helpers/run/findings.py | 34 ---- .../rule_engine/_helpers/run/memory.py | 30 --- .../rule_engine/_helpers/run/native_rows.py | 8 +- .../rule_engine/_helpers/run/rule_threads.py | 3 +- .../rule_engine/_helpers/run/rules.py | 59 +----- .../rule_engine/classes/early_sql_lint.py | 2 +- .../rule_engine/main/_prepare_sql_rules.py | 18 -- src/sqlbuild/rule_engine/main/load_config.py | 12 -- .../rule_engine/main/render_result.py | 10 - .../rule_engine/main/render_rules_progress.py | 21 --- .../rule_engine/main/render_skipped_rules.py | 9 - src/sqlbuild/rule_engine/main/run_rules.py | 41 ----- .../compile/test_dense_compile_performance.py | 3 +- .../cli/commands/main/rules/helpers.py | 2 +- .../sqlbuild/cli/commands/main/test_rules.py | 2 +- .../main/test_compile_phase_timings.py | 14 +- .../_helpers/engine/test_config.py | 2 +- .../_helpers/host/test_custom_host_pool.py | 2 +- .../main/render_result/test_render_result.py | 2 +- .../main/render_rules_progress/_test_types.py | 14 -- .../test_render_rules_progress.py | 54 ------ .../main/render_skipped_rules/_test_types.py | 10 - .../test_render_skipped_rules.py | 32 ---- .../main/run_rules/test_run_rules.py | 9 +- 49 files changed, 553 insertions(+), 454 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs create mode 100644 crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs create mode 100644 crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs create mode 100644 crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs create mode 100644 crates/sqlbuild-rules/src/rule_results/tests/helpers.rs delete mode 100644 src/sqlbuild/rule_engine/_helpers/run/cache_paths.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/run/findings.py delete mode 100644 src/sqlbuild/rule_engine/_helpers/run/memory.py delete mode 100644 src/sqlbuild/rule_engine/main/_prepare_sql_rules.py delete mode 100644 src/sqlbuild/rule_engine/main/load_config.py delete mode 100644 src/sqlbuild/rule_engine/main/render_result.py delete mode 100644 src/sqlbuild/rule_engine/main/render_rules_progress.py delete mode 100644 src/sqlbuild/rule_engine/main/render_skipped_rules.py delete mode 100644 src/sqlbuild/rule_engine/main/run_rules.py delete mode 100644 tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py delete mode 100644 tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index ab86bde0ec..c13e41103c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -9,6 +9,8 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::rules::results::rules_progress; +use crate::bindings::_helpers::rules::rules_config::load_rules_config; const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; @@ -25,9 +27,7 @@ const FAST_COLUMN_LINEAGE: &str = "FAST"; const CONTRACTS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; -const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; -const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; -const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RULE_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const EARLY_LINT_MODULE: &str = "sqlbuild.rule_engine.classes.early_sql_lint"; @@ -474,10 +474,8 @@ impl<'py> Driver<'_, 'py> { let py: Python<'py> = self.py; let options: &AnalysisOptions<'py> = self.options; let project_dir: &Bound<'py, PyAny> = &options.project_dir; - let rules_config: Bound<'py, PyAny> = call_with( - &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, - &[("project_dir", project_dir.clone())], - )?; + let rules_config: Bound<'py, PyAny> = + load_rules_config(py, project_dir.extract()?)?.into_bound(py); if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; @@ -505,7 +503,7 @@ impl<'py> Driver<'_, 'py> { py.None().into_bound(py) }; let result: Bound<'py, PyAny> = call_with( - &imported(py, RUN_RULES_MODULE, "run_rules")?, + &imported(py, RULE_THREADS_MODULE, "evaluate_rules")?, &[ ("graph", graph.clone()), ("discovered_inputs", discovered.clone()), @@ -522,17 +520,7 @@ impl<'py> Driver<'_, 'py> { ], )?; let ended: f64 = monotonic.call0()?.extract()?; - let progress: String = call_with( - &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, - &[ - ( - "elapsed_seconds", - (ended - started).into_pyobject(py)?.into_any(), - ), - ("result", result.clone()), - ], - )? - .extract()?; + let progress: String = rules_progress(ended - started, &result, false)?; self.status.complete(&progress)?; Ok(result) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index d0d189dce9..17fca3ab98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -1,7 +1,7 @@ //! The `sqb compile` command: analysis, output phases, reuse recording and printed output. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compile_driver::phases::{ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; @@ -27,7 +28,6 @@ const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; -const SKIPPED_RULES_MODULE: &str = "sqlbuild.rule_engine.main.render_skipped_rules"; const SUPPORTS_COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; /// What one full compile reports for reuse recording. @@ -297,15 +297,10 @@ fn compiled_with_status<'py>( } } close_status(status)?; - let skipped_note: Bound<'py, PyAny> = call_with( - &imported(py, SKIPPED_RULES_MODULE, "format_skipped_type_proof_rules")?, - &[("codes", analysis.getattr("skipped_type_proof_rules")?)], - )?; - if !skipped_note.is_none() { - output.call_method1( - "print_stderr_line", - (format!("note: {}", skipped_note.str()?.to_cow()?),), - )?; + if let Some(skipped_note) = + format_skipped_type_proof_rules(analysis.getattr("skipped_type_proof_rules")?.extract()?) + { + output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } let dag_artifact_path: Bound<'py, PyAny> = if rules_failed || request.getattr("dag_path")?.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index fbe1f63570..2a71c1c096 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -12,6 +12,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; +use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; +use crate::bindings::_helpers::rules::rules_config::load_rules_config; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -21,10 +23,7 @@ const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; const RULE_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; -const RULES_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; -const RULES_PROGRESS_MODULE: &str = "sqlbuild.rule_engine.main.render_rules_progress"; -const RULES_RESULT_MODULE: &str = "sqlbuild.rule_engine.main.render_result"; -const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const OPEN_CONNECTION_MODULE: &str = "sqlbuild.runtime.contracts.main.open_connection"; const RUNTIME_MODELS_MODULE: &str = "sqlbuild.runtime.contracts.models"; @@ -259,10 +258,8 @@ fn run_rules_gate<'py>( rules_error.call1(("compiler rules require a resolved project directory",))?, )); } - let mut config: Bound<'py, PyAny> = call_with( - &imported(py, RULES_CONFIG_MODULE, "load_rules_config")?, - &[("project_dir", project_dir.clone())], - )?; + let mut config: Bound<'py, PyAny> = + load_rules_config(py, project_dir.extract()?)?.into_bound(py); if options.getattr("no_cache")?.is_truthy()? { let cache: Bound<'py, PyAny> = call_with( &imported(py, RULE_MODELS_MODULE, "RulesCacheConfig")?, @@ -292,7 +289,7 @@ fn run_rules_gate<'py>( ], )?; let result: Bound<'py, PyAny> = call_with( - &imported(py, RUN_RULES_MODULE, "run_rules")?, + &imported(py, RULE_THREADS_MODULE, "evaluate_rules")?, &[ ("graph", graph.clone()), ("discovered_inputs", discovered.clone()), @@ -305,17 +302,7 @@ fn run_rules_gate<'py>( )?; if !on_progress.is_none() { let elapsed: f64 = monotonic(py)? - started; - let message: Bound<'py, PyAny> = call_with( - &imported(py, RULES_PROGRESS_MODULE, "format_rules_progress")?, - &[ - ("elapsed_seconds", elapsed.into_pyobject(py)?.into_any()), - ("result", result.clone()), - ( - "note_skipped_rules", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], - )?; + let message: String = rules_progress(elapsed, &result, true)?; on_progress.call1((message,))?; } let findings: Bound<'py, PyAny> = result.getattr("findings")?; @@ -335,19 +322,9 @@ fn run_rules_gate<'py>( ("cache_misses", result.getattr("cache_misses")?), ], )?; - let rendered: Bound<'py, PyAny> = call_with( - &imported(py, RULES_RESULT_MODULE, "format_result")?, - &[ - ("result", summary), - ( - "json_output", - false.into_pyobject(py)?.to_owned().into_any(), - ), - ], - )?; + let rendered: String = format_rules_result(&summary, false)?; Err(PyErr::from_value(rules_error.call1((format!( - "configured compiler rules failed:\n{}", - rendered.str()? + "configured compiler rules failed:\n{rendered}" ),))?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index ec1a2b0d2c..13cae7439d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -15,6 +15,7 @@ pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; pub(crate) mod rule_cache; +pub(crate) mod rule_run; pub(crate) mod rules_config; pub(crate) mod run_lint; pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 0aca5e2a68..27a85fab45 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -4,8 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, - guidance, lint, lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rules_config, - run_lint, selection, + guidance, lint, lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rule_run, + rules_config, run_lint, selection, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -17,6 +17,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { dsl_headers::register(module)?; custom_rule_evidence::register(module)?; rule_cache::register(module)?; + rule_run::register(module)?; custom_rule_cache::register(module)?; builtin_catalogue::register(module)?; catalogue::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs index 5a0bcbd5ec..9e5c883e7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/results.rs @@ -6,6 +6,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; use sqlbuild_rules::rule_results::main::format_result::format_result; +use sqlbuild_rules::rule_results::main::format_rules_progress::format_rules_progress; +use sqlbuild_rules::rule_results::main::format_skipped_rules::format_skipped_rules; use sqlbuild_rules::rule_results::models::{ResultCounts, RuleFinding}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; @@ -13,7 +15,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const DATACLASSES_MODULE: &str = "dataclasses"; const UTF8: &str = "utf-8"; -fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn rule_finding(finding: &Bound<'_, PyAny>) -> PyResult { Ok(RuleFinding { code: finding.getattr("code")?.extract()?, path: finding @@ -40,7 +42,10 @@ fn rule_findings(findings: &Bound<'_, PyAny>) -> PyResult> { /// One rules result for the requested CLI surface. #[pyfunction] #[pyo3(signature = (*, result, json_output))] -fn format_rules_result(result: &Bound<'_, PyAny>, json_output: bool) -> PyResult { +pub(crate) fn format_rules_result( + result: &Bound<'_, PyAny>, + json_output: bool, +) -> PyResult { let counts = ResultCounts { evaluated_models: result.getattr("evaluated_models")?.extract()?, unevaluated_resources: result.getattr("unevaluated_resources")?.extract()?, @@ -54,6 +59,34 @@ fn format_rules_result(result: &Bound<'_, PyAny>, json_output: bool) -> PyResult )) } +/// The rules phase completion line for one `RulesRunResult`, optionally noting skipped rules. +pub(crate) fn rules_progress( + elapsed_seconds: f64, + result: &Bound<'_, PyAny>, + note_skipped_rules: bool, +) -> PyResult { + let skipped: Option> = if note_skipped_rules { + Some(result.getattr("skipped_type_proof_rules")?.extract()?) + } else { + None + }; + Ok(format_rules_progress( + elapsed_seconds, + ( + result.getattr("built_in_ms")?.extract()?, + result.getattr("custom_ms")?.extract()?, + ), + skipped.as_deref(), + )) +} + +/// Describe type-proof rules skipped because SQL analysis is disabled, if any. +#[pyfunction] +#[pyo3(signature = (*, codes))] +pub(crate) fn format_skipped_type_proof_rules(codes: Vec) -> Option { + format_skipped_rules(&codes) +} + /// The authored lines of one finding's file; an unreadable file has none. fn file_lines(project_dir: &Bound<'_, PyAny>, path: &str) -> PyResult> { let py = project_dir.py(); @@ -101,6 +134,7 @@ fn with_duplicate_literal_hints<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(format_rules_result, module)?)?; + module.add_function(wrap_pyfunction!(format_skipped_type_proof_rules, module)?)?; module.add_function(wrap_pyfunction!(with_duplicate_literal_hints, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs new file mode 100644 index 0000000000..6797c7f30c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -0,0 +1,172 @@ +//! The finished result of one rules run, the bulk cache locations and freed-memory release. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path as bulk_cache_path; +use sqlbuild_rules::rule_results::main::group_unevaluated::group_unevaluated; +use sqlbuild_rules::rule_results::models::RuleFinding; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::rules::results::rule_finding; + +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; +const NATIVE_ROWS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.native_rows"; +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const PATHLIB_MODULE: &str = "pathlib"; +const CTYPES_MODULE: &str = "ctypes"; + +fn integer(object: &Bound<'_, PyAny>, field: &str) -> PyResult { + object.getattr(field)?.extract() +} + +fn python_finding<'py>(py: Python<'py>, finding: RuleFinding) -> PyResult> { + call_with( + &imported(py, MODELS_MODULE, "Finding")?, + &[ + ("code", finding.code.into_bound_py_any(py)?), + ( + "path", + imported(py, PATHLIB_MODULE, "Path")?.call1((finding.path,))?, + ), + ("line", finding.line.into_bound_py_any(py)?), + ("column", finding.column.into_bound_py_any(py)?), + ("message", finding.message.into_bound_py_any(py)?), + ("remediation", finding.remediation.into_bound_py_any(py)?), + ("unevaluated", finding.unevaluated.into_bound_py_any(py)?), + ( + "affected_rules", + PyTuple::new(py, finding.affected_rules)?.into_any(), + ), + ("fixable", finding.fixable.into_bound_py_any(py)?), + ], + ) +} + +/// Finalize the combined SQL, built-in and custom findings of one planned rules run. +#[pyfunction] +#[pyo3(signature = (*, plan, sql_result, result))] +fn finish_rule_run<'py>( + plan: &Bound<'py, PyAny>, + sql_result: &Bound<'py, PyAny>, + result: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = plan.py(); + let mut evaluated_codes: Vec> = Vec::new(); + for rule in plan.getattr("selected")?.try_iter()? { + evaluated_codes.push(rule?.getattr("code")?); + } + let mut combined: Vec> = Vec::new(); + for findings in [sql_result.getattr("findings")?, result.getattr("findings")?] { + combined.extend(findings.try_iter()?.collect::>>()?); + } + let finalized: Vec = call_with( + &imported(py, NATIVE_ROWS_MODULE, "finalize_rules_rows")?, + &[ + ("config", plan.getattr("config")?), + ("project_dir", plan.getattr("project_dir")?), + ( + "evaluated_codes", + PyTuple::new(py, evaluated_codes)?.into_any(), + ), + ("findings", PyTuple::new(py, combined)?.into_any()), + ], + )? + .try_iter()? + .map(|finding| rule_finding(&finding?)) + .collect::>()?; + let grouped: Vec = compiler_guard(|| Ok(group_unevaluated(finalized)))?; + let unevaluated: BTreeSet = grouped + .iter() + .filter(|finding| finding.unevaluated) + .map(|finding| finding.path.clone()) + .collect(); + let mut unevaluated_models: i64 = 0; + for model in plan.getattr("project")?.getattr("models")?.try_iter()? { + let path: String = model? + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?; + unevaluated_models += i64::from(unevaluated.contains(&path)); + } + let findings: Vec> = grouped + .into_iter() + .map(|finding| python_finding(py, finding)) + .collect::>()?; + let total = |field: &str| -> PyResult> { + (integer(result, field)? + integer(sql_result, field)?).into_bound_py_any(py) + }; + call_with( + &imported(py, MODELS_MODULE, "RulesRunResult")?, + &[ + ("findings", PyTuple::new(py, findings)?.into_any()), + ( + "evaluated_models", + (integer(result, "evaluated_models")? - unevaluated_models) + .max(0) + .into_bound_py_any(py)?, + ), + ( + "unevaluated_resources", + unevaluated.len().into_bound_py_any(py)?, + ), + ( + "built_in_ms", + (integer(sql_result, "sql_ms")? + integer(result, "built_in_ms")?) + .into_bound_py_any(py)?, + ), + ("custom_ms", result.getattr("custom_ms")?), + ("custom_cpu_ms", result.getattr("custom_cpu_ms")?), + ("cache_hits", total("cache_hits")?), + ("cache_misses", total("cache_misses")?), + ( + "skipped_type_proof_rules", + plan.getattr("skipped_type_proof_rules")?, + ), + ], + ) +} + +/// Run the selected built-in, SQL and custom rules; custom rules run on their worker threads. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn run_rules<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + imported(py, RULE_THREADS_MODULE, "evaluate_rules")?.call((), options.as_ref()) +} + +/// One bulk Rules cache file owned by the native compiler engine. +#[pyfunction] +#[pyo3(signature = (*, project_dir, file_name))] +fn rules_bulk_cache_path(project_dir: PathBuf, file_name: &str) -> PathBuf { + bulk_cache_path(&project_dir, file_name) +} + +/// Hand freed glibc heap pages back to the OS; a no-op elsewhere or without `malloc_trim`. +#[pyfunction] +fn release_freed_memory(py: Python<'_>) -> PyResult<()> { + if !cfg!(target_os = "linux") { + return Ok(()); + } + let trim = || -> PyResult<()> { + let process: Bound<'_, PyAny> = imported(py, CTYPES_MODULE, "CDLL")?.call1((py.None(),))?; + let _ = process.getattr("malloc_trim")?.call1((0,))?; + Ok(()) + }; + let _ = trim(); + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(finish_rule_run, module)?)?; + module.add_function(wrap_pyfunction!(run_rules, module)?)?; + module.add_function(wrap_pyfunction!(rules_bulk_cache_path, module)?)?; + module.add_function(wrap_pyfunction!(release_freed_memory, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 436ffbffa2..f70369c857 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -243,7 +243,10 @@ fn rules_error(py: Python<'_>, class: &str, message: String, help: Option, project_dir: PathBuf) -> PyResult> { +pub(crate) fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult> { + let project_dir: PathBuf = project_dir + .canonicalize() + .or_else(|_| std::path::absolute(&project_dir))?; let config: RulesConfig = compiler_guard(|| Ok(load_config(&project_dir)))? .map_err(|message| rules_error(py, "RulesError", message, None))?; if let Some(error) = forbidden_exceptions(&config) { @@ -257,7 +260,21 @@ fn load_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult Ok(config_object(py, &config)?.unbind()) } +/// The project's rules configuration, or `None` when it is invalid and the rules phase reports it. +#[pyfunction] +#[pyo3(signature = (*, project_dir))] +fn valid_rules_config(py: Python<'_>, project_dir: PathBuf) -> PyResult>> { + match load_rules_config(py, project_dir) { + Ok(config) => Ok(Some(config)), + Err(error) if error.is_instance(py, &py.import(ERRORS_MODULE)?.getattr("RulesError")?) => { + Ok(None) + } + Err(error) => Err(error), + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(load_rules_config, module)?)?; + module.add_function(wrap_pyfunction!(valid_rules_config, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-rules/src/rule_cache/constants.rs b/crates/sqlbuild-rules/src/rule_cache/constants.rs index 4e3c6cd34a..4b5361cf03 100644 --- a/crates/sqlbuild-rules/src/rule_cache/constants.rs +++ b/crates/sqlbuild-rules/src/rule_cache/constants.rs @@ -13,3 +13,6 @@ pub(crate) const BUILD_IDENTITY_KEY: &str = "build_identity"; pub(crate) const ENTRIES_KEY: &str = "entries"; pub(crate) const IDENTITY_KEY: &str = "identity"; pub(crate) const FINDINGS_KEY: &str = "findings"; +pub(crate) const TARGET_DIRECTORY: &str = "target"; +pub(crate) const RULES_CACHE_DIRECTORY: &str = "rules-cache-native-v1"; +pub(crate) const BULK_DIRECTORY: &str = "bulk"; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs index 528348321d..20a4012180 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -4,3 +4,4 @@ pub mod lookup_rule_cache; pub mod model_rule_key; pub mod parse_rule_cache; pub mod rule_cache_text; +pub mod rules_bulk_cache_path; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs b/crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs new file mode 100644 index 0000000000..070c25aeeb --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/rules_bulk_cache_path.rs @@ -0,0 +1,14 @@ +//! Where one bulk Rules cache file lives, namespaced by the native compiler engine. + +use std::path::{Path, PathBuf}; + +use crate::rule_cache::constants::{BULK_DIRECTORY, RULES_CACHE_DIRECTORY, TARGET_DIRECTORY}; + +/// `/target/rules-cache-native-v1/bulk/`. +pub fn rules_bulk_cache_path(project_dir: &Path, file_name: &str) -> PathBuf { + project_dir + .join(TARGET_DIRECTORY) + .join(RULES_CACHE_DIRECTORY) + .join(BULK_DIRECTORY) + .join(file_name) +} diff --git a/crates/sqlbuild-rules/src/rule_results/constants.rs b/crates/sqlbuild-rules/src/rule_results/constants.rs index e2bcd24e2a..55a1a8d737 100644 --- a/crates/sqlbuild-rules/src/rule_results/constants.rs +++ b/crates/sqlbuild-rules/src/rule_results/constants.rs @@ -13,3 +13,4 @@ pub(crate) const FIX_AVAILABLE_NOTE: &str = "Fix available: `sqb format --fix` applies it after compiler verification"; pub(crate) const JSON_INDENT: usize = 2; pub(crate) const CURRENT_DIRECTORY: &str = "."; +pub(crate) const UNEVALUATED_CODE: &str = "rules-unevaluated"; diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs b/crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs new file mode 100644 index 0000000000..5eda51af7b --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_rules_progress.rs @@ -0,0 +1,20 @@ +//! The rules phase completion line. + +use crate::rule_results::main::format_skipped_rules::format_skipped_rules; + +/// The rules phase wall time first, then built-in and custom rule time, with the skipped note. +pub fn format_rules_progress( + elapsed_seconds: f64, + (built_in_ms, custom_ms): (i64, i64), + skipped_rules: Option<&[String]>, +) -> String { + let message: String = format!( + "Evaluated rules. ({elapsed_seconds:.2}s; built-in {:.2}s, custom {:.2}s)", + built_in_ms as f64 / 1000.0, + custom_ms as f64 / 1000.0 + ); + match skipped_rules.and_then(format_skipped_rules) { + Some(note) => format!("{message}; {note}"), + None => message, + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs new file mode 100644 index 0000000000..81490cd695 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/format_skipped_rules.rs @@ -0,0 +1,12 @@ +//! The note naming type-proof rules skipped because SQL analysis is disabled. + +/// The skipped type-proof rules note, or `None` when no rule was skipped. +pub fn format_skipped_rules(codes: &[String]) -> Option { + if codes.is_empty() { + return None; + } + Some(format!( + "skipped type-proof rules ({}) because SQL analysis is disabled", + codes.join(", ") + )) +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs b/crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs new file mode 100644 index 0000000000..129a372513 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/main/group_unevaluated.rs @@ -0,0 +1,60 @@ +//! Unevaluated findings grouped per resource and reason, in deterministic finding order. + +use crate::rule_results::constants::UNEVALUATED_CODE; +use crate::rule_results::models::RuleFinding; + +/// One `rules-unevaluated` finding per path and reason, then every finding sorted by location. +pub fn group_unevaluated(findings: Vec) -> Vec { + let mut result: Vec = Vec::with_capacity(findings.len()); + let mut groups: Vec<((String, String), Vec)> = Vec::new(); + for finding in findings { + if !finding.unevaluated { + result.push(finding); + continue; + } + let prefix: String = format!("Rule {} could not be evaluated", finding.code); + let reason: String = finding + .message + .strip_prefix(&prefix) + .unwrap_or(&finding.message) + .trim_start_matches([':', ' ']) + .to_owned(); + let key: (String, String) = (finding.path.clone(), reason); + match groups.iter_mut().find(|(existing, _)| *existing == key) { + Some((_, failures)) => failures.push(finding), + None => groups.push((key, vec![finding])), + } + } + for ((_, reason), failures) in groups { + let mut affected: Vec = failures.iter().map(|f| f.code.clone()).collect(); + affected.sort(); + affected.dedup(); + let Some(first) = failures.into_iter().reduce(|best, item| { + if (item.line, item.column) < (best.line, best.column) { + item + } else { + best + } + }) else { + continue; + }; + result.push(RuleFinding { + code: UNEVALUATED_CODE.into(), + message: format!( + "{} selected Rules could not evaluate this resource: {reason}", + affected.len() + ), + affected_rules: affected, + ..first + }); + } + result.sort_by(|left, right| { + (&left.path, left.line, left.column, &left.code).cmp(&( + &right.path, + right.line, + right.column, + &right.code, + )) + }); + result +} diff --git a/crates/sqlbuild-rules/src/rule_results/main/mod.rs b/crates/sqlbuild-rules/src/rule_results/main/mod.rs index b7461bbc4e..a361ef467b 100644 --- a/crates/sqlbuild-rules/src/rule_results/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_results/main/mod.rs @@ -1,2 +1,5 @@ pub mod duplicate_literal_hints; pub mod format_result; +pub mod format_rules_progress; +pub mod format_skipped_rules; +pub mod group_unevaluated; diff --git a/crates/sqlbuild-rules/src/rule_results/tests.rs b/crates/sqlbuild-rules/src/rule_results/tests.rs index d63fc626d1..b007d17f4e 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_rule_results.rs"] mod rule_results; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules/src/rule_results/tests/helpers.rs b/crates/sqlbuild-rules/src/rule_results/tests/helpers.rs new file mode 100644 index 0000000000..e251d603fa --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_results/tests/helpers.rs @@ -0,0 +1,14 @@ +use crate::rule_results::models::RuleFinding; + +/// One unevaluated finding of `code` in `models/a.sql` at `line`. +pub(crate) fn unevaluated(code: &str, line: i64) -> RuleFinding { + RuleFinding { + code: code.into(), + path: "models/a.sql".into(), + line, + column: 1, + message: format!("Rule {code} could not be evaluated: too deep"), + unevaluated: true, + ..RuleFinding::default() + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs index cc70c1672c..97038120ff 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_rule_results.rs @@ -2,8 +2,13 @@ use std::convert::Infallible; use crate::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; use crate::rule_results::main::format_result::format_result; +use crate::rule_results::main::format_rules_progress::format_rules_progress; +use crate::rule_results::main::group_unevaluated::group_unevaluated; use crate::rule_results::models::{ResultCounts, RuleFinding}; -use crate::rule_results::tests::test_types::{HintTestCase, ResultTestCase}; +use crate::rule_results::tests::helpers::unevaluated; +use crate::rule_results::tests::test_types::{ + GroupTestCase, HintTestCase, ProgressTestCase, ResultTestCase, +}; const SUFFIX: &str = " and reference it with `@const` in each file. R."; @@ -124,3 +129,81 @@ fn given_rules_result_when_formatting_then_matches_python_text_and_json() { ); } } + +#[test] +fn given_rules_run_when_formatting_progress_then_reports_phase_wall_time_first() { + let test_cases = [ + ProgressTestCase { + description: "wall time leads concurrent built-in and custom rule time", + elapsed_seconds: 14.5, + timings_ms: (5_600, 13_400), + skipped_rules: None, + expected_message: "Evaluated rules. (14.50s; built-in 5.60s, custom 13.40s)", + }, + ProgressTestCase { + description: "skipped type-proof rules follow the timings when requested", + elapsed_seconds: 0.25, + timings_ms: (100, 0), + skipped_rules: Some(vec!["SQBRCONTRACT105".into()]), + expected_message: "Evaluated rules. (0.25s; built-in 0.10s, custom 0.00s); skipped type-proof rules (SQBRCONTRACT105) because SQL analysis is disabled", + }, + ProgressTestCase { + description: "no skipped rules add no note", + elapsed_seconds: 1.0, + timings_ms: (0, 0), + skipped_rules: Some(vec![]), + expected_message: "Evaluated rules. (1.00s; built-in 0.00s, custom 0.00s)", + }, + ]; + for test_case in test_cases { + let message: String = format_rules_progress( + test_case.elapsed_seconds, + test_case.timings_ms, + test_case.skipped_rules.as_deref(), + ); + assert_eq!( + message, test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unevaluated_findings_when_grouping_then_one_finding_per_path_and_reason() { + let test_cases = [GroupTestCase { + description: "two rules failing for one reason group into one sorted finding", + findings: vec![ + unevaluated("SQBRB", 3), + RuleFinding { + code: "SQBRZ".into(), + path: "models/a.sql".into(), + line: 1, + ..RuleFinding::default() + }, + unevaluated("SQBRA", 2), + ], + expected_codes: vec!["SQBRZ", "rules-unevaluated"], + expected_messages: vec![ + "", + "2 selected Rules could not evaluate this resource: too deep", + ], + }]; + for test_case in test_cases { + let grouped: Vec = group_unevaluated(test_case.findings); + let codes: Vec<&str> = grouped + .iter() + .map(|finding| finding.code.as_str()) + .collect(); + let messages: Vec<&str> = grouped + .iter() + .map(|finding| finding.message.as_str()) + .collect(); + assert_eq!(codes, test_case.expected_codes, "{}", test_case.description); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs index 97f61de3cc..7d1a7ce23d 100644 --- a/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/rule_results/tests/test_types.rs @@ -1,3 +1,5 @@ +use crate::rule_results::models::RuleFinding; + pub(crate) struct HintTestCase { pub(crate) description: &'static str, pub(crate) paths: Vec<&'static str>, @@ -9,3 +11,18 @@ pub(crate) struct ResultTestCase { pub(crate) json_output: bool, pub(crate) expected_text: &'static str, } + +pub(crate) struct ProgressTestCase { + pub(crate) description: &'static str, + pub(crate) elapsed_seconds: f64, + pub(crate) timings_ms: (i64, i64), + pub(crate) skipped_rules: Option>, + pub(crate) expected_message: &'static str, +} + +pub(crate) struct GroupTestCase { + pub(crate) description: &'static str, + pub(crate) findings: Vec, + pub(crate) expected_codes: Vec<&'static str>, + pub(crate) expected_messages: Vec<&'static str>, +} diff --git a/scripts/cold_compile_performance/_helpers/varied_benchmark.py b/scripts/cold_compile_performance/_helpers/varied_benchmark.py index 3cc0fcc722..7fe7dd708c 100644 --- a/scripts/cold_compile_performance/_helpers/varied_benchmark.py +++ b/scripts/cold_compile_performance/_helpers/varied_benchmark.py @@ -7,8 +7,7 @@ from scripts.cold_compile_performance._helpers.dense_project import dense_model_name from scripts.cold_compile_performance._helpers.varied_project import write_varied_compile_project from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError -from sqlbuild._native import build_rule_catalogue, select_catalogue_rules -from sqlbuild.rule_engine.main.load_config import load_rules_config +from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import VariedCompileCacheTestCase from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0ab6974209..33cdd8868b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -332,7 +332,17 @@ from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.rule_engine.models import Finding, Rule, RulesConfig, RulesResult +from sqlbuild.rule_engine.models import ( + Finding, + PreparedSqlLint, + Rule, + RuleRunPlan, + RulesConfig, + RulesResult, + RulesRunResult, + SqlExpansionReuse, + SqlRulesEvaluation, +) from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.models import LifecycleEvent @@ -2000,6 +2010,7 @@ def apply_lint_suppressions( allow_suppressions: bool, ) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... +def valid_rules_config(*, project_dir: Path) -> RulesConfig | None: ... def selected_rule_codes( *, config: RulesConfig, custom_payloads: list[dict[str, object]] ) -> list[str]: ... @@ -3814,6 +3825,24 @@ def custom_rule_test_evidence( *, rule: Rule, project_dir: Path ) -> list[tuple[str, int, int, str]]: ... def format_rules_result(*, result: RulesResult, json_output: bool) -> str: ... +def format_skipped_type_proof_rules(*, codes: tuple[str, ...]) -> str | None: ... +def finish_rule_run( + *, plan: RuleRunPlan, sql_result: SqlRulesEvaluation, result: RulesResult +) -> RulesRunResult: ... +def run_rules( + *, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + config: RulesConfig, + project_dir: Path, + dialect: str, + selected_keys: frozenset[CompiledObjectKey] | None = None, + prepared_sql: PreparedSqlLint | None = None, + expansion_reuse: SqlExpansionReuse | None = None, + no_sql_analysis: bool = False, +) -> RulesRunResult: ... +def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: ... +def release_freed_memory() -> None: ... def with_duplicate_literal_hints( *, findings: tuple[Finding, ...], project_dir: Path ) -> tuple[Finding, ...]: ... diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 140c20c2cd..5acdaa97b9 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -15,6 +15,7 @@ custom_rule_identity, custom_rule_implementation_fingerprint, custom_rule_import_closure, + rules_bulk_cache_path, ) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView @@ -22,7 +23,6 @@ from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys, is_tree_fact from sqlbuild.rule_engine._helpers.host.custom_host_pool import run_custom_hosts -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index 2d039faef4..6717de9446 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -6,7 +6,7 @@ from pathlib import Path from tempfile import TemporaryDirectory -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import build_compiled_project, discover_project_inputs, load_rules_config from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -17,7 +17,6 @@ from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.exceptions import RuleAssertionError, RuleUsageError from sqlbuild.rule_engine.main._evaluate import evaluate -from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( Rule, RuleCase, diff --git a/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py b/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py deleted file mode 100644 index ea496307ac..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Locations of the Rules caches, namespaced by compiler engine.""" - -from pathlib import Path - -from sqlbuild.compiler.frontier.constants import ( - NATIVE_CACHE_NAMESPACE_SUFFIX, - TARGET_DIRECTORY_NAME, -) -from sqlbuild.rule_engine.constants import ( - RULES_BULK_CACHE_DIRECTORY_NAME, - RULES_CACHE_DIRECTORY_NAME, -) - - -def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: - """Return one bulk Rules cache file owned by the active compiler engine.""" - - return ( - project_dir - / TARGET_DIRECTORY_NAME - / f"{RULES_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}" - / RULES_BULK_CACHE_DIRECTORY_NAME - / file_name - ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/findings.py b/src/sqlbuild/rule_engine/_helpers/run/findings.py deleted file mode 100644 index db350a1de3..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/findings.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public evaluation faults grouped after per-Rule suppression policy.""" - -from dataclasses import replace -from pathlib import Path - -from sqlbuild.rule_engine.models import Finding - - -def group_unevaluated_findings(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: - grouped: dict[tuple[Path, str], list[Finding]] = {} - result: list[Finding] = [] - for finding in findings: - if not finding.unevaluated: - result.append(finding) - continue - reason: str = finding.message.removeprefix( - f"Rule {finding.code} could not be evaluated" - ).lstrip(": ") - grouped.setdefault((finding.path, reason), []).append(finding) - for (_, reason), failures in grouped.items(): - affected_rules: tuple[str, ...] = tuple(sorted({failure.code for failure in failures})) - first: Finding = min(failures, key=lambda failure: (failure.line, failure.column)) - result.append( - replace( - first, - code="rules-unevaluated", - affected_rules=affected_rules, - message=( - f"{len(affected_rules)} selected Rules could not evaluate this resource: " - f"{reason}" - ), - ) - ) - return tuple(result) diff --git a/src/sqlbuild/rule_engine/_helpers/run/memory.py b/src/sqlbuild/rule_engine/_helpers/run/memory.py deleted file mode 100644 index ddc6c45869..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/memory.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Return memory the C allocator keeps after earlier compile phases freed it.""" - -from __future__ import annotations - -import ctypes -import sys -from collections.abc import Callable -from functools import cache -from typing import Any - - -def release_freed_memory() -> None: - """Hand freed glibc heap pages back to the OS; a no-op on other platforms and allocators.""" - - trim: Callable[[int], int] | None = _malloc_trim() - if trim is not None: - _ = trim(0) - - -@cache -def _malloc_trim() -> Callable[[int], int] | None: - if not sys.platform.startswith("linux"): - return None - try: - trim: Any = ctypes.CDLL(None).malloc_trim - except (OSError, AttributeError): - return None - trim.argtypes = (ctypes.c_size_t,) - trim.restype = ctypes.c_int - return trim diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index 7af6ef3860..704ed1f96e 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,11 +9,15 @@ import orjson import sqlbuild._native as _native -from sqlbuild._native import compiled_project_facts, declaration_rows, scope_metadata_projection +from sqlbuild._native import ( + compiled_project_facts, + declaration_rows, + rules_bulk_cache_path, + scope_metadata_projection, +) from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import ( NATIVE_RULES_CACHE_FILE, NATIVE_RULES_MEMO_FILE, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 962a16c4ad..7810636dfb 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -7,7 +7,7 @@ from pathlib import Path from threading import Event -from sqlbuild._native import build_lint_expansion_context, run_lint +from sqlbuild._native import build_lint_expansion_context, finish_rule_run, run_lint from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -22,7 +22,6 @@ from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules from sqlbuild.rule_engine._helpers.run.rules import ( early_lint_expansions, - finish_rule_run, plan_rule_run, run_planned_sql_rules, sql_lint_preparation, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index d89d114471..aff62ac0c0 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -17,13 +17,16 @@ load_sql_rule_cache, lookup_sql_rule_cache, project_with, + release_freed_memory, resolve_rule_ignore_selectors, + rules_bulk_cache_path, run_lint, select_catalogue_rules, selected_sql_rule_codes, sql_file_rule_key, sql_model_rule_keys, sql_rule_lint_key, + valid_rules_config, with_duplicate_literal_hints, write_sql_rule_cache, ) @@ -54,13 +57,7 @@ from sqlbuild.lint.types import RelationKeys from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings -from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory -from sqlbuild.rule_engine._helpers.run.native_rows import finalize_rules_rows from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( Finding, PreparedSqlLint, @@ -68,8 +65,6 @@ Rule, RuleRunPlan, RulesConfig, - RulesResult, - RulesRunResult, SqlExpansionReuse, SqlRulesEvaluation, ) @@ -191,43 +186,6 @@ def run_planned_sql_rules( return replace(sql_result, sql_ms=round((time.monotonic() - sql_started) * 1000)) -def finish_rule_run( - *, plan: RuleRunPlan, sql_result: SqlRulesEvaluation, result: RulesResult -) -> RulesRunResult: - """Finalize the combined SQL, built-in and custom findings of one run.""" - - finalized: tuple[Finding, ...] = finalize_rules_rows( - config=plan.config, - project_dir=plan.project_dir, - evaluated_codes=tuple(rule.code for rule in plan.selected), - findings=(*sql_result.findings, *result.findings), - ) - findings: tuple[Finding, ...] = tuple( - sorted( - group_unevaluated_findings(finalized), - key=lambda item: (item.path.as_posix(), item.line, item.column, item.code), - ) - ) - return RulesRunResult( - findings=findings, - evaluated_models=max( - 0, - result.evaluated_models - - len( - {item.path for item in findings if item.unevaluated} - & {model.relative_path for model in plan.project.models} - ), - ), - unevaluated_resources=len({item.path for item in findings if item.unevaluated}), - built_in_ms=sql_result.sql_ms + result.built_in_ms, - custom_ms=result.custom_ms, - custom_cpu_ms=result.custom_cpu_ms, - cache_hits=result.cache_hits + sql_result.cache_hits, - cache_misses=result.cache_misses + sql_result.cache_misses, - skipped_type_proof_rules=plan.skipped_type_proof_rules, - ) - - def _prepare_sql_rule_inputs( *, rules: tuple[Rule, ...], @@ -274,11 +232,12 @@ def sql_lint_preparation( project_dir: Path | None = inputs.discovered_inputs.project_dir if project_dir is None or any(diagnostic.is_error for diagnostic in inputs.diagnostics): return None - try: - config: RulesConfig = load_rules_config(project_dir=project_dir) - except RulesError: - return None - if not config.select or (config.cache.enabled and _read_sql_rule_cache(project_dir)): + config: RulesConfig | None = valid_rules_config(project_dir=project_dir) + if ( + config is None + or not config.select + or (config.cache.enabled and _read_sql_rule_cache(project_dir)) + ): return None codes: tuple[str, ...] = _selected_sql_codes(config) if not codes: diff --git a/src/sqlbuild/rule_engine/classes/early_sql_lint.py b/src/sqlbuild/rule_engine/classes/early_sql_lint.py index 1e0cd3d9c7..2b319c38aa 100644 --- a/src/sqlbuild/rule_engine/classes/early_sql_lint.py +++ b/src/sqlbuild/rule_engine/classes/early_sql_lint.py @@ -8,7 +8,7 @@ from types import TracebackType from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine.main._prepare_sql_rules import prepare_sql_rules +from sqlbuild.rule_engine._helpers.run.rule_threads import prepare_sql_rules from sqlbuild.rule_engine.models import PreparedSqlLint, SqlExpansionReuse diff --git a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py deleted file mode 100644 index c9be51978e..0000000000 --- a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Start reusable SQL lint work after all compiler inputs have been expanded.""" - -from __future__ import annotations - -from concurrent.futures import Executor -from threading import Event - -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine._helpers.run.rule_threads import prepare_sql_rules as prepare -from sqlbuild.rule_engine.models import PreparedSqlLint - - -def prepare_sql_rules( - *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event -) -> PreparedSqlLint | None: - """Prepare configured SQL checks without publishing findings before compilation finishes.""" - - return prepare(inputs=inputs, executor=executor, dialect=dialect, stop=stop) diff --git a/src/sqlbuild/rule_engine/main/load_config.py b/src/sqlbuild/rule_engine/main/load_config.py deleted file mode 100644 index eb3a354929..0000000000 --- a/src/sqlbuild/rule_engine/main/load_config.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Rules configuration loading entrypoint.""" - -from pathlib import Path - -from sqlbuild._native import load_rules_config as _load_rules_config -from sqlbuild.rule_engine.models import RulesConfig - - -def load_rules_config(*, project_dir: Path) -> RulesConfig: - """Load strict rules configuration from a SQLBuild project.""" - - return _load_rules_config(project_dir=project_dir.resolve()) diff --git a/src/sqlbuild/rule_engine/main/render_result.py b/src/sqlbuild/rule_engine/main/render_result.py deleted file mode 100644 index bb6afd1c1c..0000000000 --- a/src/sqlbuild/rule_engine/main/render_result.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Rules result formatting entrypoint.""" - -from sqlbuild._native import format_rules_result -from sqlbuild.rule_engine.models import RulesResult - - -def format_result(*, result: RulesResult, json_output: bool) -> str: - """Format one rules result for the requested CLI surface.""" - - return format_rules_result(result=result, json_output=json_output) diff --git a/src/sqlbuild/rule_engine/main/render_rules_progress.py b/src/sqlbuild/rule_engine/main/render_rules_progress.py deleted file mode 100644 index 11f4bc5531..0000000000 --- a/src/sqlbuild/rule_engine/main/render_rules_progress.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Rules phase completion line entrypoint.""" - -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -from sqlbuild.rule_engine.models import RulesRunResult - - -def format_rules_progress( - *, elapsed_seconds: float, result: RulesRunResult, note_skipped_rules: bool = False -) -> str: - """Report the rules phase wall time first, then built-in and custom rule time.""" - - message: str = ( - f"Evaluated rules. ({elapsed_seconds:.2f}s; built-in {result.built_in_ms / 1000:.2f}s, " - f"custom {result.custom_ms / 1000:.2f}s)" - ) - skipped_rules_note: str | None = ( - format_skipped_type_proof_rules(codes=result.skipped_type_proof_rules) - if note_skipped_rules - else None - ) - return message if skipped_rules_note is None else f"{message}; {skipped_rules_note}" diff --git a/src/sqlbuild/rule_engine/main/render_skipped_rules.py b/src/sqlbuild/rule_engine/main/render_skipped_rules.py deleted file mode 100644 index f883250905..0000000000 --- a/src/sqlbuild/rule_engine/main/render_skipped_rules.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Skipped type-proof rules note entrypoint.""" - - -def format_skipped_type_proof_rules(*, codes: tuple[str, ...]) -> str | None: - """Describe type-proof rules skipped because SQL analysis is disabled, if any.""" - - if not codes: - return None - return f"skipped type-proof rules ({', '.join(codes)}) because SQL analysis is disabled" diff --git a/src/sqlbuild/rule_engine/main/run_rules.py b/src/sqlbuild/rule_engine/main/run_rules.py deleted file mode 100644 index ea50bdb95a..0000000000 --- a/src/sqlbuild/rule_engine/main/run_rules.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Unified compiler-rule orchestration entrypoint.""" - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine._helpers.run.rule_threads import evaluate_rules -from sqlbuild.rule_engine.models import ( - PreparedSqlLint, - RulesConfig, - RulesRunResult, - SqlExpansionReuse, -) - - -def run_rules( - *, - graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, - config: RulesConfig, - project_dir: Path, - dialect: str, - selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, - no_sql_analysis: bool = False, -) -> RulesRunResult: - """Run selected native built-ins before selected custom Python rules.""" - - return evaluate_rules( - graph=graph, - discovered_inputs=discovered_inputs, - config=config, - project_dir=project_dir, - dialect=dialect, - selected_keys=selected_keys, - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - no_sql_analysis=no_sql_analysis, - ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 01ce72bf63..5e0749c357 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -14,8 +14,7 @@ from scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit import ( assert_required_cgroup_memory_limit, ) -from sqlbuild._native import build_rule_catalogue, select_catalogue_rules -from sqlbuild.rule_engine.main.load_config import load_rules_config +from sqlbuild._native import build_rule_catalogue, load_rules_config, select_catalogue_rules from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 1d9438aea9..4f1330ff82 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -11,7 +11,7 @@ from typing import Any from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path +from sqlbuild._native import rules_bulk_cache_path from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import replace_project_text __all__ = ("write_custom_rules",) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 08b31e3398..98648b433a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -16,13 +16,13 @@ import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module import sqlbuild.rule_engine._helpers.run.rules as rules_module +from sqlbuild._native import rules_bulk_cache_path from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintRunResult -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, CustomRuleCacheSequenceTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index c20ea521fa..deee9d90ef 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from pathlib import Path from unittest.mock import Mock import pytest @@ -15,8 +16,8 @@ from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings -from sqlbuild.rule_engine.main import load_config, run_rules -from sqlbuild.rule_engine.models import RulesConfig, RulesRunResult +from sqlbuild.rule_engine._helpers.run import rule_threads +from sqlbuild.rule_engine.models import RulesRunResult from sqlbuild.runtime.contracts.main import open_connection from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase @@ -50,6 +51,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_disjoint( test_case: PipelinePhaseTimingTestCase, monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ) -> None: monotonic: Mock = Mock(side_effect=test_case.clock_values) project: CompiledProjectView = CompiledProject( @@ -66,11 +68,11 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) monkeypatch.setattr(open_connection, "open_connection_with_hooks", Mock(return_value=object())) monkeypatch.setattr(plan_result, "build_pipeline_result", Mock(return_value=pipeline_result)) - monkeypatch.setattr(load_config, "load_rules_config", Mock(return_value=RulesConfig())) + (tmp_path / "sqlbuild_project.toml").write_text('name = "timing"\n', encoding="utf-8") monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) monkeypatch.setattr( - run_rules, - "run_rules", + rule_threads, + "evaluate_rules", Mock( return_value=RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) ), @@ -84,7 +86,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di with tracker.scope(): result: CompilePipelineResult = _native.run_compile_pipeline( - discovered_inputs=Mock(), + discovered_inputs=Mock(project_dir=tmp_path), adapter=adapter, options=CompilePipelineOptions(connection_config={}), hooks=progress_hooks[test_case.report_progress], diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 095e19d3d9..a584d47f1c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -4,8 +4,8 @@ import pytest +from sqlbuild._native import load_rules_config from sqlbuild.rule_engine.exceptions import RulesConfigError, RulesError -from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import LayoutConfig, RulesConfig, SqlTestRulesConfig from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( AllowExceptionsConfigTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 213a40c2aa..d14fe28676 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -5,9 +5,9 @@ import pytest +from sqlbuild._native import rules_bulk_cache_path from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host, custom_host_pool -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from sqlbuild.rule_engine.models import CustomRulesOutcome diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py index a9dad4358d..e2ac06ade3 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/render_result/test_render_result.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.rule_engine.main.render_result import format_result +from sqlbuild._native import format_rules_result as format_result from sqlbuild.rule_engine.models import Finding, RulesResult from tests.unit.src.sqlbuild.rule_engine.main.render_result._test_types import ( RenderResultParityTestCase, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py deleted file mode 100644 index 928e4a1882..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/_test_types.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class RulesProgressTestCase: - description: str - elapsed_seconds: float - built_in_ms: int - custom_ms: int - expected_message: str - skipped_type_proof_rules: tuple[str, ...] = () - note_skipped_rules: bool = False diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py b/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py deleted file mode 100644 index 0e28eb9de1..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_rules_progress/test_render_rules_progress.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.rule_engine.main.render_rules_progress import format_rules_progress -from sqlbuild.rule_engine.models import RulesRunResult -from tests.unit.src.sqlbuild.rule_engine.main.render_rules_progress._test_types import ( - RulesProgressTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - RulesProgressTestCase( - description="wall time leads concurrent built-in and custom rule time", - elapsed_seconds=14.5, - built_in_ms=5_600, - custom_ms=13_400, - expected_message="Evaluated rules. (14.50s; built-in 5.60s, custom 13.40s)", - ), - RulesProgressTestCase( - description="skipped type-proof rules follow the timings when requested", - elapsed_seconds=0.25, - built_in_ms=100, - custom_ms=0, - skipped_type_proof_rules=("SQBRCONTRACT105",), - note_skipped_rules=True, - expected_message=( - "Evaluated rules. (0.25s; built-in 0.10s, custom 0.00s); skipped type-proof " - "rules (SQBRCONTRACT105) because SQL analysis is disabled" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_rules_run_when_formatting_progress_then_reports_phase_wall_time_first( - test_case: RulesProgressTestCase, -) -> None: - result: RulesRunResult = RulesRunResult( - findings=(), - evaluated_models=0, - built_in_ms=test_case.built_in_ms, - custom_ms=test_case.custom_ms, - skipped_type_proof_rules=test_case.skipped_type_proof_rules, - ) - - message: str = format_rules_progress( - elapsed_seconds=test_case.elapsed_seconds, - result=result, - note_skipped_rules=test_case.note_skipped_rules, - ) - - assert message == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py deleted file mode 100644 index cb6b90fdfe..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/_test_types.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class SkippedTypeProofRulesNoteTestCase: - description: str - codes: tuple[str, ...] - expected_note: str | None diff --git a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py deleted file mode 100644 index 812a893850..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/main/render_skipped_rules/test_render_skipped_rules.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -from tests.unit.src.sqlbuild.rule_engine.main.render_skipped_rules._test_types import ( - SkippedTypeProofRulesNoteTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - SkippedTypeProofRulesNoteTestCase( - description="skipped rules produce one note", - codes=("SQBRCONTRACT105",), - expected_note=( - "skipped type-proof rules (SQBRCONTRACT105) because SQL analysis is disabled" - ), - ), - SkippedTypeProofRulesNoteTestCase( - description="no skipped rules produce no note", - codes=(), - expected_note=None, - ), - ), - ids=lambda case: case.description, -) -def test_given_skipped_type_proof_rules_when_formatting_note_then_names_skipped_codes( - test_case: SkippedTypeProofRulesNoteTestCase, -) -> None: - assert format_skipped_type_proof_rules(codes=test_case.codes) == test_case.expected_note diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index 145414cc0e..d6f68e22ac 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -4,13 +4,16 @@ import pytest -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + discover_project_inputs, + load_rules_config, + run_rules, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.rule_engine.main.load_config import load_rules_config -from sqlbuild.rule_engine.main.run_rules import run_rules from sqlbuild.rule_engine.models import RulesRunResult from tests.unit.src.sqlbuild.rule_engine.main.run_rules._test_types import ( TypeProofRuleSkipTestCase, From 83e6db5852de94421c5a4f36f624a4bfe32cf905 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:23:49 +0100 Subject: [PATCH 12/32] fix: call native rule engine functions from the native rules command --- .../bindings/_helpers/plan_command/rules.rs | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index fae218bd04..3b48776801 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -10,13 +10,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const LOAD_CONFIG_MODULE: &str = "sqlbuild.rule_engine.main.load_config"; -const CATALOGUE_MODULE: &str = "sqlbuild.rule_engine.main._build_catalogue"; -const SKILLS_MODULE: &str = "sqlbuild.rule_engine.main.skills"; -const RENDER_RULE_MODULE: &str = "sqlbuild.rule_engine.main.render_rule"; -const RENDER_RESULT_MODULE: &str = "sqlbuild.rule_engine.main.render_result"; -const SKIPPED_RULES_MODULE: &str = "sqlbuild.rule_engine.main.render_skipped_rules"; -const RUN_RULES_MODULE: &str = "sqlbuild.rule_engine.main.run_rules"; const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; @@ -60,7 +53,7 @@ fn run_skills( let py: Python<'_> = request.py(); let check: bool = request.getattr("skills_check")?.is_truthy()?; let fresh: bool = call_with( - &imported(py, SKILLS_MODULE, "install_skills")?, + &imported(py, NATIVE_MODULE, "install_rules_skills")?, &[ ("config", config.clone()), ("project_dir", project_dir.clone()), @@ -138,7 +131,7 @@ fn show_rule( let rule: Bound<'_, PyAny> = rule?; if rule.getattr("code")?.extract::()? == selector { let text: String = call_with( - &imported(py, RENDER_RULE_MODULE, "format_rule")?, + &imported(py, NATIVE_MODULE, "format_rule")?, &[("rule", rule), ("config", config.clone())], )? .extract()?; @@ -246,7 +239,7 @@ fn run_rule_selection( replaced.set_item("ignore", PyTuple::empty(py))?; let dialect: Bound<'_, PyAny> = adapter.call_method0("sql_analysis_dialect")?; let result: Bound<'_, PyAny> = call_with( - &imported(py, RUN_RULES_MODULE, "run_rules")?, + &imported(py, NATIVE_MODULE, "run_rules")?, &[ ("graph", graph), ("discovered_inputs", discovered), @@ -274,7 +267,7 @@ fn run_rule_selection( ], )?; let note: Bound<'_, PyAny> = call_with( - &imported(py, SKIPPED_RULES_MODULE, "format_skipped_type_proof_rules")?, + &imported(py, NATIVE_MODULE, "format_skipped_type_proof_rules")?, &[("codes", result.getattr("skipped_type_proof_rules")?)], )?; if !note.is_none() { @@ -285,7 +278,7 @@ fn run_rule_selection( fields.set_item(field, result.getattr(field)?)?; } let text: String = call_with( - &imported(py, RENDER_RESULT_MODULE, "format_result")?, + &imported(py, NATIVE_MODULE, "format_rules_result")?, &[ ( "result", @@ -322,7 +315,7 @@ fn run_rules_command(request: &Bound<'_, PyAny>) -> PyResult { } .call_method0("resolve")?; let config: Bound<'_, PyAny> = call_with( - &imported(py, LOAD_CONFIG_MODULE, "load_rules_config")?, + &imported(py, NATIVE_MODULE, "load_rules_config")?, &[("project_dir", project_dir.clone())], )?; let action: String = request.getattr("action")?.extract()?; @@ -330,7 +323,7 @@ fn run_rules_command(request: &Bound<'_, PyAny>) -> PyResult { return run_skills(request, &config, &project_dir); } let catalogue: Bound<'_, PyAny> = call_with( - &imported(py, CATALOGUE_MODULE, "build_catalogue")?, + &imported(py, NATIVE_MODULE, "build_rule_catalogue")?, &[ ("config", config.clone()), ("project_dir", project_dir.clone()), From 55a7f647a6ef426aa90a509ce126f0bff4d85896 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:29:29 +0100 Subject: [PATCH 13/32] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 123cef517f..82e53cda56 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 46953 +remaining_lines = 45878 [[stays]] path = "executor" From a8dccf0d8c479cce730e49e5d2f1e6c966a2554f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:08:46 +0100 Subject: [PATCH 14/32] refactor: resolve planner graph identities and staleness without recursion --- .../_helpers/identity_resolver.rs | 133 +++++++++++------- .../_helpers/staleness_walk.rs | 101 ++++++++++--- .../src/planner_identity/tests/helpers.rs | 58 ++++++++ .../tests/test_planner_identity.rs | 66 ++++++++- .../src/planner_identity/tests/test_types.rs | 7 + .../compiler/planner/main/_test_types.py | 7 + .../sqlbuild/compiler/planner/main/helpers.py | 21 +++ .../planner/main/identity/_test_types.py | 7 + .../compiler/planner/main/identity/helpers.py | 24 ++++ .../main/identity/test_deep_graph_identity.py | 30 ++++ .../planner/main/test_selection_staleness.py | 25 ++++ 11 files changed, 406 insertions(+), 73 deletions(-) create mode 100644 tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs index de8f20a2c2..ca922cbe9e 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/identity_resolver.rs @@ -1,4 +1,4 @@ -//! Memoized identity resolution over one dependency graph, each node once. +//! Memoized identity resolution over one dependency graph, each node once, with an explicit stack. use std::collections::{HashMap, HashSet}; @@ -13,6 +13,21 @@ enum Mode { Write, } +/// A node whose upstream hashes are being resolved. +struct Frame<'a> { + key: &'a GraphKey, + node: &'a IdentityNode, + local_hash: &'a str, + next_upstream: usize, + upstream_hashes: Vec, +} + +/// What entering a node yields: its hash already, or a frame to resolve. +enum Entered<'a> { + Resolved(Option), + Pending(Frame<'a>), +} + /// Expected or selected-write identity resolution state. pub(crate) struct IdentityResolver<'a> { nodes: &'a HashMap, @@ -49,69 +64,91 @@ impl<'a> IdentityResolver<'a> { } /// A node's expected hash, composed from its upstream expected hashes. - pub(crate) fn resolve_expected(&mut self, key: &GraphKey) -> Option { - if let Some(position) = self.positions.get(key) { - return self.hashes[*position].1.clone(); - } - let nodes: &'a HashMap = self.nodes; - let Some((node, local_hash)) = nodes - .get(key) - .and_then(|node| node.local_hash.as_deref().map(|local| (node, local))) - else { - self.record(key, None); - return None; - }; - if self.visiting.contains(key) { - self.record(key, Some(local_hash.to_owned())); - return Some(local_hash.to_owned()); - } - let composed: String = self.compose(key, (node, local_hash), Mode::Expected); - self.record(key, Some(composed.clone())); - Some(composed) + pub(crate) fn resolve_expected(&mut self, key: &'a GraphKey) -> Option { + self.resolve(key, Mode::Expected) } /// A selected node's hash recomputed from upstream write hashes; other nodes keep theirs. - pub(crate) fn resolve_write(&mut self, key: &GraphKey) -> Option { - if !self.selected.is_some_and(|selected| selected.contains(key)) { - return self.recorded(key); + pub(crate) fn resolve_write(&mut self, key: &'a GraphKey) -> Option { + self.resolve(key, Mode::Write) + } + + fn resolve(&mut self, root: &'a GraphKey, mode: Mode) -> Option { + let mut stack: Vec> = match self.enter(root, mode) { + Entered::Resolved(hash) => return hash, + Entered::Pending(frame) => vec![frame], + }; + while let Some(frame) = stack.last_mut() { + if let Some(upstream_key) = frame.node.upstream_keys.get(frame.next_upstream) { + frame.next_upstream += 1; + match self.enter(upstream_key, mode) { + Entered::Resolved(hash) => frame.upstream_hashes.extend(hash), + Entered::Pending(upstream_frame) => stack.push(upstream_frame), + } + continue; + } + let Some(finished) = stack.pop() else { + break; + }; + let composed: String = self.finish(&finished, mode); + match stack.last_mut() { + Some(parent) => parent.upstream_hashes.push(composed), + None => return Some(composed), + } } - if let Some(hash) = self.resolved_selected.get(key) { - return Some(hash.clone()); + None + } + + fn enter(&mut self, key: &'a GraphKey, mode: Mode) -> Entered<'a> { + if let Mode::Write = mode { + if !self.selected.is_some_and(|selected| selected.contains(key)) { + return Entered::Resolved(self.recorded(key)); + } + if let Some(hash) = self.resolved_selected.get(key) { + return Entered::Resolved(Some(hash.clone())); + } + } else if let Some(position) = self.positions.get(key) { + return Entered::Resolved(self.hashes[*position].1.clone()); } let nodes: &'a HashMap = self.nodes; let Some((node, local_hash)) = nodes .get(key) .and_then(|node| node.local_hash.as_deref().map(|local| (node, local))) else { - return self.recorded(key); + return Entered::Resolved(match mode { + Mode::Expected => { + self.record(key, None); + None + } + Mode::Write => self.recorded(key), + }); }; if self.visiting.contains(key) { - return Some(local_hash.to_owned()); + if let Mode::Expected = mode { + self.record(key, Some(local_hash.to_owned())); + } + return Entered::Resolved(Some(local_hash.to_owned())); } - let composed: String = self.compose(key, (node, local_hash), Mode::Write); - self.record(key, Some(composed.clone())); - self.resolved_selected.insert(key.clone(), composed.clone()); - Some(composed) + self.visiting.insert(key.clone()); + Entered::Pending(Frame { + key, + node, + local_hash, + next_upstream: 0, + upstream_hashes: Vec::new(), + }) } - fn compose( - &mut self, - key: &GraphKey, - (node, local_hash): (&IdentityNode, &str), - mode: Mode, - ) -> String { - self.visiting.insert(key.clone()); - let mut upstream_hashes: Vec = Vec::new(); - for upstream_key in &node.upstream_keys { - let upstream_hash: Option = match mode { - Mode::Expected => self.resolve_expected(upstream_key), - Mode::Write => self.resolve_write(upstream_key), - }; - upstream_hashes.extend(upstream_hash); + fn finish(&mut self, frame: &Frame<'a>, mode: Mode) -> String { + self.visiting.remove(frame.key); + let upstream: Vec<&str> = frame.upstream_hashes.iter().map(String::as_str).collect(); + let composed: String = compose_graph_identity(frame.local_hash, &upstream); + self.record(frame.key, Some(composed.clone())); + if let Mode::Write = mode { + self.resolved_selected + .insert(frame.key.clone(), composed.clone()); } - self.visiting.remove(key); - let upstream: Vec<&str> = upstream_hashes.iter().map(String::as_str).collect(); - compose_graph_identity(local_hash, &upstream) + composed } fn recorded(&self, key: &GraphKey) -> Option { diff --git a/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs b/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs index 7a4cb1ff1f..5670b88247 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/_helpers/staleness_walk.rs @@ -1,10 +1,25 @@ -//! The upstream staleness walk of one selected model. +//! The upstream staleness walk of one selected model, with an explicit stack. use std::collections::{BTreeSet, HashMap, HashSet}; use crate::planner_identity::constants::{MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE}; use crate::planner_identity::models::{GraphKey, StalenessGraph}; +/// A model whose parents are being visited. +struct Frame<'a> { + key: &'a GraphKey, + parents: &'a [GraphKey], + next_parent: usize, + in_run_set: bool, + ancestor_stale: bool, +} + +/// What entering a node yields: its staleness already, or a frame to walk. +enum Entered<'a> { + Resolved(bool), + Pending(Frame<'a>), +} + /// The upstream walk of one selected model. pub(crate) struct StalenessWalk<'a> { pub(crate) graph: &'a StalenessGraph, @@ -14,16 +29,48 @@ pub(crate) struct StalenessWalk<'a> { pub(crate) stale: HashMap, } -impl StalenessWalk<'_> { - pub(crate) fn visit(&mut self, key: &GraphKey) -> bool { +impl<'a> StalenessWalk<'a> { + pub(crate) fn visit(&mut self, root: &'a GraphKey) -> bool { + let mut stack: Vec> = match self.enter(root) { + Entered::Resolved(is_stale) => return is_stale, + Entered::Pending(frame) => vec![frame], + }; + let graph: &'a StalenessGraph = self.graph; + while let Some(frame) = stack.last_mut() { + if let Some(parent_key) = frame.parents.get(frame.next_parent) { + frame.next_parent += 1; + match self.enter(parent_key) { + Entered::Resolved(parent_stale) => { + frame.ancestor_stale = + absorbed(frame.ancestor_stale, (graph, parent_key), parent_stale); + } + Entered::Pending(parent_frame) => stack.push(parent_frame), + } + continue; + } + let Some(finished) = stack.pop() else { + break; + }; + let is_stale: bool = self.finish(&finished); + match stack.last_mut() { + Some(child) => { + child.ancestor_stale = + absorbed(child.ancestor_stale, (graph, finished.key), is_stale); + } + None => return is_stale, + } + } + false + } + + fn enter(&mut self, key: &'a GraphKey) -> Entered<'a> { if let Some(cached) = self.stale.get(key) { - return *cached; + return Entered::Resolved(*cached); } if self.visiting.contains(key) { - return false; + return Entered::Resolved(false); } - self.visiting.insert(key.clone()); - let graph: &StalenessGraph = self.graph; + let graph: &'a StalenessGraph = self.graph; let is_stale: bool = match key.kind.as_str() { MODEL_RESOURCE => { let in_run_set: bool = graph.run.models.contains(&key.name); @@ -31,27 +78,31 @@ impl StalenessWalk<'_> { self.names.insert(key.name.clone()); true } else { - let mut ancestor_stale: bool = false; - for parent_key in graph.upstream_deps.get(key).into_iter().flatten() { - let parent_stale: bool = self.visit(parent_key); - ancestor_stale = parent_stale || ancestor_stale; - if run_parent_changed(graph, parent_key) { - ancestor_stale = true; - } - } - if ancestor_stale && !in_run_set && key != self.model_key { - self.names.insert(key.name.clone()); - } - ancestor_stale + self.visiting.insert(key.clone()); + return Entered::Pending(Frame { + key, + parents: graph.upstream_deps.get(key).map_or(&[], Vec::as_slice), + next_parent: 0, + in_run_set, + ancestor_stale: false, + }); } } SEED_RESOURCE => self.leaf(key, (&graph.changed.seeds, &graph.run.seeds)), SOURCE_RESOURCE => self.leaf(key, (&graph.changed.sources, &graph.run.sources)), _ => false, }; - self.visiting.remove(key); self.stale.insert(key.clone(), is_stale); - is_stale + Entered::Resolved(is_stale) + } + + fn finish(&mut self, frame: &Frame<'a>) -> bool { + if frame.ancestor_stale && !frame.in_run_set && frame.key != self.model_key { + self.names.insert(frame.key.name.clone()); + } + self.visiting.remove(frame.key); + self.stale.insert(frame.key.clone(), frame.ancestor_stale); + frame.ancestor_stale } fn leaf( @@ -67,6 +118,14 @@ impl StalenessWalk<'_> { } } +fn absorbed( + ancestor_stale: bool, + (graph, parent_key): (&StalenessGraph, &GraphKey), + parent_stale: bool, +) -> bool { + parent_stale || ancestor_stale || run_parent_changed(graph, parent_key) +} + fn run_parent_changed(graph: &StalenessGraph, parent_key: &GraphKey) -> bool { let (run, changed): (&HashSet, &HashSet) = match parent_key.kind.as_str() { MODEL_RESOURCE => (&graph.run.models, &graph.changed.models), diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs index 6b2872ee41..57a7e0faf0 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/helpers.rs @@ -1,5 +1,6 @@ use std::collections::{HashMap, HashSet}; +use crate::planner_identity::_helpers::compose_graph_identity::compose_graph_identity; use crate::planner_identity::models::{GraphKey, IdentityGraph, IdentityNode, StalenessGraph}; pub(super) fn key(kind: &str, name: &str) -> GraphKey { @@ -87,3 +88,60 @@ pub(super) fn ladder_identity_graph(layer_count: usize) -> IdentityGraph { } graph } + +/// A chain where each model reads the previous one, resolved from its most downstream model. +pub(super) fn chain_identity_graph(depth: usize) -> IdentityGraph { + let mut graph = IdentityGraph::default(); + for index in 0..depth { + let upstream_keys: Vec = index + .checked_sub(1) + .map(|upstream| model(&format!("m{upstream}"))) + .into_iter() + .collect(); + graph.nodes.insert( + model(&format!("m{index}")), + IdentityNode { + upstream_keys, + local_hash: Some(format!("local_m{index}")), + }, + ); + } + graph.execution_order = (0..depth) + .rev() + .map(|index| model(&format!("m{index}"))) + .collect(); + graph +} + +/// The most downstream hash of `chain_identity_graph`, composed one link at a time. +pub(super) fn chain_head_hash(depth: usize) -> Option { + let mut hash: Option = None; + for index in 0..depth { + let upstream: Vec<&str> = hash.iter().map(String::as_str).collect(); + hash = Some(compose_graph_identity( + &format!("local_m{index}"), + &upstream, + )); + } + hash +} + +/// The staleness graph of a model chain whose most upstream model changed outside the run set. +pub(super) fn chain_staleness_graph(depth: usize) -> StalenessGraph { + let head: String = format!("m{}", depth - 1); + let mut graph = StalenessGraph { + upstream_deps: (1..depth) + .map(|index| { + ( + model(&format!("m{index}")), + vec![model(&format!("m{}", index - 1))], + ) + }) + .collect(), + selected_model_names: vec![head.clone()], + ..StalenessGraph::default() + }; + graph.changed.models = names(&["m0"]); + graph.run.models = HashSet::from([head]); + graph +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index 69447c9b93..4ce46945f6 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -16,13 +16,15 @@ use crate::planner_identity::models::{ StalenessNames, }; use crate::planner_identity::tests::helpers::{ - cyclic_identity_graph, identity_graph, key, ladder_identity_graph, model, names, - staleness_graph, texts, + chain_head_hash, chain_identity_graph, chain_staleness_graph, cyclic_identity_graph, + identity_graph, key, ladder_identity_graph, model, names, staleness_graph, texts, }; use crate::planner_identity::tests::test_types::{ - BuildabilityTestCase, FunctionSqlTestCase, HashTestCase, IdentityGraphSizeTestCase, - IdentityGraphTestCase, OrderTestCase, StalenessTestCase, WarningTextTestCase, + BuildabilityTestCase, DeepChainTestCase, FunctionSqlTestCase, HashTestCase, + IdentityGraphSizeTestCase, IdentityGraphTestCase, OrderTestCase, StalenessTestCase, + WarningTextTestCase, }; +use crate::planner_identity::types::IdentityHashes; #[test] fn given_persisted_identity_inputs_when_hashing_then_matches_pinned_python_hashes() { @@ -172,6 +174,62 @@ fn given_dense_ladder_when_resolving_write_hashes_then_resolves_each_node_once() } } +#[test] +fn given_deep_chain_when_walking_graph_then_does_not_overflow_the_stack() { + let test_cases = [DeepChainTestCase { + description: "one hundred thousand chained models", + depth: 100_000, + expected_hash_count: 100_000, + expected_trigger_count: 99_999, + }]; + for test_case in test_cases { + let graph = chain_identity_graph(test_case.depth); + let head: GraphKey = model(&format!("m{}", test_case.depth - 1)); + let expected: IdentityHashes = expected_graph_identity_hashes(&graph); + assert_eq!( + expected.len(), + test_case.expected_hash_count, + "{}", + test_case.description + ); + assert_eq!( + expected.last().map(|(_, hash)| hash.clone()), + Some(chain_head_hash(test_case.depth)), + "{}", + test_case.description + ); + let selected: HashSet = graph.nodes.keys().cloned().collect(); + let written: Vec<(GraphKey, String)> = + graph_write_identity_hashes(&graph, &selected, Vec::new()); + assert_eq!( + written.len(), + test_case.expected_hash_count, + "{}", + test_case.description + ); + assert_eq!( + written + .iter() + .find(|(key, _)| *key == head) + .map(|(_, hash)| hash.clone()), + chain_head_hash(test_case.depth), + "{}", + test_case.description + ); + let warnings: Vec = + classify_selection_staleness(&chain_staleness_graph(test_case.depth)); + assert_eq!( + warnings + .iter() + .map(|warning| warning.trigger_names.len()) + .sum::(), + test_case.expected_trigger_count, + "{}", + test_case.description + ); + } +} + #[test] fn given_function_fields_when_rendering_then_writes_one_field_per_line() { let test_cases = [FunctionSqlTestCase { diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs index 8ad36977d2..9c701080c4 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_types.rs @@ -61,3 +61,10 @@ pub(super) struct FunctionSqlTestCase { pub(super) fields: FunctionFingerprintFields, pub(super) expected_sql: &'static str, } + +pub(super) struct DeepChainTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + pub(super) expected_hash_count: usize, + pub(super) expected_trigger_count: usize, +} diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index 621c272656..a67263cbbc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -181,3 +181,10 @@ class ProtectedRebuildBlockedTestCase: description: str external_blocked_model_names: tuple[str, ...] expected_action: str + + +@dataclass(frozen=True) +class DeepChainStalenessTestCase: + description: str + depth: int + expected_trigger_count: int diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 2cc44d4bd6..6fd2cae9c2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -31,6 +31,8 @@ PlannerPolicies, PlannerSelection, PlanOutput, + SelectionStalenessGraph, + SelectionStalenessNodeKey, ) from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry @@ -177,3 +179,22 @@ def create_protected_model_state(*, adapter: DuckDbAdapter, connection: Any) -> render_qualified_name=adapter.render_qualified_name, render_framework_type=adapter.render_framework_type, ) + + +def build_chain_staleness_graph(*, depth: int) -> SelectionStalenessGraph: + """Build a model chain whose most upstream model changed and only the head runs.""" + + keys: list[SelectionStalenessNodeKey] = [ + SelectionStalenessNodeKey(resource_type="model", name=f"m{index}") for index in range(depth) + ] + head: str = keys[-1].name + return SelectionStalenessGraph( + upstream_deps={key: (upstream,) for key, upstream in zip(keys[1:], keys[:-1], strict=True)}, + selected_model_names=frozenset({head}), + run_model_names=frozenset({head}), + run_seed_names=frozenset(), + run_source_names=frozenset(), + changed_model_names=frozenset({"m0"}), + changed_seed_names=frozenset(), + changed_source_names=frozenset(), + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py index 8e10a52242..b573b77eb9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/_test_types.py @@ -55,3 +55,10 @@ class PersistedIdentityTestCase: description: str build: Callable[[Path], object] expected_identity: object + + +@dataclass(frozen=True) +class DeepChainIdentityTestCase: + description: str + depth: int + expected_hash_count: int diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index be89a4876a..8f7c4ad4ae 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -171,3 +171,27 @@ def build_identity_seed(directory: Path) -> SimpleNamespace: columns=(SchemaColumn(name="id", type="int"),), ), ) + + +def build_chain_graph_identities(*, depth: int) -> DirectModelVersionIdentities: + """Build identities of a model chain resolved from its most downstream model.""" + + keys: list[CompiledObjectKey] = [ + CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=f"m{index}") + for index in range(depth) + ] + upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { + key: tuple(keys[index - 1 : index]) for index, key in enumerate(keys) + } + scope: PlannerScope = PlannerScope( + upstream_deps=upstream, + downstream_deps={}, + all_keys={key.name: key for key in keys}, + models_by_name={ + key.name: _build_model(key=key, query_sql="SELECT 1 AS id", deps=upstream[key]) + for key in keys + }, + selected_keys=frozenset(keys), + execution_order=tuple(reversed(keys)), + ) + return build_direct_model_version_identities(functions=(), scope=scope, dialect=None) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py new file mode 100644 index 0000000000..95faa636e2 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner.models import DirectModelVersionIdentities +from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( + DeepChainIdentityTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner.main.identity.helpers import ( + build_chain_graph_identities, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DeepChainIdentityTestCase( + description="one hundred thousand chained models", + depth=100_000, + expected_hash_count=100_000, + ), + ], + ids=lambda case: case.description, +) +def test_given_deep_model_chain_when_building_identities_then_resolves_every_model( + test_case: DeepChainIdentityTestCase, +) -> None: + identities: DirectModelVersionIdentities = build_chain_graph_identities(depth=test_case.depth) + + assert len(identities.model_version_hashes) == test_case.expected_hash_count diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py index 47d74f9eb2..e58e05e028 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py @@ -9,8 +9,10 @@ SelectionStalenessWarning, ) from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( + DeepChainStalenessTestCase, SelectionStalenessClassifierTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner.main.helpers import build_chain_staleness_graph MODEL_A: SelectionStalenessNodeKey = SelectionStalenessNodeKey(resource_type="model", name="a") MODEL_B: SelectionStalenessNodeKey = SelectionStalenessNodeKey(resource_type="model", name="b") @@ -481,3 +483,26 @@ def test_given_graph_when_classifying_then_reports_expected_warnings( ) assert warnings == test_case.expected_warnings + + +@pytest.mark.parametrize( + "test_case", + [ + DeepChainStalenessTestCase( + description="one hundred thousand chained models with a changed root", + depth=100_000, + expected_trigger_count=99_999, + ), + ], + ids=lambda case: case.description, +) +def test_given_deep_chain_when_classifying_then_walks_without_exhausting_the_stack( + test_case: DeepChainStalenessTestCase, +) -> None: + warnings: tuple[SelectionStalenessWarning, ...] = classify_selection_staleness_warnings( + build_chain_staleness_graph(depth=test_case.depth) + ) + + assert sum(len(warning.trigger_names) for warning in warnings) == ( + test_case.expected_trigger_count + ) From d140d6f73e187919fb1ee89b02dcd5d4be3aab63 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:45:38 +0100 Subject: [PATCH 15/32] refactor: resolve planner selectors and split unit-test selectors natively --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/planner_selection/_helpers/mod.rs | 1 + .../_helpers/unit_test_tokens.rs | 164 +++++++++++ .../src/planner_selection/constants.rs | 16 ++ .../src/planner_selection/main/mod.rs | 4 + .../main/split_selector_expansion.rs | 30 ++ .../main/split_unit_test_selectors.rs | 30 ++ .../main/sqlbuild_model_selector_names.rs | 43 +++ .../main/unit_test_selector_rejection.rs | 15 + .../src/planner_selection/mod.rs | 9 + .../src/planner_selection/models.rs | 50 ++++ .../src/planner_selection/tests.rs | 6 + .../src/planner_selection/tests/helpers.rs | 57 ++++ .../tests/test_planner_selection.rs | 174 ++++++++++++ .../src/planner_selection/tests/test_types.rs | 24 ++ .../bindings/_helpers/check/check_command.rs | 5 +- .../bindings/_helpers/clone/clone_command.rs | 3 +- .../_helpers/compile_driver/analysis.rs | 4 +- .../compile_driver/selection_phases.rs | 3 +- .../_helpers/contract_command/command.rs | 5 +- .../bindings/_helpers/dbt_interop/graph.rs | 6 +- .../bindings/_helpers/diff_report/command.rs | 3 +- .../_helpers/format_command/selection.rs | 5 +- .../_helpers/freshness/freshness_command.rs | 7 +- .../bindings/_helpers/graph/project_graph.rs | 47 +--- .../src/bindings/_helpers/graph/views.rs | 32 ++- .../_helpers/lineage/lineage_command.rs | 13 +- .../bindings/_helpers/load/load_command.rs | 3 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_selection/carrier.rs | 143 ++++++++++ .../_helpers/planner_selection/mod.rs | 5 + .../planner_selection/model_selectors.rs | 121 ++++++++ .../planner_selection/registration.rs | 11 + .../_helpers/planner_selection/selectors.rs | 264 ++++++++++++++++++ .../_helpers/planner_selection/unit_tests.rs | 178 ++++++++++++ .../_helpers/scenario_command/setup.rs | 3 +- .../src/bindings/main/register.rs | 5 +- scripts/fensu_policy/constants.py | 3 - .../rules/canonical_operations.py | 8 +- src/sqlbuild/_native.pyi | 71 ++++- .../_helpers/build_planning/defer_clone.py | 2 +- .../pipeline/main/_project_graph_selection.py | 15 - .../main/_reject_unit_test_selectors.py | 25 -- .../pipeline/main/unit_test_selection.py | 25 -- .../compiler/planner/_helpers/graph/scope.py | 10 +- .../planner/_helpers/graph/selectors.py | 230 --------------- .../compiler/planner/_helpers/reuse/clone.py | 4 +- .../_helpers/selection/unit_test_selectors.py | 207 -------------- .../main/selection/_build_resources.py | 27 -- .../main/selection/_graph_selection.py | 18 -- .../planner/main/selection/_selector_match.py | 24 -- .../main/selection/_selector_tokens.py | 27 -- .../selection/_split_unit_test_selectors.py | 26 -- .../planner/main/selection/_upstream.py | 16 -- .../planner/main/selection/selection.py | 29 -- .../main/selection/selector_expansion.py | 23 -- .../planner/main/selection/selector_parse.py | 12 - .../selection/sqlbuild_model_selectors.py | 39 --- .../selection/unit_test_selector_error.py | 12 - .../dagster/_helpers/selection.py | 9 +- .../rules/test_canonical_operations.py | 4 +- .../planner/_helpers/selection/helpers.py | 33 +++ .../selection/test_unit_test_selectors.py | 24 +- .../planner/_helpers/test_selectors.py | 60 ++-- .../planner/main/test_selector_expansion.py | 2 +- .../main/test_sqlbuild_model_selectors.py | 4 +- .../python_nodes/_helpers/test_selectors.py | 3 +- .../_helpers/test_unified_selectors.py | 2 +- 68 files changed, 1570 insertions(+), 915 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs create mode 100644 crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs delete mode 100644 src/sqlbuild/compiler/pipeline/main/_project_graph_selection.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/_reject_unit_test_selectors.py delete mode 100644 src/sqlbuild/compiler/pipeline/main/unit_test_selection.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/selectors.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_build_resources.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_graph_selection.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_selector_match.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_upstream.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/selection.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/selector_expansion.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/selector_parse.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py create mode 100644 tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 3cf976f9b6..2f4aba477e 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -21,6 +21,7 @@ pub mod lineage_graph; pub mod manifest; pub mod planner_changes; pub mod planner_identity; +pub mod planner_selection; pub mod planner_strategy; pub mod python_plan; pub mod python_selection; diff --git a/crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs b/crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs new file mode 100644 index 0000000000..aab0ae3d7c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod unit_test_tokens; diff --git a/crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs b/crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs new file mode 100644 index 0000000000..b1c6182885 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/_helpers/unit_test_tokens.rs @@ -0,0 +1,164 @@ +//! Classify selector groups, tokens and intersection parts as unit-test or resource selectors. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::graph::errors::SelectorError; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::main::selector_name_help::selector_name_help; +use crate::graph::models::ParsedSelector; +use crate::planner_selection::constants::{ + INTERSECTION_SEPARATOR, NAME_KIND, PATTERN_CHARACTERS, TEST_KIND, TOKEN_SEPARATOR, + UNIT_TEST_SELECTOR_CODE, UNKNOWN_SELECTOR_CODE, +}; +use crate::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; +use crate::planner_selection::models::UnitTestSelectorNames; + +/// The unit tests a selector names and whether it also selects resources. +type Classified = (BTreeSet, bool); + +/// The unit tests the groups select and the groups left with their unit-test tokens removed. +pub(crate) fn split_groups( + groups: &[String], + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result<(BTreeSet, Vec), SelectorError> { + let mut tests: BTreeSet = BTreeSet::new(); + let mut kept_groups: Vec = Vec::new(); + for group in groups { + let tokens: Vec<&str> = group + .split(char::is_whitespace) + .filter(|token| !token.is_empty()) + .collect(); + let mut kept_tokens: Vec<&str> = Vec::new(); + for token in &tokens { + let (matched, keep): Classified = split_token(token, names, accepts_unit_tests)?; + tests.extend(matched); + if keep { + kept_tokens.push(token); + } + } + if kept_tokens.len() == tokens.len() { + kept_groups.push(group.clone()); + } else if !kept_tokens.is_empty() { + kept_groups.push(kept_tokens.join(TOKEN_SEPARATOR)); + } + } + Ok((tests, kept_groups)) +} + +fn split_token( + token: &str, + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result { + let parts: Vec<&str> = token.split(INTERSECTION_SEPARATOR).collect(); + let mut classified: Vec = Vec::with_capacity(parts.len()); + for part in &parts { + classified.push(classify(part, names, accepts_unit_tests)?); + } + let selects_tests: bool = classified + .iter() + .any(|(matched, keep)| !matched.is_empty() && !keep); + if parts.len() > 1 && selects_tests { + for part in &parts { + parse_selector(part)?; + } + return Err(SelectorError { + code: UNIT_TEST_SELECTOR_CODE, + message: format!( + "selector '{token}' intersects a unit test with ','; unit tests can only be selected whole, by name" + ), + help: None, + }); + } + if parts.len() > 1 { + return Ok((BTreeSet::new(), true)); + } + Ok(classified + .into_iter() + .next() + .unwrap_or((BTreeSet::new(), true))) +} + +fn classify( + part: &str, + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result { + let Ok(ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + }) = parse_selector(part) + else { + return Ok((BTreeSet::new(), true)); + }; + if kind != NAME_KIND && kind != TEST_KIND { + return Ok((BTreeSet::new(), true)); + } + let matched_tests: BTreeSet = matching(&value, &names.tests); + if kind == NAME_KIND && !matching(&value, &names.resources).is_empty() { + return Ok((BTreeSet::new(), true)); + } + if matched_tests.is_empty() { + if kind == TEST_KIND || accepts_unit_tests { + return Err(unknown(&kind, &value, names)); + } + return Ok((BTreeSet::new(), true)); + } + if !accepts_unit_tests { + return Err(unit_test_selector_rejection(part)); + } + if upstream || downstream { + return Err(SelectorError { + code: UNIT_TEST_SELECTOR_CODE, + message: format!( + "selector '{part}' expands a unit test with '+'; unit tests have no lineage, so select them by name without '+'" + ), + help: None, + }); + } + Ok((matched_tests, false)) +} + +fn unknown(kind: &str, value: &str, names: &UnitTestSelectorNames) -> SelectorError { + let label: &str = if is_pattern(value) { "pattern" } else { "name" }; + let (subject, candidates): (&str, Vec<&str>) = if kind == NAME_KIND { + ( + "selector", + names + .tests + .union(&names.resources) + .map(String::as_str) + .collect(), + ) + } else { + ( + "unit test", + names.tests.iter().map(String::as_str).collect(), + ) + }; + SelectorError { + code: UNKNOWN_SELECTOR_CODE, + message: format!("unknown {subject} {label} '{value}'"), + help: selector_name_help(value, candidates), + } +} + +fn matching(value: &str, names: &BTreeSet) -> BTreeSet { + if !is_pattern(value) { + return names.get(value).into_iter().cloned().collect(); + } + names + .iter() + .filter(|name| glob_match(name, value)) + .cloned() + .collect() +} + +fn is_pattern(value: &str) -> bool { + value.contains(PATTERN_CHARACTERS) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/constants.rs b/crates/sqlbuild-analysis/src/planner_selection/constants.rs new file mode 100644 index 0000000000..ea8ed27ae0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/constants.rs @@ -0,0 +1,16 @@ +//! Planner selection codes, kinds and messages. + +pub const UNIT_TEST_SELECTOR_CODE: &str = "S013"; +pub const UNKNOWN_SELECTOR_CODE: &str = "S007"; +pub const UNIT_TEST_ONLY_TEST_AND_BUILD: &str = + "only `sqb test` and `sqb build` accept unit-test selectors"; +pub(crate) const NAME_KIND: &str = "name"; +pub(crate) const TEST_KIND: &str = "test"; +pub(crate) const INTERSECTION_SEPARATOR: char = ','; +pub(crate) const TOKEN_SEPARATOR: &str = " "; +pub(crate) const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; +pub(crate) const EXPANSION_MARKER: char = '+'; +pub(crate) const TAG_PREFIX: &str = "tag:"; +pub(crate) const PATH_PREFIX: &str = "path:"; +pub(crate) const FOLDER_SEPARATOR: char = '/'; +pub(crate) const WINDOWS_SEPARATOR: char = '\\'; diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/mod.rs b/crates/sqlbuild-analysis/src/planner_selection/main/mod.rs new file mode 100644 index 0000000000..23a007902f --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/mod.rs @@ -0,0 +1,4 @@ +pub mod split_selector_expansion; +pub mod split_unit_test_selectors; +pub mod sqlbuild_model_selector_names; +pub mod unit_test_selector_rejection; diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs b/crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs new file mode 100644 index 0000000000..d9a01f8711 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/split_selector_expansion.rs @@ -0,0 +1,30 @@ +//! Split `+selector+` marker syntax from the selector core without resolving the core. + +use crate::planner_selection::constants::EXPANSION_MARKER; +use crate::planner_selection::models::SelectorExpansion; + +/// The core and expansion flags, or the input error message. +pub fn split_selector_expansion(raw: &str) -> Result { + let stripped: &str = raw.trim_matches(char::is_whitespace); + if stripped.is_empty() { + return Err("empty selector".to_owned()); + } + let core: &str = stripped + .trim_start_matches(EXPANSION_MARKER) + .trim_end_matches(EXPANSION_MARKER); + if core.is_empty() { + return Err(format!( + "selector '{stripped}' has no name after removing '+' markers" + )); + } + if core.contains(EXPANSION_MARKER) { + return Err(format!( + "selector '{stripped}' contains '+' in an unsupported position" + )); + } + Ok(SelectorExpansion { + core: core.to_owned(), + upstream: stripped.starts_with(EXPANSION_MARKER), + downstream: stripped.ends_with(EXPANSION_MARKER), + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs b/crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs new file mode 100644 index 0000000000..b2c9ca72a3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/split_unit_test_selectors.rs @@ -0,0 +1,30 @@ +//! Move unit-test selectors out of resource selectors, or reject them for this command. + +use std::collections::BTreeSet; + +use crate::graph::errors::SelectorError; +use crate::planner_selection::_helpers::unit_test_tokens::split_groups; +use crate::planner_selection::models::{ + SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, +}; + +/// The resource selectors left and the unit tests selected or excluded by name. +pub fn split_unit_test_selectors( + groups: SelectorGroups<'_>, + names: &UnitTestSelectorNames, + accepts_unit_tests: bool, +) -> Result { + let (test_names, select): (BTreeSet, Vec) = + split_groups(groups.select, names, accepts_unit_tests)?; + let (excluded_test_names, exclude): (BTreeSet, Vec) = + split_groups(groups.exclude, names, accepts_unit_tests)?; + let models_selected: bool = + groups.select.is_empty() || !select.is_empty() || test_names.is_empty(); + Ok(UnitTestSelectorSplit { + select, + exclude, + test_names, + excluded_test_names, + models_selected, + }) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs b/crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs new file mode 100644 index 0000000000..2ccd2525bf --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/sqlbuild_model_selector_names.rs @@ -0,0 +1,43 @@ +//! Resolve one SQLBuild-owned model selector core to model names. + +use crate::planner_selection::constants::{ + FOLDER_SEPARATOR, PATH_PREFIX, TAG_PREFIX, WINDOWS_SEPARATOR, +}; +use crate::planner_selection::models::{ModelSelectorNames, SelectableModel}; + +/// A model name, `tag:` or `path:` term's model names in model order; other terms match nothing. +pub fn sqlbuild_model_selector_names(models: &[SelectableModel], term: &str) -> ModelSelectorNames { + if models.iter().any(|model| model.name == term) { + return ModelSelectorNames { + names: vec![term.to_owned()], + translated: None, + }; + } + if let Some(tag) = term.strip_prefix(TAG_PREFIX) { + return ModelSelectorNames { + names: models + .iter() + .filter(|model| has_tag(model, tag)) + .map(|model| model.name.clone()) + .collect(), + translated: None, + }; + } + let Some(raw_path) = term.strip_prefix(PATH_PREFIX) else { + return ModelSelectorNames::default(); + }; + let path: String = raw_path.replace(WINDOWS_SEPARATOR, &FOLDER_SEPARATOR.to_string()); + let nested: String = format!("{path}{FOLDER_SEPARATOR}"); + ModelSelectorNames { + names: models + .iter() + .filter(|model| model.folder == path || model.folder.starts_with(&nested)) + .map(|model| model.name.clone()) + .collect(), + translated: (path != raw_path).then(|| format!("{PATH_PREFIX}{path}")), + } +} + +fn has_tag(model: &SelectableModel, tag: &str) -> bool { + model.tags.iter().any(|candidate| candidate == tag) +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs b/crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs new file mode 100644 index 0000000000..34383c5175 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/main/unit_test_selector_rejection.rs @@ -0,0 +1,15 @@ +//! The error for a unit-test selector given to a command that does not run unit tests. + +use crate::graph::errors::SelectorError; +use crate::planner_selection::constants::{UNIT_TEST_ONLY_TEST_AND_BUILD, UNIT_TEST_SELECTOR_CODE}; + +/// The coded rejection naming the selector. +pub fn unit_test_selector_rejection(selector: &str) -> SelectorError { + SelectorError { + code: UNIT_TEST_SELECTOR_CODE, + message: format!( + "selector '{selector}' selects a unit test; {UNIT_TEST_ONLY_TEST_AND_BUILD}" + ), + help: None, + } +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/mod.rs b/crates/sqlbuild-analysis/src/planner_selection/mod.rs new file mode 100644 index 0000000000..53f788143d --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/mod.rs @@ -0,0 +1,9 @@ +//! Planner selection boundary: unit-test selector splitting, `+` marker splitting and SQLBuild-owned model selectors. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_selection/models.rs b/crates/sqlbuild-analysis/src/planner_selection/models.rs new file mode 100644 index 0000000000..0d757a4f42 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/models.rs @@ -0,0 +1,50 @@ +//! Planner selection inputs and results. + +use std::collections::BTreeSet; + +/// Names selectors can match: unit tests, and every other selectable resource. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UnitTestSelectorNames { + pub tests: BTreeSet, + pub resources: BTreeSet, +} + +/// Raw `--select` and `--exclude` groups. +#[derive(Clone, Copy, Debug)] +pub struct SelectorGroups<'a> { + pub select: &'a [String], + pub exclude: &'a [String], +} + +/// Resource selectors left after unit-test selectors moved into a test selection. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct UnitTestSelectorSplit { + pub select: Vec, + pub exclude: Vec, + pub test_names: BTreeSet, + pub excluded_test_names: BTreeSet, + pub models_selected: bool, +} + +/// One selector split into `+` expansion flags and core text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectorExpansion { + pub core: String, + pub upstream: bool, + pub downstream: bool, +} + +/// A model as SQLBuild-owned model selectors see it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectableModel { + pub name: String, + pub tags: Vec, + pub folder: String, +} + +/// Names one SQLBuild model selector resolves to, and its translated path selector when it changed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelSelectorNames { + pub names: Vec, + pub translated: Option, +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests.rs b/crates/sqlbuild-analysis/src/planner_selection/tests.rs new file mode 100644 index 0000000000..753a6cec33 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planner_selection.rs"] +mod planner_selection; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs new file mode 100644 index 0000000000..9a83291566 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests/helpers.rs @@ -0,0 +1,57 @@ +use std::collections::BTreeSet; + +use crate::graph::errors::SelectorError; +use crate::planner_selection::models::{ + SelectableModel, UnitTestSelectorNames, UnitTestSelectorSplit, +}; + +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(super) fn name_set(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Unit tests `test_orders` and `test_customers`; resources `orders` and `customers`. +pub(super) fn selector_names() -> UnitTestSelectorNames { + UnitTestSelectorNames { + tests: name_set(&["test_customers", "test_orders"]), + resources: name_set(&["customers", "orders"]), + } +} + +pub(super) fn split( + select: &[&str], + tests: &[&str], + models_selected: bool, +) -> UnitTestSelectorSplit { + UnitTestSelectorSplit { + select: texts(select), + exclude: Vec::new(), + test_names: name_set(tests), + excluded_test_names: BTreeSet::new(), + models_selected, + } +} + +pub(super) fn failure(code: &'static str, message: &str, help: Option<&str>) -> SelectorError { + SelectorError { + code, + message: message.to_owned(), + help: help.map(str::to_owned), + } +} + +pub(super) fn models() -> Vec { + let model = |name: &str, tags: &[&str], folder: &str| SelectableModel { + name: name.to_owned(), + tags: texts(tags), + folder: folder.to_owned(), + }; + vec![ + model("orders", &["daily"], "models/staging"), + model("customers", &[], "models/staging/sub"), + model("report", &["daily"], "models/marts"), + ] +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs b/crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs new file mode 100644 index 0000000000..80b6a0c0b2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests/test_planner_selection.rs @@ -0,0 +1,174 @@ +use crate::planner_selection::main::split_selector_expansion::split_selector_expansion; +use crate::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; +use crate::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; +use crate::planner_selection::models::{ModelSelectorNames, SelectorExpansion, SelectorGroups}; +use crate::planner_selection::tests::helpers::{failure, models, selector_names, split, texts}; +use crate::planner_selection::tests::test_types::{ + ExpansionTestCase, ModelSelectorTestCase, UnitTestSplitTestCase, +}; + +#[test] +fn given_selector_groups_when_splitting_unit_tests_then_moves_or_rejects_them() { + let test_cases = [ + UnitTestSplitTestCase { + description: "unit test token leaves the resource token in its group", + select: texts(&["test_orders customers"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Ok(split(&["customers"], &["test_orders"], true)), + }, + UnitTestSplitTestCase { + description: "pattern selects every matching unit test and no model", + select: texts(&["test_*"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Ok(split(&[], &["test_customers", "test_orders"], false)), + }, + UnitTestSplitTestCase { + description: "resource names win over unit tests", + select: texts(&["orders"]), + exclude: Vec::new(), + accepts_unit_tests: false, + expected_split: Ok(split(&["orders"], &[], true)), + }, + UnitTestSplitTestCase { + description: "commands without unit tests reject them", + select: texts(&["test_orders"]), + exclude: Vec::new(), + accepts_unit_tests: false, + expected_split: Err(failure( + "S013", + "selector 'test_orders' selects a unit test; only `sqb test` and `sqb build` accept unit-test selectors", + None, + )), + }, + UnitTestSplitTestCase { + description: "intersections cannot include unit tests", + select: texts(&["test_orders,orders"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Err(failure( + "S013", + "selector 'test_orders,orders' intersects a unit test with ','; unit tests can only be selected whole, by name", + None, + )), + }, + UnitTestSplitTestCase { + description: "unit tests have no lineage", + select: texts(&["+test_orders"]), + exclude: Vec::new(), + accepts_unit_tests: true, + expected_split: Err(failure( + "S013", + "selector '+test_orders' expands a unit test with '+'; unit tests have no lineage, so select them by name without '+'", + None, + )), + }, + UnitTestSplitTestCase { + description: "unknown unit test suggests the closest name", + select: Vec::new(), + exclude: texts(&["test:test_order"]), + accepts_unit_tests: false, + expected_split: Err(failure( + "S007", + "unknown unit test name 'test_order'", + Some("did you mean 'test_orders'?"), + )), + }, + ]; + for test_case in test_cases { + assert_eq!( + split_unit_test_selectors( + SelectorGroups { + select: &test_case.select, + exclude: &test_case.exclude, + }, + &selector_names(), + test_case.accepts_unit_tests, + ), + test_case.expected_split, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_raw_selector_when_splitting_expansion_then_returns_core_and_flags() { + let test_cases = [ + ExpansionTestCase { + description: "both markers", + raw: " +orders+ ", + expected_expansion: Ok(SelectorExpansion { + core: "orders".to_owned(), + upstream: true, + downstream: true, + }), + }, + ExpansionTestCase { + description: "markers only", + raw: "++", + expected_expansion: Err( + "selector '++' has no name after removing '+' markers".to_owned() + ), + }, + ExpansionTestCase { + description: "inner marker", + raw: "a+b", + expected_expansion: Err( + "selector 'a+b' contains '+' in an unsupported position".to_owned() + ), + }, + ExpansionTestCase { + description: "blank", + raw: " ", + expected_expansion: Err("empty selector".to_owned()), + }, + ]; + for test_case in test_cases { + assert_eq!( + split_selector_expansion(test_case.raw), + test_case.expected_expansion, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_selector_term_when_resolving_then_returns_model_names() { + let names = |values: &[&str], translated: Option<&str>| ModelSelectorNames { + names: texts(values), + translated: translated.map(str::to_owned), + }; + let test_cases = [ + ModelSelectorTestCase { + description: "model name", + term: "report", + expected_names: names(&["report"], None), + }, + ModelSelectorTestCase { + description: "tag in model order", + term: "tag:daily", + expected_names: names(&["orders", "report"], None), + }, + ModelSelectorTestCase { + description: "windows path is translated and matches nested folders", + term: "path:models\\staging", + expected_names: names(&["orders", "customers"], Some("path:models/staging")), + }, + ModelSelectorTestCase { + description: "other terms", + term: "source:raw", + expected_names: names(&[], None), + }, + ]; + for test_case in test_cases { + assert_eq!( + sqlbuild_model_selector_names(&models(), test_case.term), + test_case.expected_names, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs new file mode 100644 index 0000000000..9c626c1f7b --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_selection/tests/test_types.rs @@ -0,0 +1,24 @@ +use crate::graph::errors::SelectorError; +use crate::planner_selection::models::{ + ModelSelectorNames, SelectorExpansion, UnitTestSelectorSplit, +}; + +pub(super) struct UnitTestSplitTestCase { + pub(super) description: &'static str, + pub(super) select: Vec, + pub(super) exclude: Vec, + pub(super) accepts_unit_tests: bool, + pub(super) expected_split: Result, +} + +pub(super) struct ExpansionTestCase { + pub(super) description: &'static str, + pub(super) raw: &'static str, + pub(super) expected_expansion: Result, +} + +pub(super) struct ModelSelectorTestCase { + pub(super) description: &'static str, + pub(super) term: &'static str, + pub(super) expected_names: ModelSelectorNames, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index bc6a986ef0..4c27aa8493 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,12 +25,11 @@ const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; -const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 116419d67a..46610db665 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -28,8 +28,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index c13e41103c..632372481f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rules_config::load_rules_config; const DISCOVER_MODULE: &str = "sqlbuild._native"; -const REJECT_SELECTORS_MODULE: &str = "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_SELECTORS_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; @@ -26,7 +26,7 @@ const FAST_COLUMN_LINEAGE: &str = "FAST"; const CONTRACTS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; -const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; +const SELECTION_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RULE_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 29e2b31dd0..5234f47d92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -10,8 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; -const SPLIT_SELECTORS_MODULE: &str = - "sqlbuild.compiler.planner.main.selection._split_unit_test_selectors"; +const SPLIT_SELECTORS_MODULE: &str = "sqlbuild._native"; const PLANNER_SCOPES_MODULE: &str = "sqlbuild.compiler.planner.main.selection._resolve_planner_scopes"; const STATIC_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._scope"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 618e90158b..052a1417c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -23,11 +23,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index 89c30819de..d0d82f18f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -26,10 +26,8 @@ use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; -const SELECTOR_EXPANSION_MODULE: &str = - "sqlbuild.compiler.planner.main.selection.selector_expansion"; -const MODEL_SELECTORS_MODULE: &str = - "sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors"; +const SELECTOR_EXPANSION_MODULE: &str = "sqlbuild._native"; +const MODEL_SELECTORS_MODULE: &str = "sqlbuild._native"; const MODEL_RESOURCE_TYPE: &str = "model"; const SELECTION_OPTIONS: (&[&str], &[&str]) = ( &["project", "graph", "select"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index a4a47d1989..7860259577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -30,8 +30,7 @@ use crate::bindings::_helpers::discovery::configuration::discover_project_config use crate::bindings::_helpers::frontier::encoder::emit_error; const NATIVE_MODULE: &str = "sqlbuild._native"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index a170f4e0d9..facf69432e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -19,9 +19,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; -const UNIT_TEST_SELECTORS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; +const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 0073094454..c344aa690c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -21,13 +21,12 @@ use crate::bindings::_helpers::freshness::freshness_observe::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const CONNECTION_MODULE: &str = "sqlbuild._native"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; -const UPSTREAM_MODULE: &str = "sqlbuild.compiler.planner.main.selection._upstream"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; +const UPSTREAM_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const STATE_READ_MODULE: &str = "sqlbuild.compiler.source_freshness.main.read"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index ee70abfb5b..c4415b453d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyclass, pymethods}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; @@ -12,20 +12,15 @@ use sqlbuild_analysis::graph::main::edge_keys::edge_keys; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::main::match_selector::match_selector; use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; -use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; -use sqlbuild_analysis::graph::main::selector_name_help::selector_name_help; use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; use sqlbuild_analysis::graph::models::{ - BuildResources, EdgeDirection, GraphIndexes, GraphResource, ParsedSelector, ProjectGraph, + BuildResources, EdgeDirection, GraphIndexes, GraphResource, ProjectGraph, }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -const KIND_SELECTOR: &str = "kind"; -const PATH_SELECTOR: &str = "path"; - /// `(key, deps, tags, model folder)`. pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); type Edges = Vec<(ObjectKey, Vec)>; @@ -43,8 +38,6 @@ type UniqueIdMap = Vec<(String, Vec)>; type FailureRow = (&'static str, String, Option); /// The keys, or the `PlannerInputError` Python raises instead. type KeysOutcome = (Option>, Option); -/// `("kind", kind, value, upstream, downstream)` or `("path", start, end, upstream, downstream)`. -type ParsedRow = (&'static str, String, String, bool, bool); /// A project graph: lineage edges, tag, path and name indexes, and selector resolution. #[pyclass(module = "sqlbuild._native", frozen)] @@ -226,40 +219,6 @@ impl NativeProjectGraph { } } -/// One selector token parsed, or the `PlannerInputError` Python raises. -#[pyfunction] -fn parse_project_selector(raw: &str) -> (Option, Option) { - match parse_selector(raw) { - Ok(ParsedSelector::Kind { - kind, - value, - upstream, - downstream, - }) => ( - Some((KIND_SELECTOR, kind, value, upstream, downstream)), - None, - ), - Ok(ParsedSelector::Path { - start, - end, - upstream, - downstream, - }) => ( - Some((PATH_SELECTOR, start, end, upstream, downstream)), - None, - ), - Err(error) => (None, Some(failure(error))), - } -} - -/// The "did you mean" help for an unknown selector name, if any. -#[pyfunction] -fn project_selector_name_help(value: &str, candidates: Vec) -> Option { - selector_name_help(value, candidates.iter().map(String::as_str).collect()) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(parse_project_selector, module)?)?; - module.add_function(wrap_pyfunction!(project_selector_name_help, module)?) + module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 3f3ce3f1e5..8d2fe707be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -137,25 +137,47 @@ fn native_graph_from_views(request: Option<&Bound<'_, PyDict>>) -> PyResult { + pub(crate) all_keys: &'a Bound<'py, PyAny>, + pub(crate) upstream: &'a Bound<'py, PyAny>, + pub(crate) downstream: &'a Bound<'py, PyAny>, + pub(crate) tag_index: &'a Bound<'py, PyAny>, + pub(crate) path_index: &'a Bound<'py, PyAny>, +} + +/// A native graph over Python's index mappings, keeping their order. +pub(crate) fn graph_from_view_mappings( + mappings: GraphViewMappings<'_, '_>, +) -> PyResult { let mut names: Vec<(String, ObjectKey)> = Vec::new(); - for item in field("all_keys")?.call_method0("items")?.try_iter()? { + for item in mappings.all_keys.call_method0("items")?.try_iter()? { let (name, key): (String, Bound<'_, PyAny>) = item?.extract()?; names.push((name, native_key(&key)?)); } let mut tags: Vec<(String, Vec)> = Vec::new(); - for item in field("tag_index")?.call_method0("items")?.try_iter()? { + for item in mappings.tag_index.call_method0("items")?.try_iter()? { let (tag, keys): (String, Bound<'_, PyAny>) = item?.extract()?; tags.push((tag, native_graph_keys(&keys)?)); } let mut paths: Vec<(ObjectKey, String)> = Vec::new(); - for item in field("path_index")?.call_method0("items")?.try_iter()? { + for item in mappings.path_index.call_method0("items")?.try_iter()? { let (key, folder): (Bound<'_, PyAny>, String) = item?.extract()?; paths.push((native_key(&key)?, folder)); } graph_from_dicts(GraphIndexes { names, - upstream: native_edges(&field("upstream")?)?, - downstream: native_edges(&field("downstream")?)?, + upstream: native_edges(mappings.upstream)?, + downstream: native_edges(mappings.downstream)?, tags, paths, }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 92524026be..6f26ec444a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -42,13 +42,12 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; -const SELECTION_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selection"; -const SELECTOR_TOKENS_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_tokens"; -const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; -const SELECTOR_MATCH_MODULE: &str = "sqlbuild.compiler.planner.main.selection._selector_match"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; +const SELECTION_MODULE: &str = "sqlbuild._native"; +const SELECTOR_TOKENS_MODULE: &str = "sqlbuild._native"; +const SELECTOR_PARSE_MODULE: &str = "sqlbuild._native"; +const SELECTOR_MATCH_MODULE: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; const UNKNOWN_SELECTOR_ERROR_CODE: &str = "S007"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 76ec46b982..9b43a3d7c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -27,8 +27,7 @@ use crate::bindings::_helpers::load::load_selection::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 30b363e5b5..3d6ee4b804 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -47,6 +47,7 @@ pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; pub(crate) mod planner_identity; +pub(crate) mod planner_selection; pub(crate) mod planner_strategy; pub(crate) mod presentation_text; pub(crate) mod project_assembly; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs new file mode 100644 index 0000000000..0bf31dc9c1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/carrier.rs @@ -0,0 +1,143 @@ +//! Carry lone surrogates through the UTF-8 selector grammar as unused private-use characters. + +use std::collections::{BTreeSet, HashSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use sqlbuild_analysis::graph::errors::SelectorError; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const PRIVATE_USE: std::ops::Range = 0xF0000..0xFFFFE; + +/// `str.translate` tables to and from carrier characters; empty when no text holds a surrogate. +pub(crate) struct Carrier<'py> { + py: Python<'py>, + tables: Option<(Bound<'py, PyDict>, Bound<'py, PyDict>)>, +} + +impl<'py> Carrier<'py> { + /// A carrier for `texts`, choosing carriers that appear in neither `texts` nor `reserved`. + pub(crate) fn new( + py: Python<'py>, + texts: &[Bound<'py, PyString>], + reserved: impl FnOnce() -> PyResult>, + ) -> PyResult { + let mut surrogates: BTreeSet = BTreeSet::new(); + let mut taken: HashSet = HashSet::new(); + for text in texts { + if let Ok(plain) = text.to_str() { + taken.extend(plain.chars()); + continue; + } + for character in text.try_iter()? { + let character: Bound<'py, PyAny> = character?; + match character.extract::() { + Ok(plain) => { + taken.insert(plain); + } + Err(_) => { + surrogates.insert(ordinal(&character)?); + } + } + } + } + if surrogates.is_empty() { + return Ok(Self { py, tables: None }); + } + for name in reserved()? { + taken.extend(name.chars()); + } + let send: Bound<'py, PyDict> = PyDict::new(py); + let back: Bound<'py, PyDict> = PyDict::new(py); + let carriers = PRIVATE_USE + .filter_map(char::from_u32) + .filter(|candidate| !taken.contains(candidate)); + for (surrogate, carrier) in surrogates.into_iter().zip(carriers) { + send.set_item(surrogate, carrier)?; + back.set_item(u32::from(carrier), surrogate_text(py, surrogate)?)?; + } + Ok(Self { + py, + tables: Some((send, back)), + }) + } + + /// A carrier for one text with nothing reserved. + pub(crate) fn single(text: &Bound<'py, PyString>) -> PyResult { + Self::new(text.py(), std::slice::from_ref(text), || Ok(Vec::new())) + } + + /// The text with surrogates replaced by their carriers. + pub(crate) fn send(&self, text: &Bound<'py, PyString>) -> PyResult { + match &self.tables { + None => Ok(text.to_str()?.to_owned()), + Some((send, _)) => text.call_method1("translate", (send,))?.extract(), + } + } + + /// The texts with surrogates replaced by their carriers. + pub(crate) fn send_all(&self, texts: &[Bound<'py, PyString>]) -> PyResult> { + texts.iter().map(|text| self.send(text)).collect() + } + + /// The text with carriers restored to their surrogates. + pub(crate) fn back(&self, text: &str) -> PyResult> { + let restored: Bound<'py, PyString> = PyString::new(self.py, text); + match &self.tables { + None => Ok(restored), + Some((_, back)) => Ok(restored + .call_method1("translate", (back,))? + .downcast_into()?), + } + } + + pub(crate) fn py(&self) -> Python<'py> { + self.py + } + + /// The `PlannerInputError` instance for a selector failure, with carriers restored. + pub(crate) fn exception(&self, error: SelectorError) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(self.py); + keywords.set_item("code", error.code)?; + let help: Option> = error + .help + .as_deref() + .map(|help| self.back(help)) + .transpose()?; + keywords.set_item("help", help)?; + imported(self.py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")? + .call((self.back(&error.message)?,), Some(&keywords)) + } + + /// The `PlannerInputError` to raise for a selector failure, with carriers restored. + pub(crate) fn error(&self, error: SelectorError) -> PyErr { + match self.exception(error) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } + } +} + +/// Python strings for each item of an iterable of `str`. +pub(crate) fn py_strings<'py>(values: &Bound<'py, PyAny>) -> PyResult>> { + values + .try_iter()? + .map(|value| Ok(value?.downcast_into::()?)) + .collect() +} + +fn ordinal(character: &Bound<'_, PyAny>) -> PyResult { + let code: u32 = character + .py() + .import("builtins")? + .getattr("ord")? + .call1((character,))? + .extract()?; + Ok(code) +} + +fn surrogate_text(py: Python<'_>, code: u32) -> PyResult> { + py.import("builtins")?.getattr("chr")?.call1((code,)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs new file mode 100644 index 0000000000..0097ebb362 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod carrier; +pub(crate) mod model_selectors; +pub(crate) mod registration; +pub(crate) mod selectors; +pub(crate) mod unit_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs new file mode 100644 index 0000000000..def3598381 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs @@ -0,0 +1,121 @@ +//! `+` marker splitting and SQLBuild-owned model selectors for dbt interop. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyBool; +use pyo3::types::{PyList, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_selection::main::split_selector_expansion::split_selector_expansion as split; +use sqlbuild_analysis::planner_selection::main::sqlbuild_model_selector_names::sqlbuild_model_selector_names; +use sqlbuild_analysis::planner_selection::models::{ + ModelSelectorNames, SelectableModel, SelectorExpansion, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_selection::carrier::Carrier; + +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; + +fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { + let raised = imported(carrier.py(), SHARED_EXCEPTIONS_MODULE, "SharedInputError") + .and_then(|class| class.call1((carrier.back(message)?,))); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// Split `+selector+` marker syntax from the selector core without resolving the core. +#[pyfunction] +fn split_selector_expansion<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + let py: Python<'py> = raw.py(); + let carrier: Carrier<'py> = Carrier::single(raw)?; + let expansion: SelectorExpansion = + split(&carrier.send(raw)?).map_err(|message| shared_input_error(&carrier, &message))?; + call_with( + &imported(py, PLANNER_MODELS_MODULE, "SelectorExpansion")?, + &[ + ("core", carrier.back(&expansion.core)?.into_any()), + ( + "upstream", + PyBool::new(py, expansion.upstream).to_owned().into_any(), + ), + ( + "downstream", + expansion + .downstream + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + ) +} + +fn model_tags(model: &Bound<'_, PyAny>) -> PyResult> { + let tags: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("tags",))?; + if !tags.is_instance_of::() && !tags.is_instance_of::() { + return Ok(Vec::new()); + } + tags.try_iter()?.map(|tag| tag?.str()?.extract()).collect() +} + +fn selectable_model(model: &Bound<'_, PyAny>) -> PyResult { + Ok(SelectableModel { + name: model.getattr("name")?.extract()?, + tags: model_tags(model)?, + folder: model + .getattr("relative_path")? + .getattr("parent")? + .call_method0("as_posix")? + .extract()?, + }) +} + +/// One SQLBuild-owned model selector core's model names, and its path selector when translated. +#[pyfunction] +#[pyo3(signature = (*, project, term))] +fn resolve_sqlbuild_model_selector_names<'py>( + project: &Bound<'py, PyAny>, + term: &Bound<'py, PyString>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let models: Vec = project + .getattr("models")? + .try_iter()? + .map(|model| selectable_model(&model?)) + .collect::>()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(term), || { + Ok(models + .iter() + .flat_map(|model| [model.name.clone(), model.folder.clone()]) + .chain(models.iter().flat_map(|model| model.tags.clone())) + .collect()) + })?; + let resolved: ModelSelectorNames = sqlbuild_model_selector_names(&models, &carrier.send(term)?); + let translated: Option> = resolved + .translated + .as_deref() + .map(|path| carrier.back(path)) + .transpose()?; + PyTuple::new( + py, + [ + PyTuple::new(py, resolved.names)?.into_any(), + translated.into_pyobject(py)?.into_any(), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(split_selector_expansion, module)?)?; + module.add_function(wrap_pyfunction!( + resolve_sqlbuild_model_selector_names, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs new file mode 100644 index 0000000000..74289715bc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/registration.rs @@ -0,0 +1,11 @@ +//! Register planner selection bindings. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::planner_selection::{model_selectors, selectors, unit_tests}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + model_selectors::register(module)?; + selectors::register(module)?; + unit_tests::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs new file mode 100644 index 0000000000..2694a72991 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -0,0 +1,264 @@ +//! Planner selector parsing and resolution over native project graphs, raising `PlannerInputError`. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyFrozenSet, PySetMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::models::{ParsedSelector, ProjectGraph}; +use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::unit_test_selector_rejection; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; +use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; + +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const VIEW_OPTIONS: &[&str] = &[ + "all_keys", + "upstream_deps", + "downstream_deps", + "tag_index", + "path_index", +]; + +/// The parsed selector as Python's `ParsedSelector` or `PathSelector`. +pub(crate) fn parsed_selector<'py>( + carrier: &Carrier<'py>, + parsed: ParsedSelector, +) -> PyResult> { + let py: Python<'py> = carrier.py(); + match parsed { + ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } => call_with( + &imported(py, PLANNER_MODELS_MODULE, "ParsedSelector")?, + &[ + ( + "kind", + imported(py, PLANNER_TYPES_MODULE, "SelectorKind")?.call1((kind,))?, + ), + ("value", carrier.back(&value)?.into_any()), + ("upstream", PyBool::new(py, upstream).to_owned().into_any()), + ( + "downstream", + PyBool::new(py, downstream).to_owned().into_any(), + ), + ], + ), + ParsedSelector::Path { + start, + end, + upstream, + downstream, + } => call_with( + &imported(py, PLANNER_MODELS_MODULE, "PathSelector")?, + &[ + ("start_name", carrier.back(&start)?.into_any()), + ("end_name", carrier.back(&end)?.into_any()), + ("upstream", PyBool::new(py, upstream).to_owned().into_any()), + ( + "downstream", + PyBool::new(py, downstream).to_owned().into_any(), + ), + ], + ), + } +} + +fn graph_names(graph: &ProjectGraph) -> Vec { + graph.names.iter().map(|(name, _)| name.clone()).collect() +} + +fn selected<'py>( + carrier: &Carrier<'py>, + outcome: Result, SelectorError>, +) -> PyResult> { + let keys: Vec = outcome.map_err(|error| carrier.error(error))?; + compiled_keys(carrier.py(), &keys) +} + +fn compiled_keys<'py>(py: Python<'py>, keys: &[ObjectKey]) -> PyResult> { + PyFrozenSet::new(py, KeyCache::new(py)?.keys(keys)?) +} + +fn resolved<'py>( + graph: &ProjectGraph, + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = select.py(); + let select: Vec> = py_strings(select)?; + let exclude: Vec> = py_strings(exclude)?; + let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph)))?; + selected( + &carrier, + resolve_selectors( + graph, + &carrier.send_all(&select)?, + &carrier.send_all(&exclude)?, + ), + ) +} + +fn view_graph(mappings: GraphViewMappings<'_, '_>) -> PyResult { + graph_from_view_mappings(mappings) +} + +/// Parse one selector token into Python's `ParsedSelector` or `PathSelector`. +#[pyfunction] +fn parse_project_selector<'py>(raw: &Bound<'py, PyString>) -> PyResult> { + let carrier: Carrier<'py> = Carrier::single(raw)?; + let parsed: ParsedSelector = + parse_selector(&carrier.send(raw)?).map_err(|error| carrier.error(error))?; + parsed_selector(&carrier, parsed) +} + +/// `--select` minus `--exclude` on a native graph, adding the functions selected models need. +#[pyfunction] +#[pyo3(signature = (*, graph, select, exclude))] +fn resolve_graph_selection<'py>( + graph: PyRef<'py, NativeProjectGraph>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + resolved(graph.graph(), (select, exclude)) +} + +/// `--select` minus `--exclude` on a project graph, adding the functions selected models need. +#[pyfunction] +#[pyo3(signature = (*, graph, select, exclude))] +fn select_project_graph<'py>( + graph: &Bound<'py, PyAny>, + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, +) -> PyResult> { + let native: PyRef<'py, NativeProjectGraph> = graph.getattr("native")?.extract()?; + resolved(native.graph(), (select, exclude)) +} + +/// Model, source, seed and function selectors resolved against compiled graph mappings. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_project_selectors<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selectors", + options, + (&[&["select", "exclude"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + resolved( + graph.graph(), + (&arguments.value("select")?, &arguments.value("exclude")?), + ) +} + +/// The keys whitespace-separated selector tokens match, without required build resources. +#[pyfunction] +#[pyo3(signature = (**options))] +fn resolve_project_selector_tokens<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "resolve_project_selector_tokens", + options, + (&[&["selectors"], VIEW_OPTIONS].concat(), &[]), + )?; + let graph: NativeProjectGraph = arguments_graph(&arguments)?; + let texts: Vec> = py_strings(&arguments.value("selectors")?)?; + let carrier: Carrier<'py> = Carrier::new(py, &texts, || Ok(graph_names(graph.graph())))?; + selected( + &carrier, + resolve_selector_tokens(graph.graph(), &carrier.send_all(&texts)?), + ) +} + +fn arguments_graph(arguments: &KeywordArguments<'_>) -> PyResult { + view_graph(GraphViewMappings { + all_keys: &arguments.value("all_keys")?, + upstream: &arguments.value("upstream_deps")?, + downstream: &arguments.value("downstream_deps")?, + tag_index: &arguments.value("tag_index")?, + path_index: &arguments.value("path_index")?, + }) +} + +/// The keys one parsed selector matches before `+` graph expansion. +#[pyfunction] +#[pyo3(signature = (*, parsed, all_keys, tag_index, path_index))] +fn match_project_selector<'py>( + parsed: &Bound<'py, PyAny>, + all_keys: &Bound<'py, PyAny>, + tag_index: &Bound<'py, PyAny>, + path_index: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = parsed.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let graph: NativeProjectGraph = view_graph(GraphViewMappings { + all_keys, + upstream: &empty, + downstream: &empty, + tag_index, + path_index, + })?; + let value: Bound<'py, PyString> = parsed.getattr("value")?.downcast_into()?; + let carrier: Carrier<'py> = Carrier::new(py, std::slice::from_ref(&value), || { + Ok(graph_names(graph.graph())) + })?; + let kind: String = parsed.getattr("kind")?.str()?.extract()?; + selected( + &carrier, + match_selector(graph.graph(), &kind, &carrier.send(&value)?) + .map(|keys| keys.into_iter().collect()), + ) +} + +/// The upstream key closure of one compiled graph key. +#[pyfunction] +#[pyo3(signature = (*, key, upstream_deps))] +fn expand_project_upstream_keys<'py>( + key: Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, +) -> PyResult> { + let visited = closure(vec![key], upstream_deps, (false, None))?; + PyFrozenSet::new(upstream_deps.py(), visited.iter()) +} + +/// The error for a unit-test selector given where unit tests never run. +#[pyfunction] +#[pyo3(signature = (*, selector))] +fn unit_test_selector_error<'py>(selector: &Bound<'py, PyString>) -> PyResult> { + let carrier: Carrier<'py> = Carrier::single(selector)?; + carrier.exception(unit_test_selector_rejection(&carrier.send(selector)?)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_project_selector, module)?)?; + module.add_function(wrap_pyfunction!(resolve_graph_selection, module)?)?; + module.add_function(wrap_pyfunction!(select_project_graph, module)?)?; + module.add_function(wrap_pyfunction!(resolve_project_selectors, module)?)?; + module.add_function(wrap_pyfunction!(resolve_project_selector_tokens, module)?)?; + module.add_function(wrap_pyfunction!(match_project_selector, module)?)?; + module.add_function(wrap_pyfunction!(expand_project_upstream_keys, module)?)?; + module.add_function(wrap_pyfunction!(unit_test_selector_error, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs new file mode 100644 index 0000000000..c60a8eacd6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -0,0 +1,178 @@ +//! Unit-test selector splitting over discovered project names. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyFrozenSet, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors as split; +use sqlbuild_analysis::planner_selection::models::{ + SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; + +const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; +const RESOURCE_ENTRY_FILES: [(&str, &str); 3] = [ + ("schema_files", "model_entries"), + ("schema_files", "seed_entries"), + ("source_files", "source_entries"), +]; +const RESOURCE_PATH_FILES: [&str; 4] = [ + "model_files", + "seed_files", + "sql_function_files", + "python_function_files", +]; +const RESOURCE_NODES: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; + +fn stem(path: &Bound<'_, PyAny>) -> PyResult { + path.getattr("stem")?.extract() +} + +fn unit_test_names(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for test_file in discovered.getattr("test_files")?.try_iter()? { + let test_file: Bound<'_, PyAny> = test_file?; + for block in test_file.getattr("blocks")?.try_iter()? { + let name: Bound<'_, PyAny> = block?.getattr("name")?; + names.insert(if name.is_truthy()? { + name.extract()? + } else { + stem(&test_file.getattr("relative_path")?)? + }); + } + } + Ok(names) +} + +fn resource_names(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut names: BTreeSet = BTreeSet::new(); + for field in RESOURCE_PATH_FILES { + for file in discovered.getattr(field)?.try_iter()? { + names.insert(stem(&file?.getattr("file_path")?)?); + } + } + for (field, entries) in RESOURCE_ENTRY_FILES { + for file in discovered.getattr(field)?.try_iter()? { + for entry in file?.getattr(entries)?.try_iter()? { + names.insert(entry?.getattr("name")?.extract()?); + } + } + } + for field in RESOURCE_NODES { + for node in discovered.getattr(field)?.try_iter()? { + names.insert(node?.getattr("name")?.extract()?); + } + } + Ok(names) +} + +fn restored<'py>(carrier: &Carrier<'py>, texts: &[String]) -> PyResult> { + let texts: Vec> = texts + .iter() + .map(|text| carrier.back(text)) + .collect::>()?; + PyTuple::new(carrier.py(), texts) +} + +fn split_python<'py>( + (select, exclude): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + discovered: &Bound<'py, PyAny>, + accepts_unit_tests: bool, +) -> PyResult> { + let py: Python<'py> = select.py(); + let names = UnitTestSelectorNames { + tests: unit_test_names(discovered)?, + resources: resource_names(discovered)?, + }; + let select: Vec> = py_strings(select)?; + let exclude: Vec> = py_strings(exclude)?; + let texts: Vec> = select.iter().chain(&exclude).cloned().collect(); + let carrier: Carrier<'py> = Carrier::new(py, &texts, || { + Ok(names.tests.union(&names.resources).cloned().collect()) + })?; + let (select, exclude): (Vec, Vec) = + (carrier.send_all(&select)?, carrier.send_all(&exclude)?); + let split: UnitTestSelectorSplit = split( + SelectorGroups { + select: &select, + exclude: &exclude, + }, + &names, + accepts_unit_tests, + ) + .map_err(|error| carrier.error(error))?; + let selection: Bound<'py, PyAny> = call_with( + &imported(py, PLANNER_MODELS_MODULE, "SqlTestSelection")?, + &[ + ("names", PyFrozenSet::new(py, &split.test_names)?.into_any()), + ( + "excluded_names", + PyFrozenSet::new(py, &split.excluded_test_names)?.into_any(), + ), + ( + "models_selected", + split + .models_selected + .into_pyobject(py)? + .to_owned() + .into_any(), + ), + ], + )?; + call_with( + &imported(py, PLANNER_MODELS_MODULE, "UnitTestSelectorSplit")?, + &[ + ("select", restored(&carrier, &split.select)?.into_any()), + ("exclude", restored(&carrier, &split.exclude)?.into_any()), + ("sql_test_selection", selection), + ], + ) +} + +/// Move unit-test selectors out of resource selectors, or reject them for this command. +#[pyfunction] +#[pyo3(signature = (*, select, exclude, discovered_inputs, accepts_unit_tests))] +fn split_project_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + accepts_unit_tests: bool, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, accepts_unit_tests) +} + +/// Split unit-test selectors from resource selectors for a command that runs unit tests. +#[pyfunction] +#[pyo3(signature = (*, select, exclude, discovered_inputs))] +fn split_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, true) +} + +/// Raise when a selector names a unit test; return the unchanged resource selectors. +#[pyfunction] +#[pyo3(signature = (*, select, exclude, discovered_inputs))] +fn reject_unit_test_selectors<'py>( + select: &Bound<'py, PyAny>, + exclude: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + split_python((select, exclude), discovered_inputs, false) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(split_project_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(split_unit_test_selectors, module)?)?; + module.add_function(wrap_pyfunction!(reject_unit_test_selectors, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 8e58cd6110..821c8ef319 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -25,8 +25,7 @@ use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index e1d55181ec..578f78ab95 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -8,8 +8,8 @@ use crate::bindings::_helpers::{ compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_identity, - planner_strategy, project_assembly, refactor, references, render, rules, scopes, - semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + planner_selection, planner_strategy, project_assembly, refactor, references, render, rules, + scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; @@ -47,6 +47,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { plan_output::registration::register(module)?; planner_changes::registration::register(module)?; planner_identity::registration::register(module)?; + planner_selection::registration::register(module)?; planner_strategy::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 0c241bde84..e2bc094ac3 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -46,9 +46,6 @@ "src/sqlbuild/integrations/dbt/_helpers/planning/model_planning.py", } ) -ALLOWED_SELECTOR_PARSE_PATH: str = ( - "src/sqlbuild/compiler/planner/main/selection/selector_expansion.py" -) CLIENT_STYLE_PREFIXES: frozenset[tuple[str, ...]] = frozenset( {("src", "sqlbuild", "adapters"), ("src", "sqlbuild", "integrations")} ) diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index 69bc123b1f..b2e10b9933 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -11,7 +11,6 @@ from scripts.fensu_policy.constants import ( ALLOWED_DBT_REF_SCAN_PATHS, ALLOWED_MACRO_LOAD_PATHS, - ALLOWED_SELECTOR_PARSE_PATH, ALLOWED_SOURCE_FRESHNESS_INSERT_PREFIXES, DBT_INTEGRATION_PATH_PREFIX, GRAPH_KEY_CLASS_NAMES, @@ -97,15 +96,12 @@ def dbt_graph_projection(*, module: object, ctx: RuleContext) -> list[Fault]: family=Family.CUSTOM, slug="selector-marker-parsing", message="selector + markers must be parsed by split_selector_expansion", - remediation="Use compiler.planner.main.selection.selector_expansion.split_selector_expansion.", + remediation="Use sqlbuild._native.split_selector_expansion.", ) def selector_marker_parsing(*, module: object, ctx: RuleContext) -> list[Fault]: del module path: str = "/".join(ctx.repo_relative_parts()) - if ( - not (DBT_INTEGRATION_PATH_PREFIX in path or PLANNER_PATH_PREFIX in path) - or path == ALLOWED_SELECTOR_PARSE_PATH - ): + if not (DBT_INTEGRATION_PATH_PREFIX in path or PLANNER_PATH_PREFIX in path): return [] return [ ctx.fault_at(location=call.location) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 33cdd8868b..9cb0c22209 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -195,6 +195,7 @@ from sqlbuild.compiler.pipeline.models import ( from sqlbuild.compiler.planner.classes.migration_state_inspection import ( MigrationStateInspection, ) +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, ChangeDetectionResult, @@ -205,6 +206,8 @@ from sqlbuild.compiler.planner.models import ( MaximumStartSafetyEvidence, MissingUpstream, ModelPlanEntry, + ParsedSelector, + PathSelector, PlannerChangeResults, PlannerResolvedActions, PlannerRuntime, @@ -216,7 +219,9 @@ from sqlbuild.compiler.planner.models import ( SchemaFinding, SelectionStalenessGraph, SelectionStalenessWarning, + SelectorExpansion, SqlTestPlanEntry, + UnitTestSelectorSplit, WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( @@ -1221,10 +1226,68 @@ class NativeProjectGraph: self, selected: list[_GraphKey], include: tuple[bool, bool, bool] ) -> list[_GraphKey]: ... -def parse_project_selector( - raw: str, -) -> tuple[tuple[str, str, str, bool, bool] | None, _SelectorFailure | None]: ... -def project_selector_name_help(value: str, candidates: list[str]) -> str | None: ... +def parse_project_selector(raw: str) -> ParsedSelector | PathSelector: ... +def resolve_graph_selection( + *, graph: NativeProjectGraph, select: Sequence[str], exclude: Sequence[str] +) -> frozenset[CompiledObjectKey]: ... +def select_project_graph( + *, graph: ProjectGraph, select: Sequence[str], exclude: Sequence[str] +) -> frozenset[CompiledObjectKey]: ... +def resolve_project_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + all_keys: Mapping[str, CompiledObjectKey], + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> frozenset[CompiledObjectKey]: ... +def resolve_project_selector_tokens( + *, + selectors: Sequence[str], + all_keys: Mapping[str, CompiledObjectKey], + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + downstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> frozenset[CompiledObjectKey]: ... +def match_project_selector( + *, + parsed: ParsedSelector, + all_keys: Mapping[str, CompiledObjectKey], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> frozenset[CompiledObjectKey]: ... +def expand_project_upstream_keys( + *, + key: CompiledObjectKey, + upstream_deps: Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]], +) -> frozenset[CompiledObjectKey]: ... +def unit_test_selector_error(*, selector: str) -> PlannerInputError: ... +def split_project_unit_test_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + discovered_inputs: DiscoveredProjectInputs, + accepts_unit_tests: bool, +) -> UnitTestSelectorSplit: ... +def split_unit_test_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + discovered_inputs: DiscoveredProjectInputs, +) -> UnitTestSelectorSplit: ... +def reject_unit_test_selectors( + *, + select: Sequence[str], + exclude: Sequence[str], + discovered_inputs: DiscoveredProjectInputs, +) -> UnitTestSelectorSplit: ... +def split_selector_expansion(raw: str) -> SelectorExpansion: ... +def resolve_sqlbuild_model_selector_names( + *, project: CompiledProjectView, term: str +) -> tuple[tuple[str, ...], str | None]: ... class NativeReuseAttempt: def outcome(self) -> str: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index e5d80c7ce0..d26fbb6e19 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -11,6 +11,7 @@ enforce_model_execution_limit, resolve_target_config, run_clone_pipeline_with_options, + split_unit_test_selectors, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.planning.external_refs import ( @@ -35,7 +36,6 @@ CompiledResourceType, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.unit_test_selection import split_unit_test_selectors from sqlbuild.compiler.pipeline.models import ( ClonePipelineConnection, ClonePipelineOptions, diff --git a/src/sqlbuild/compiler/pipeline/main/_project_graph_selection.py b/src/sqlbuild/compiler/pipeline/main/_project_graph_selection.py deleted file mode 100644 index 8a5017538f..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_project_graph_selection.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Selector resolution over a project graph.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection._graph_selection import resolve_graph_selection - - -def select_project_graph( - *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> frozenset[CompiledObjectKey]: - """Resolve `--select`/`--exclude` on the graph, adding the functions they need to build.""" - - return resolve_graph_selection(graph=graph.native, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/pipeline/main/_reject_unit_test_selectors.py b/src/sqlbuild/compiler/pipeline/main/_reject_unit_test_selectors.py deleted file mode 100644 index adedaf289c..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/_reject_unit_test_selectors.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public validation that a command which never runs unit tests selects none.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.main.selection._split_unit_test_selectors import ( - split_project_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import UnitTestSelectorSplit - - -def reject_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, -) -> UnitTestSelectorSplit: - """Raise when a selector names a unit test; return the unchanged resource selectors.""" - - return split_project_unit_test_selectors( - select=select, - exclude=exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=False, - ) diff --git a/src/sqlbuild/compiler/pipeline/main/unit_test_selection.py b/src/sqlbuild/compiler/pipeline/main/unit_test_selection.py deleted file mode 100644 index 064b50b1b4..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/unit_test_selection.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public split of unit-test selectors from resource selectors for one command.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.main.selection._split_unit_test_selectors import ( - split_project_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import UnitTestSelectorSplit - - -def split_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, -) -> UnitTestSelectorSplit: - """Split unit-test selectors from resource selectors for a command that runs unit tests.""" - - return split_project_unit_test_selectors( - select=select, - exclude=exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=True, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py index a2e84bf9be..8fee6aa857 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py @@ -7,16 +7,14 @@ build_execution_edge_origins, build_execution_upstream_deps, lineage_graph_names, + parse_project_selector, + resolve_graph_selection, topologically_order_keys, ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys from sqlbuild.compiler.planner._helpers.graph.loader_dag import expand_selected_loader_dependencies -from sqlbuild.compiler.planner._helpers.graph.selectors import ( - parse_selector, - resolve_graph_selectors, -) from sqlbuild.compiler.planner.models import ( ParsedSelector, PathSelector, @@ -46,7 +44,7 @@ def build_planner_scope( resolved_selected_keys: frozenset[CompiledObjectKey] = ( selected_keys if selected_keys is not None - else resolve_graph_selectors(graph=project.lineage_graph, select=select, exclude=exclude) + else resolve_graph_selection(graph=project.lineage_graph, select=select, exclude=exclude) ) executable_dependency_source_keys: frozenset[CompiledObjectKey] = ( _upstream_source_selector_keys( @@ -105,7 +103,7 @@ def _upstream_source_selector_keys( for token in raw_select.split(): part: str for part in token.split(","): - parsed: ParsedSelector | PathSelector = parse_selector(part) + parsed: ParsedSelector | PathSelector = parse_project_selector(part) if isinstance(parsed, PathSelector) or not parsed.upstream: continue if parsed.kind not in {SelectorKind.NAME, SelectorKind.SOURCE}: diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py b/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py deleted file mode 100644 index 9d6412f3fe..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Planner selector boundary over the native selector grammar and graph resolution.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable, Iterator, Mapping - -import sqlbuild._native as _native -from sqlbuild._native import compiled_graph_keys, native_graph_from_views, native_graph_keys -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner.constants import ( - UNIT_TEST_SELECTOR_ERROR_CODE, - UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector -from sqlbuild.compiler.planner.types import SelectorKind - -_KIND_SELECTOR: str = "kind" -_SURROGATE_START: int = 0xD800 -_SURROGATE_END: int = 0xDFFF -_PRIVATE_USE_START: int = 0xF0000 -_PRIVATE_USE_END: int = 0xFFFFE - -type _Edges = Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] -type _Failure = tuple[str, str, str | None] -type _Outcome = tuple[list[tuple[str, str]] | None, _Failure | None] - - -class _SurrogateCarrier: - """Carry lone surrogates through the UTF-8 grammar as unused private-use characters.""" - - def __init__(self, *, texts: Iterable[str], reserved: Callable[[], Iterable[str]]) -> None: - joined: str = "".join(texts) - surrogates: list[str] = sorted( - {character for character in joined if _is_surrogate(character)} - ) - self._sent: dict[str, str] = {} - self._back: dict[str, str] = {} - if not surrogates: - return - taken: set[str] = set(joined).union(*map(set, reserved())) - candidates: Iterator[str] = ( - chr(point) for point in range(_PRIVATE_USE_START, _PRIVATE_USE_END) - ) - for surrogate, carrier in zip( - surrogates, - (character for character in candidates if character not in taken), - strict=False, - ): - self._sent[surrogate] = carrier - self._back[carrier] = surrogate - - def send(self, text: str) -> str: - return text.translate(str.maketrans(self._sent)) if self._sent else text - - def back(self, text: str | None) -> str | None: - if text is None or not self._back: - return text - return text.translate(str.maketrans(self._back)) - - def failure(self, failure: _Failure | None) -> _Failure | None: - if failure is None: - return None - code, message, help_text = failure - return (code, self.back(message) or "", self.back(help_text)) - - -def _is_surrogate(character: str) -> bool: - return _SURROGATE_START <= ord(character) <= _SURROGATE_END - - -def _graph_names(graph: _native.NativeProjectGraph) -> Callable[[], Iterable[str]]: - return lambda: (name for name, _ in graph.names()) - - -def parse_selector(raw: str) -> ParsedSelector | PathSelector: - """Parse one raw selector token into a structured form.""" - - carrier: _SurrogateCarrier = _SurrogateCarrier(texts=(raw,), reserved=tuple) - parsed, failure = _native.parse_project_selector(carrier.send(raw)) - if parsed is None: - raise _selector_error(carrier.failure(failure)) - shape, first, second, upstream, downstream = parsed - if shape == _KIND_SELECTOR: - return ParsedSelector( - kind=SelectorKind(first), - value=carrier.back(second) or "", - upstream=upstream, - downstream=downstream, - ) - return PathSelector( - start_name=carrier.back(first) or "", - end_name=carrier.back(second) or "", - upstream=upstream, - downstream=downstream, - ) - - -def resolve_graph_selectors( - *, - graph: _native.NativeProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[CompiledObjectKey]: - """Resolve select/exclude strings against a native graph, adding required functions.""" - - carrier: _SurrogateCarrier = _SurrogateCarrier( - texts=(*select, *exclude), reserved=_graph_names(graph) - ) - return _selected( - outcome=graph.resolve(list(map(carrier.send, select)), list(map(carrier.send, exclude))), - carrier=carrier, - ) - - -def resolve_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_keys: Mapping[str, CompiledObjectKey], - upstream: _Edges, - downstream: _Edges, - tag_index: Mapping[str, Iterable[CompiledObjectKey]] | None = None, - path_index: Mapping[CompiledObjectKey, str] | None = None, -) -> frozenset[CompiledObjectKey]: - """Resolve raw select/exclude strings into a final set of object keys.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index or {}, - path_index=path_index or {}, - ) - return resolve_graph_selectors(graph=graph, select=select, exclude=exclude) - - -def resolve_selector_tokens( - *, - selectors: tuple[str, ...], - all_keys: Mapping[str, CompiledObjectKey], - upstream: _Edges, - downstream: _Edges, - tag_index: Mapping[str, Iterable[CompiledObjectKey]], - path_index: Mapping[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Union the keys matched by whitespace-separated selector tokens, without build expansion.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - carrier: _SurrogateCarrier = _SurrogateCarrier(texts=selectors, reserved=_graph_names(graph)) - return _selected(outcome=graph.tokens(list(map(carrier.send, selectors))), carrier=carrier) - - -def match_selector_keys( - *, - parsed: ParsedSelector, - all_keys: Mapping[str, CompiledObjectKey], - tag_index: Mapping[str, Iterable[CompiledObjectKey]], - path_index: Mapping[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Return the keys one parsed selector matches before `+` graph expansion.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys=all_keys, upstream={}, downstream={}, tag_index=tag_index, path_index=path_index - ) - carrier: _SurrogateCarrier = _SurrogateCarrier( - texts=(parsed.value,), reserved=_graph_names(graph) - ) - return _selected( - outcome=graph.matched(str(parsed.kind), carrier.send(parsed.value)), carrier=carrier - ) - - -def expand_required_build_resources( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream: _Edges, - downstream: _Edges, - include_upstream_functions: bool = True, - include_upstream_seeds: bool = False, - include_downstream_functions: bool = False, -) -> frozenset[CompiledObjectKey]: - """Add non-model resources needed to build a coherent selected model scope.""" - - graph: _native.NativeProjectGraph = native_graph_from_views( - all_keys={}, upstream=upstream, downstream=downstream, tag_index={}, path_index={} - ) - return compiled_graph_keys( - graph.build_resources( - native_graph_keys(selected_keys), - (include_upstream_functions, include_upstream_seeds, include_downstream_functions), - ) - ) - - -def selector_name_help(*, value: str, candidates: Iterable[str]) -> str | None: - """Suggest the nearest selectable names for an unknown selector name.""" - - names: list[str] = list(candidates) - carrier: _SurrogateCarrier = _SurrogateCarrier(texts=(value,), reserved=lambda: names) - return carrier.back(_native.project_selector_name_help(carrier.send(value), names)) - - -def unit_test_selector_rejection(*, selector: str) -> PlannerInputError: - """The error for a unit-test selector given to a command that does not run unit tests.""" - - return PlannerInputError( - f"selector '{selector}' selects a unit test; {UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD}", - code=UNIT_TEST_SELECTOR_ERROR_CODE, - ) - - -def _selected(*, outcome: _Outcome, carrier: _SurrogateCarrier) -> frozenset[CompiledObjectKey]: - keys, failure = outcome - if keys is None: - raise _selector_error(carrier.failure(failure)) - return compiled_graph_keys(keys) - - -def _selector_error(failure: _Failure | None) -> PlannerInputError: - if failure is None: - return PlannerInputError("selector resolution returned neither keys nor an error") - code, message, help_text = failure - return PlannerInputError(message, code=code, help=help_text) diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index d48996d26e..b735c1ee9d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -9,6 +9,7 @@ build_downstream_deps, build_execution_upstream_deps, get_materialization_type, + resolve_graph_selection, topologically_order_keys, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -22,7 +23,6 @@ CompiledProjectView, CompiledSourceView, ) -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors from sqlbuild.compiler.planner._helpers.resolve.refs import ( build_function_locations, build_model_locations, @@ -65,7 +65,7 @@ def build_clone_plan_output( downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( upstream_deps ) - selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selectors( + selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selection( graph=project.lineage_graph, select=select, exclude=exclude ) return PlanOutput( diff --git a/src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py b/src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py deleted file mode 100644 index 17b39561dc..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/selection/unit_test_selectors.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Unit-test selectors: split them from resource selectors or reject them where tests never run.""" - -from __future__ import annotations - -from fnmatch import fnmatchcase - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.graph.selectors import ( - parse_selector, - selector_name_help, - unit_test_selector_rejection, -) -from sqlbuild.compiler.planner.constants import ( - UNIT_TEST_SELECTOR_ERROR_CODE, - UNKNOWN_SELECTOR_ERROR_CODE, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - ParsedSelector, - PathSelector, - SqlTestSelection, - UnitTestSelectorNames, - UnitTestSelectorSplit, -) -from sqlbuild.compiler.planner.types import SelectorKind - - -def split_discovered_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, - accepts_unit_tests: bool, -) -> UnitTestSelectorSplit: - """Split unit-test selectors from resource selectors using discovered project names.""" - - return split_unit_test_selectors( - select=select, - exclude=exclude, - names=UnitTestSelectorNames( - tests=_unit_test_names(discovered_inputs), - resources=_resource_names(discovered_inputs), - ), - accepts_unit_tests=accepts_unit_tests, - ) - - -def split_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - names: UnitTestSelectorNames, - accepts_unit_tests: bool, -) -> UnitTestSelectorSplit: - """Move unit-test selectors out of resource selectors, or reject them for this command.""" - - selected_tests, resource_select = _split_groups( - groups=select, names=names, accepts_unit_tests=accepts_unit_tests - ) - excluded_tests, resource_exclude = _split_groups( - groups=exclude, names=names, accepts_unit_tests=accepts_unit_tests - ) - return UnitTestSelectorSplit( - select=resource_select, - exclude=resource_exclude, - sql_test_selection=SqlTestSelection( - names=selected_tests, - excluded_names=excluded_tests, - models_selected=not select or bool(resource_select) or not selected_tests, - ), - ) - - -def _split_groups( - *, groups: tuple[str, ...], names: UnitTestSelectorNames, accepts_unit_tests: bool -) -> tuple[frozenset[str], tuple[str, ...]]: - tests: set[str] = set() - kept_groups: list[str] = [] - for group in groups: - tokens: list[str] = group.split() - kept_tokens: list[str] = [] - for token in tokens: - matched, keep = _split_token( - token=token, names=names, accepts_unit_tests=accepts_unit_tests - ) - tests.update(matched) - if keep: - kept_tokens.append(token) - if len(kept_tokens) == len(tokens): - kept_groups.append(group) - elif kept_tokens: - kept_groups.append(" ".join(kept_tokens)) - return frozenset(tests), tuple(kept_groups) - - -def _split_token( - *, token: str, names: UnitTestSelectorNames, accepts_unit_tests: bool -) -> tuple[frozenset[str], bool]: - parts: list[str] = token.split(",") - classified: list[tuple[frozenset[str], bool]] = [ - _classify(part=part, names=names, accepts_unit_tests=accepts_unit_tests) for part in parts - ] - selects_tests: bool = any(matched and not keep for matched, keep in classified) - if len(parts) > 1 and selects_tests: - for part in parts: - parse_selector(part) - raise PlannerInputError( - f"selector '{token}' intersects a unit test with ','; unit tests can only be " - "selected whole, by name", - code=UNIT_TEST_SELECTOR_ERROR_CODE, - ) - if len(parts) > 1: - return frozenset(), True - return classified[0] - - -def _classify( - *, part: str, names: UnitTestSelectorNames, accepts_unit_tests: bool -) -> tuple[frozenset[str], bool]: - """Return the unit tests one selector selects and whether it also selects resources.""" - - try: - parsed: ParsedSelector | PathSelector = parse_selector(part) - except PlannerInputError: - return frozenset(), True - if isinstance(parsed, PathSelector) or parsed.kind not in { - SelectorKind.NAME, - SelectorKind.TEST, - }: - return frozenset(), True - matched_tests: frozenset[str] = _matching(value=parsed.value, names=names.tests) - selects_resources: bool = parsed.kind == SelectorKind.NAME and bool( - _matching(value=parsed.value, names=names.resources) - ) - if selects_resources: - return frozenset(), True - if not matched_tests: - if parsed.kind == SelectorKind.TEST or accepts_unit_tests: - _raise_unknown(parsed=parsed, names=names) - return frozenset(), True - if not accepts_unit_tests: - raise unit_test_selector_rejection(selector=part) - if parsed.upstream or parsed.downstream: - raise PlannerInputError( - f"selector '{part}' expands a unit test with '+'; unit tests have no lineage, so " - "select them by name without '+'", - code=UNIT_TEST_SELECTOR_ERROR_CODE, - ) - return matched_tests, False - - -def _raise_unknown(*, parsed: ParsedSelector, names: UnitTestSelectorNames) -> None: - label: str = "pattern" if _is_pattern(parsed.value) else "name" - candidates: frozenset[str] = names.tests - subject: str = "unit test" - if parsed.kind == SelectorKind.NAME: - candidates = names.tests | names.resources - subject = "selector" - raise PlannerInputError( - f"unknown {subject} {label} '{parsed.value}'", - code=UNKNOWN_SELECTOR_ERROR_CODE, - help=selector_name_help(value=parsed.value, candidates=candidates), - ) - - -def _matching(*, value: str, names: frozenset[str]) -> frozenset[str]: - if not _is_pattern(value): - return frozenset({value}) & names - return frozenset(name for name in names if fnmatchcase(name, value)) - - -def _is_pattern(value: str) -> bool: - return any(character in value for character in "*?[") - - -def _unit_test_names(discovered_inputs: DiscoveredProjectInputs) -> frozenset[str]: - names: set[str] = set() - for test_file in discovered_inputs.test_files: - names.update(block.name or test_file.relative_path.stem for block in test_file.blocks) - return frozenset(names) - - -def _resource_names(discovered_inputs: DiscoveredProjectInputs) -> frozenset[str]: - names: set[str] = {model_file.file_path.stem for model_file in discovered_inputs.model_files} - for schema_file in discovered_inputs.schema_files: - names.update(entry.name for entry in schema_file.model_entries) - names.update(entry.name for entry in schema_file.seed_entries) - for source_file in discovered_inputs.source_files: - names.update(entry.name for entry in source_file.source_entries) - names.update(seed_file.file_path.stem for seed_file in discovered_inputs.seed_files) - names.update( - function_file.file_path.stem - for function_file in ( - *discovered_inputs.sql_function_files, - *discovered_inputs.python_function_files, - ) - ) - names.update( - node.name - for node in ( - *discovered_inputs.loader_functions, - *discovered_inputs.task_functions, - *discovered_inputs.asset_functions, - *discovered_inputs.check_functions, - ) - ) - return frozenset(names) diff --git a/src/sqlbuild/compiler/planner/main/selection/_build_resources.py b/src/sqlbuild/compiler/planner/main/selection/_build_resources.py deleted file mode 100644 index c8f1499525..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_build_resources.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Compiler-internal required-build-resource expansion entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import expand_required_build_resources - - -def expand_build_resource_selection( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - include_upstream_functions: bool = True, - include_upstream_seeds: bool = False, - include_downstream_functions: bool = False, -) -> frozenset[CompiledObjectKey]: - """Expand a selected SQL scope with required build resources.""" - - return expand_required_build_resources( - selected_keys=selected_keys, - upstream=upstream, - downstream=downstream, - include_upstream_functions=include_upstream_functions, - include_upstream_seeds=include_upstream_seeds, - include_downstream_functions=include_downstream_functions, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py b/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py deleted file mode 100644 index 71c954d22f..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public selector resolution over a native project graph.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors - - -def resolve_graph_selection( - *, - graph: _native.NativeProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[CompiledObjectKey]: - """Resolve select/exclude strings natively, adding the functions selected models need.""" - - return resolve_graph_selectors(graph=graph, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/planner/main/selection/_selector_match.py b/src/sqlbuild/compiler/planner/main/selection/_selector_match.py deleted file mode 100644 index 88244d4e29..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_selector_match.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public entrypoint matching one parsed selector before graph expansion.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import match_selector_keys -from sqlbuild.compiler.planner.models import ParsedSelector - - -def match_project_selector( - *, - parsed: ParsedSelector, - all_keys: dict[str, CompiledObjectKey], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Return the keys one parsed selector matches before `+` graph expansion.""" - - return match_selector_keys( - parsed=parsed, - all_keys=all_keys, - tag_index=tag_index, - path_index=path_index, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py b/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py deleted file mode 100644 index a69d10ba0f..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_selector_tokens.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public selector-token resolution entrypoint without required build resources.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selector_tokens - - -def resolve_project_selector_tokens( - *, - selectors: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Resolve selector strings to the keys they match, without adding required build resources.""" - - return resolve_selector_tokens( - selectors=selectors, - all_keys=all_keys, - upstream=upstream_deps, - downstream=downstream_deps, - tag_index=tag_index, - path_index=path_index, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py b/src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py deleted file mode 100644 index 4d7a8a7270..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_split_unit_test_selectors.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Compiler-internal unit-test selector split entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner._helpers.selection.unit_test_selectors import ( - split_discovered_unit_test_selectors, -) -from sqlbuild.compiler.planner.models import UnitTestSelectorSplit - - -def split_project_unit_test_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - discovered_inputs: DiscoveredProjectInputs, - accepts_unit_tests: bool, -) -> UnitTestSelectorSplit: - """Move unit-test selectors out of resource selectors, or reject them for this command.""" - - return split_discovered_unit_test_selectors( - select=select, - exclude=exclude, - discovered_inputs=discovered_inputs, - accepts_unit_tests=accepts_unit_tests, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_upstream.py b/src/sqlbuild/compiler/planner/main/selection/_upstream.py deleted file mode 100644 index d62e6c8789..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_upstream.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Public upstream graph expansion entrypoint for planner consumers.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure -from sqlbuild.compiler.compile.models import CompiledObjectKey - - -def expand_project_upstream_keys( - *, - key: CompiledObjectKey, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return the upstream key closure for one compiled graph key.""" - - return frozenset(transitive_closure(start=key, edges=upstream_deps)) diff --git a/src/sqlbuild/compiler/planner/main/selection/selection.py b/src/sqlbuild/compiler/planner/main/selection/selection.py deleted file mode 100644 index 8190db37a9..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/selection.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public selector-resolution entrypoints for planner consumers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selectors - - -def resolve_project_selectors( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Resolve model/source/seed/function selectors against a compiled graph.""" - - return resolve_selectors( - select=select, - exclude=exclude, - all_keys=all_keys, - upstream=upstream_deps, - downstream=downstream_deps, - tag_index=tag_index, - path_index=path_index, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/selector_expansion.py b/src/sqlbuild/compiler/planner/main/selection/selector_expansion.py deleted file mode 100644 index 2db5d73406..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/selector_expansion.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public selector expansion marker parsing entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.constants import SELECTOR_EXPANSION_MARKER -from sqlbuild.compiler.planner.models import SelectorExpansion -from sqlbuild.errors.contracts.exceptions import SharedInputError - - -def split_selector_expansion(raw: str) -> SelectorExpansion: - """Split +selector+ marker syntax from the selector core without resolving the core.""" - - stripped: str = raw.strip() - if not stripped: - raise SharedInputError("empty selector") - upstream: bool = stripped.startswith("+") - downstream: bool = stripped.endswith("+") - core: str = stripped.lstrip("+").rstrip("+") - if not core: - raise SharedInputError(f"selector '{stripped}' has no name after removing '+' markers") - if SELECTOR_EXPANSION_MARKER in core: - raise SharedInputError(f"selector '{stripped}' contains '+' in an unsupported position") - return SelectorExpansion(core=core, upstream=upstream, downstream=downstream) diff --git a/src/sqlbuild/compiler/planner/main/selection/selector_parse.py b/src/sqlbuild/compiler/planner/main/selection/selector_parse.py deleted file mode 100644 index 4cdc4a7652..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/selector_parse.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public selector parsing entrypoint for planner consumers.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.graph.selectors import parse_selector -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector - - -def parse_project_selector(raw: str) -> ParsedSelector | PathSelector: - """Parse one project selector token into a structured form.""" - - return parse_selector(raw) diff --git a/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py b/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py deleted file mode 100644 index 6e662fdae3..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Public SQLBuild model selector resolution entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView - - -def resolve_sqlbuild_model_selector_names( - *, project: CompiledProjectView, term: str -) -> tuple[tuple[str, ...], str | None]: - """Resolve one SQLBuild-owned model selector core to names and path translation.""" - - models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} - if term in models_by_name: - return (term,), None - if term.startswith("tag:"): - tag: str = term.removeprefix("tag:") - names: list[str] = [] - for model in project.models: - raw_tags: object = model.config.values.get("tags") - tags: tuple[str, ...] = () - if isinstance(raw_tags, tuple): - tags = tuple(str(item) for item in raw_tags) - elif isinstance(raw_tags, list): - tags = tuple(str(item) for item in raw_tags) - if tag in tags: - names.append(model.name) - return tuple(names), None - if term.startswith("path:"): - raw_path: str = term.removeprefix("path:") - translated_path: str = raw_path.replace("\\", "/") - names = [ - model.name - for model in project.models - if (model_path := model.relative_path.parent.as_posix()) == translated_path - or model_path.startswith(f"{translated_path}/") - ] - return tuple(names), f"path:{translated_path}" if translated_path != raw_path else None - return (), None diff --git a/src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py b/src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py deleted file mode 100644 index 2aab683cb6..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/unit_test_selector_error.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public error for unit-test selectors where unit tests never run.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.graph.selectors import unit_test_selector_rejection -from sqlbuild.compiler.planner.exceptions import PlannerInputError - - -def unit_test_selector_error(*, selector: str) -> PlannerInputError: - """The error for a unit-test selector given where unit tests never run.""" - - return unit_test_selector_rejection(selector=selector) diff --git a/src/sqlbuild/integrations/dagster/_helpers/selection.py b/src/sqlbuild/integrations/dagster/_helpers/selection.py index 3007e05646..25ac702707 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/selection.py +++ b/src/sqlbuild/integrations/dagster/_helpers/selection.py @@ -5,12 +5,13 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlbuild._native import path_nodes, transitive_closure_many -from sqlbuild.compiler.planner.constants import SELECTOR_KIND_SEPARATOR -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector -from sqlbuild.compiler.planner.main.selection.unit_test_selector_error import ( +from sqlbuild._native import ( + parse_project_selector, + path_nodes, + transitive_closure_many, unit_test_selector_error, ) +from sqlbuild.compiler.planner.constants import SELECTOR_KIND_SEPARATOR from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.planner.types import SelectorKind from sqlbuild.integrations.dagster._helpers.dag import load_sqlbuild_dag diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index b4605751e6..312bb5130e 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -153,8 +153,8 @@ def test_given_dbt_code_when_checking_graph_projection_then_matches_contract( expected_fault_count=2, ), CustomRuleTestCase( - description="central selector parsing passes", - path="src/sqlbuild/compiler/planner/main/selection/selector_expansion.py", + description="selector parsing outside the planner and dbt passes", + path="src/sqlbuild/cli/commands/_helpers/selection.py", source="result = raw.startswith('+'), raw.lstrip('+')\n", expected_fault_count=0, ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py new file mode 100644 index 0000000000..1858c9a743 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/helpers.py @@ -0,0 +1,33 @@ +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + + +def build_selector_discovered_inputs( + *, tests: frozenset[str], resources: frozenset[str] +) -> DiscoveredProjectInputs: + """Discovered inputs naming these unit tests and, as model files, these resources.""" + + inputs: SimpleNamespace = SimpleNamespace( + test_files=( + SimpleNamespace( + relative_path=Path("tests/unit_tests.sql"), + blocks=tuple(SimpleNamespace(name=name) for name in sorted(tests)), + ), + ), + model_files=tuple( + SimpleNamespace(file_path=Path(f"models/{name}.sql")) for name in sorted(resources) + ), + schema_files=(), + source_files=(), + seed_files=(), + sql_function_files=(), + python_function_files=(), + loader_functions=(), + task_functions=(), + asset_functions=(), + check_functions=(), + ) + return cast(DiscoveredProjectInputs, inputs) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index 3c4352512d..0dfa091810 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -4,21 +4,19 @@ import pytest -from sqlbuild.compiler.planner._helpers.selection.unit_test_selectors import ( - split_unit_test_selectors, -) +from sqlbuild._native import split_project_unit_test_selectors +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ( - SqlTestSelection, - UnitTestSelectorNames, - UnitTestSelectorSplit, -) +from sqlbuild.compiler.planner.models import SqlTestSelection, UnitTestSelectorSplit from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner._helpers.selection.helpers import ( + build_selector_discovered_inputs, +) -_NAMES: UnitTestSelectorNames = UnitTestSelectorNames( +_DISCOVERED: DiscoveredProjectInputs = build_selector_discovered_inputs( tests=frozenset({"order_status", "order_totals_check", "customer_status"}), resources=frozenset({"orders", "order_totals", "customers", "raw_orders"}), ) @@ -93,10 +91,10 @@ def test_given_selectors_when_splitting_unit_tests_then_tests_and_resources_separate( test_case: UnitTestSelectorSplitTestCase, ) -> None: - split: UnitTestSelectorSplit = split_unit_test_selectors( + split: UnitTestSelectorSplit = split_project_unit_test_selectors( select=test_case.select, exclude=test_case.exclude, - names=_NAMES, + discovered_inputs=_DISCOVERED, accepts_unit_tests=True, ) @@ -183,10 +181,10 @@ def test_given_invalid_unit_test_selectors_when_splitting_then_it_raises_clear_e test_case: UnitTestSelectorErrorTestCase, ) -> None: with pytest.raises(PlannerInputError) as raised: - split_unit_test_selectors( + split_project_unit_test_selectors( select=test_case.select, exclude=(), - names=_NAMES, + discovered_inputs=_DISCOVERED, accepts_unit_tests=test_case.accepts_unit_tests, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index 467de551a9..63ff653331 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -2,15 +2,16 @@ import pytest -from sqlbuild._native import build_downstream_deps, project_lineage_views +from sqlbuild._native import ( + build_downstream_deps, + parse_project_selector, + project_lineage_views, + resolve_project_selectors, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.selectors import ( - parse_selector, - resolve_selectors, -) from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.planner.types import SelectorKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -135,7 +136,7 @@ def test_given_raw_selector_when_parsing_then_returns_expected_result( test_case: ParseSelectorTestCase, ) -> None: - result: ParsedSelector | PathSelector = parse_selector(test_case.raw) + result: ParsedSelector | PathSelector = parse_project_selector(test_case.raw) assert result == test_case.expected_result @@ -180,7 +181,7 @@ def test_given_invalid_selector_when_parsing_then_raises( test_case: ParseSelectorErrorTestCase, ) -> None: with pytest.raises(test_case.expected_error_type): - parse_selector(test_case.raw) + parse_project_selector(test_case.raw) @pytest.mark.parametrize( @@ -331,12 +332,14 @@ def test_given_selectors_when_resolving_then_returns_expected_names( upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[1] downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[2] - result: frozenset[CompiledObjectKey] = resolve_selectors( + result: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, + path_index={}, ) result_names: frozenset[str] = frozenset(key.name for key in result) @@ -389,12 +392,14 @@ def test_given_model_table_function_dependency_when_selecting_then_graph_expands key.name: key for key in (base_key, function_key, consumer_key) } - selected: frozenset[CompiledObjectKey] = resolve_selectors( + selected: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, + path_index={}, ) assert frozenset(key.name for key in selected) == test_case.expected_names @@ -431,12 +436,14 @@ def test_given_unknown_name_when_resolving_then_raises( downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[2] with pytest.raises(test_case.expected_error_type): - resolve_selectors( + resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, + path_index={}, ) @@ -483,13 +490,14 @@ def test_given_tag_selectors_when_resolving_then_returns_expected_names( upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[1] downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = diamond_graph[2] - result: frozenset[CompiledObjectKey] = resolve_selectors( + result: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, tag_index=diamond_tag_index, + path_index={}, ) result_names: frozenset[str] = frozenset(key.name for key in result) @@ -603,12 +611,13 @@ def test_given_path_selectors_when_resolving_then_returns_expected_names( downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = path_graph[2] path_idx: dict[CompiledObjectKey, str] = path_graph[3] - result: frozenset[CompiledObjectKey] = resolve_selectors( + result: frozenset[CompiledObjectKey] = resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, path_index=path_idx, ) result_names: frozenset[str] = frozenset(key.name for key in result) @@ -658,12 +667,13 @@ def test_given_invalid_path_selector_when_resolving_then_raises_with_message( path_idx: dict[CompiledObjectKey, str] = path_graph[3] with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): - resolve_selectors( + resolve_project_selectors( select=test_case.select, exclude=test_case.exclude, all_keys=all_keys, - upstream=upstream, - downstream=downstream, + upstream_deps=upstream, + downstream_deps=downstream, + tag_index={}, path_index=path_idx, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py index 52b4ca3ca9..54984ac676 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion +from sqlbuild._native import split_selector_expansion from sqlbuild.compiler.planner.models import SelectorExpansion from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( SelectorExpansionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py index ae9d266cf7..e46555e333 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_sqlbuild_model_selectors.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( - resolve_sqlbuild_model_selector_names, -) +from sqlbuild._native import resolve_sqlbuild_model_selector_names from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( SqlbuildModelSelectorNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index 2bd507852b..86b9b54830 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import resolve_python_node_selection -from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector +from sqlbuild._native import parse_project_selector, resolve_python_node_selection from sqlbuild.compiler.planner.models import ParsedSelector from sqlbuild.compiler.planner.types import SelectorKind from sqlbuild.compiler.python_nodes.models import PythonNodeGraph diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 888a3350c6..47297041ae 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -4,9 +4,9 @@ import pytest +from sqlbuild._native import resolve_project_selectors from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection from sqlbuild.refs import seed from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( From 91a04ce03daef553c48af80c202895c90bb0cc34 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:46:30 +0100 Subject: [PATCH 16/32] fix: call native selection from the native rules command --- .../src/bindings/_helpers/plan_command/rules.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index 3b48776801..bc90ce8511 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -11,9 +11,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; -const REJECT_UNIT_TESTS_MODULE: &str = - "sqlbuild.compiler.pipeline.main._reject_unit_test_selectors"; -const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main._project_graph_selection"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const LIST_ACTION: &str = "list"; const SHOW_ACTION: &str = "show"; const SKILLS_ACTION: &str = "skills"; From 7cabcf127d13a3d83a449c18283491258d93af35 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:55:23 +0100 Subject: [PATCH 17/32] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 82e53cda56..8629e933a4 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 45878 +remaining_lines = 45121 [[stays]] path = "executor" From 458f28d5ff66c832b2223fc131c6d1bb7f021929 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:17:26 +0100 Subject: [PATCH 18/32] refactor: run CLI parsing, cursor overrides, colour capability and progress setup natively --- .../_helpers/build_command/execution.rs | 12 ++-- .../_helpers/build_command/invocation.rs | 25 ++++---- .../_helpers/build_command/planning.rs | 11 ++-- .../src/bindings/_helpers/cli/argv.rs | 58 ++++++++++++++---- .../src/bindings/_helpers/cli/entry.rs | 20 ++----- .../bindings/_helpers/cli/progress_text.rs | 22 ++++++- .../bindings/_helpers/clone/clone_command.rs | 8 +-- .../_helpers/compile_driver/command.rs | 8 +-- .../_helpers/contract_command/command.rs | 9 +-- .../bindings/_helpers/cost_command/command.rs | 6 +- .../src/bindings/_helpers/cursor/mod.rs | 1 + .../src/bindings/_helpers/cursor/overrides.rs | 59 +++++++++++++++++++ .../bindings/_helpers/dbt_command/support.rs | 7 +-- .../_helpers/debug_command/run_debug.rs | 7 +-- .../_helpers/diff_report/registration.rs | 7 +-- .../_helpers/format_command/command.rs | 7 +-- .../_helpers/freshness/freshness_command.rs | 8 +-- .../bindings/_helpers/init_command/command.rs | 6 +- .../_helpers/janitor/janitor_command.rs | 8 +-- .../_helpers/lineage/lineage_command.rs | 6 +- .../bindings/_helpers/load/load_command.rs | 23 +++----- .../bindings/_helpers/plan_command/plan.rs | 22 +++---- .../_helpers/presentation_text/mod.rs | 1 + .../presentation_text/registration.rs | 5 +- .../_helpers/presentation_text/terminal.rs | 28 +++++++++ .../_helpers/refactor/refactor_command.rs | 8 +-- .../_helpers/scenario_command/setup.rs | 8 +-- .../bindings/_helpers/scopes/scope_command.rs | 7 +-- .../_helpers/sql_test_command/test_command.rs | 3 +- .../_helpers/workspace_commands/commands.rs | 6 +- scripts/fensu_policy/constants.py | 1 - .../rules/canonical_operations.py | 4 +- src/sqlbuild/_native.pyi | 8 +-- .../commands/_helpers/entry/observability.py | 2 +- .../cli/commands/_helpers/entry/parsing.py | 33 ----------- .../cli/commands/_helpers/input/__init__.py | 0 .../cli/commands/_helpers/input/parsing.py | 32 ---------- .../_helpers/runtime/command_invocation.py | 27 +++++---- .../entrypoint/_dispatch_with_compute_logs.py | 2 +- .../cli/commands/main/execution/__init__.py | 0 .../main/execution/connection_progress.py | 24 -------- src/sqlbuild/cli/commands/models.py | 10 ++++ .../compile_reuse/_helpers/native_reuse.py | 3 +- src/sqlbuild/cli/progress/__init__.py | 1 - src/sqlbuild/cli/progress/main/__init__.py | 1 - .../main/_build_command_progress_reporters.py | 29 --------- .../main/verbose_inspection_output.py | 22 ------- src/sqlbuild/cli/progress/models.py | 25 -------- .../dbt/_helpers/pipeline/execution_phases.py | 5 +- .../integrations/dbt/main/pipeline/plan.py | 6 +- .../_helpers/terminal_capabilities.py | 10 ---- .../presentation/main/supports_color.py | 11 ---- .../rules/test_canonical_operations.py | 8 +-- .../commands/_helpers/entry/test_parsing.py | 2 +- .../_helpers/runtime/test_warehouses.py | 2 +- .../cli/commands/main/dbt/test_init.py | 6 +- .../cli/commands/main/entry/test_main.py | 7 ++- .../cli/commands/main/execution/helpers.py | 4 +- .../cli/commands/main/init/test_run.py | 6 +- .../main/playground/test_playground.py | 4 +- .../cli/commands/main/scope/test_scope.py | 4 +- .../commands/shared/_helpers/test_progress.py | 2 +- .../verbose_inspection_output/_test_types.py | 3 +- .../test_verbose_inspection_output.py | 16 ++--- .../unit/src/sqlbuild/presentation/helpers.py | 40 +++++++++++++ 65 files changed, 351 insertions(+), 415 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/entry/parsing.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/input/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/input/parsing.py delete mode 100644 src/sqlbuild/cli/commands/main/execution/__init__.py delete mode 100644 src/sqlbuild/cli/commands/main/execution/connection_progress.py delete mode 100644 src/sqlbuild/cli/progress/__init__.py delete mode 100644 src/sqlbuild/cli/progress/main/__init__.py delete mode 100644 src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py delete mode 100644 src/sqlbuild/cli/progress/main/verbose_inspection_output.py delete mode 100644 src/sqlbuild/cli/progress/models.py delete mode 100644 src/sqlbuild/presentation/main/supports_color.py create mode 100644 tests/unit/src/sqlbuild/presentation/helpers.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 61bbfd3bce..eaad8e76b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -15,9 +15,9 @@ use crate::bindings::_helpers::build_command::support::{ boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; @@ -186,13 +186,11 @@ fn cursors<'py>( if !overrides.is_truthy()? { overrides = imported(py, PLANNER_MODELS, "CursorOverrides")?.call0()?; } - let timestamp: Bound<'py, PyAny> = imported(py, PARSING_MODULE, "parse_cursor_timestamp")?; - let integer: Bound<'py, PyAny> = imported(py, PARSING_MODULE, "parse_cursor_integer")?; Ok([ - timestamp.call1((overrides.getattr("start_ts")?,))?, - timestamp.call1((overrides.getattr("end_ts")?,))?, - integer.call1((overrides.getattr("start_int")?,))?, - integer.call1((overrides.getattr("end_int")?,))?, + cursor_timestamp(&overrides.getattr("start_ts")?)?, + cursor_timestamp(&overrides.getattr("end_ts")?)?, + cursor_integer(&overrides.getattr("start_int")?)?, + cursor_integer(&overrides.getattr("end_int")?)?, ]) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 3d109f203a..50c7e75909 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -10,13 +10,12 @@ use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const REPORTERS_MODULE: &str = "sqlbuild.cli.progress.main._build_command_progress_reporters"; const SETTINGS_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; @@ -168,24 +167,28 @@ pub(crate) fn resolve_build_context<'py>( )?; let machine_output: bool = request.getattr("json_output")?.is_truthy()?; let debug: bool = request.getattr("debug")?.is_truthy()?; - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && !machine_output - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !machine_output && stdout_supports_color(py)?; let stream: Bound<'py, PyAny> = py.import("sys")?.getattr(if debug || machine_output { "stderr" } else { "stdout" })?; - let reporters: Bound<'py, PyAny> = call_with( - &imported(py, REPORTERS_MODULE, "build_command_progress_reporters")?, + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, &[ ("adapter_name", text(py, &adapter_name)), ("stream", stream.clone()), ("use_color", boolean(py, use_color)), ], )?; + let planning_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", boolean(py, use_color)), + ], + )?; let load_sources: Bound<'py, PyAny> = request.getattr("load_sources")?; let should_load_sources: bool = request.getattr("reload_sources")?.is_truthy()? || if load_sources.is_none() { @@ -209,8 +212,8 @@ pub(crate) fn resolve_build_context<'py>( ("connection_config", connection_config), ("use_color", boolean(py, use_color)), ("progress_stream", stream.clone()), - ("connection_progress", reporters.getattr("connection")?), - ("planning_progress", reporters.getattr("planning")?), + ("connection_progress", connection_progress), + ("planning_progress", planning_progress), ("should_load_sources", boolean(py, should_load_sources)), ("effective_target_name", target_name), ("execution_limits", limits), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 02bcdfc731..e486e528b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -8,13 +8,13 @@ use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, keyword_relay, none, }; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; const DEFER_CLONE_MODULE: &str = "sqlbuild.cli.commands._helpers.build_planning.defer_clone"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const VERBOSE_INSPECTION_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; @@ -119,11 +119,8 @@ pub(crate) fn compile_build_plan<'py>(context: &BuildContext<'py>) -> PyResult = call_with( - &imported(py, VERBOSE_INSPECTION_MODULE, "verbose_inspection_output")?, - &[("enabled", context.request.getattr("verbose")?)], - )?; - within(&manager, |_| { + let verbose: bool = context.request.getattr("verbose")?.is_truthy()?; + with_verbose_inspection(py, verbose, || { call_with( &imported(py, NATIVE_MODULE, "run_compile_pipeline")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs index 901810863c..3a9e4d51d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs @@ -5,14 +5,17 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{ PyBool, PyBoolMethods, PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, }; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::argv::main::parse_cli_arguments::parse_cli_arguments as parse_arguments; use sqlbuild_cli::argv::models::{ArgvOutcome, ArgvSettings, Stream, TypeFailure, Value}; use sqlbuild_cli::argv::types::ArgvHost; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::bindings::_helpers::cli::surrogates::{python_text, rust_text, uses_placeholders}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NAMESPACE_MODULE: &str = "sqlbuild.cli.commands.classes.cli_namespace"; +const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const INT_CONVERTER: &str = "int"; const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; const VARS_CONVERTER: &str = "parse_cli_vars"; @@ -171,20 +174,17 @@ fn object_of(py: Python<'_>, host: &PythonHost<'_>, value: Value) -> PyResult>, + argv: &[Bound<'_, PyString>], defaults: &Bound<'_, PyList>, - use_color: bool, - columns: i64, + settings: ArgvSettings, ) -> PyResult { let py: Python<'_> = defaults.py(); let mut arguments: Vec = Vec::with_capacity(argv.len()); let mut plain_placeholders: bool = false; let mut surrogates: bool = false; - for arg in &argv { + for arg in argv { let (text, mapped) = rust_text(arg)?; surrogates |= mapped; plain_placeholders |= !mapped && uses_placeholders(&text); @@ -205,7 +205,6 @@ fn parse_cli_arguments( let (name, object): (String, Bound<'_, PyAny>) = item.extract()?; values.push((name, host.value_of(&object)?)); } - let settings: ArgvSettings = ArgvSettings { columns, use_color }; let outcome: ArgvOutcome = parse_arguments(&arguments, values, settings, &mut host); let sys: Bound<'_, PyModule> = py.import("sys")?; for (stream, text) in outcome.writes { @@ -236,7 +235,46 @@ fn parse_cli_arguments( } } +/// Parse CLI arguments into a `CliNamespace`; help, version and errors print and set the exit code. +#[pyfunction] +#[pyo3(signature = (argv, *, use_color))] +pub(crate) fn parse_cli_invocation<'py>( + py: Python<'py>, + argv: Vec>, + use_color: bool, +) -> PyResult> { + let args: Bound<'py, PyAny> = imported(py, NAMESPACE_MODULE, "CliNamespace")?.call0()?; + let defaults: Bound<'py, PyList> = py + .get_type::() + .call1((args.getattr("__dict__")?.call_method0("items")?,))? + .downcast_into::()?; + let columns: i64 = py + .import("shutil")? + .call_method0("get_terminal_size")? + .getattr("columns")? + .extract()?; + let settings: ArgvSettings = ArgvSettings { columns, use_color }; + let (rows, exit_code): ParsedRows = parse_cli_arguments(&argv, &defaults, settings)?; + let parsed: Bound<'py, PyAny> = imported(py, ENTRY_MODELS_MODULE, "ParsedCliInvocation")?; + let Some(rows) = rows else { + return call_with( + &parsed, + &[ + ("args", py.None().into_bound(py)), + ("exit_code", exit_code.into_bound_py_any(py)?), + ], + ); + }; + for (name, value) in rows { + args.setattr(name.as_str(), value)?; + } + call_with( + &parsed, + &[("args", args), ("exit_code", py.None().into_bound(py))], + ) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_cli_arguments, module)?)?; + module.add_function(wrap_pyfunction!(parse_cli_invocation, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index eec3e3caea..905fc37cee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -12,14 +12,15 @@ use sqlbuild_cli::dispatch::main::failure_report::failure_report; use sqlbuild_cli::dispatch::main::invocation_policy::invocation_policy; use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPolicy}; +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; use sqlbuild_core::cli_text::models::CliStyle; const NO_COLOR_OPTION: &str = "--no-color"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.entry.parsing"; const COMPUTE_LOGS_MODULE: &str = "sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"; const OBSERVABILITY_MODULE: &str = "sqlbuild.cli.commands._helpers.entry.observability"; @@ -30,7 +31,6 @@ const EXTERNAL_SCOPE_MODULE: &str = const INVOCATION_CONTEXT_MODULE: &str = "sqlbuild.runtime.output_capture.main.invocation_context"; const OUTPUT_CAPTURE_ERRORS_MODULE: &str = "sqlbuild.runtime.output_capture.exceptions"; const DIAGNOSTICS_MODULE: &str = "sqlbuild.diagnostics.main.diagnostics_context"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const TRANSIENT_LINE_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SKILLS_MODULE: &str = "sqlbuild._native"; const RULES_ERRORS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; @@ -518,20 +518,8 @@ fn parsed_invocation<'py>( identity: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = argv.py(); - let use_color: bool = !argv.contains(NO_COLOR_OPTION)? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; - let parsed: Bound<'py, PyAny> = call_with( - &imported(py, PARSING_MODULE, "parse_cli_invocation")?, - &[ - ("argv", argv.clone()), - ( - "use_color", - PyBool::new(py, use_color).to_owned().into_any(), - ), - ], - )?; + let use_color: bool = !argv.contains(NO_COLOR_OPTION)? && stdout_supports_color(py)?; + let parsed: Bound<'py, PyAny> = parse_cli_invocation(py, argv.extract()?, use_color)?; let args: Bound<'py, PyAny> = parsed.getattr("args")?; if args.is_none() { let exit_code: Bound<'py, PyAny> = parsed.getattr("exit_code")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs index 1df6d8f56d..6fb23a1220 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs @@ -16,7 +16,9 @@ use sqlbuild_core::progress::models::{ExpectationStep, InspectionRecord}; use sqlbuild_core::progress::types::DifferenceSample; use crate::bindings::_helpers::cli::progress::write_persistent; -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; + +const INSPECTION_SINK_MODULE: &str = "sqlbuild.adapter.relations.main.inspection_query_sink"; const HEADER_OPTIONS: (&[&str], &[&str]) = ( &["stream", "command", "target", "concurrency", "use_color"], @@ -161,6 +163,24 @@ impl VerboseInspectionWriter { } } +/// Run `body`, writing each inspection read and then a total to stderr while `enabled`. +pub(crate) fn with_verbose_inspection<'py, T>( + py: Python<'py>, + enabled: bool, + body: impl FnOnce() -> PyResult, +) -> PyResult { + if !enabled { + return body(); + } + let writer: Bound<'py, VerboseInspectionWriter> = + Bound::new(py, VerboseInspectionWriter::new())?; + let sink: Bound<'py, PyAny> = imported(py, INSPECTION_SINK_MODULE, "inspection_query_sink")? + .call1((writer.getattr("write")?,))?; + let value: T = within(&sink, |_| body())?; + writer.get().write_total(py)?; + Ok(value) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(format_execution_header, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 46610db665..20fdc524b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -24,10 +24,10 @@ use crate::bindings::_helpers::clone::clone_execution::{ use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -127,10 +127,8 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let use_color: bool = !request.getattr("no_color")?.extract::()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let style: CliStyle = CliStyle { use_color }; let stream: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; let progress: Bound<'py, PyAny> = progress_reporter(&stream, style)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 17fca3ab98..30f86b80b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compile_driver::phases::{ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; @@ -28,7 +29,6 @@ const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; -const SUPPORTS_COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; /// What one full compile reports for reuse recording. struct CommandOutcome<'py> { @@ -448,10 +448,8 @@ fn record_output<'py>( &format_compile_json(py, Some(keywords(py, &arguments)?))?, ) } else { - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && imported(py, SUPPORTS_COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(py)?; arguments.push(( "use_color", PyBool::new(py, use_color).to_owned().into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 052a1417c3..eb7a6439b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -20,6 +20,7 @@ use sqlbuild_core::contract_command::models::{ use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; @@ -27,7 +28,6 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; const GENERATE_ACTION: &str = "generate"; @@ -348,11 +348,8 @@ fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && !json_output - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !json_output && stdout_supports_color(py)?; Ok(Progress { stream, use_color }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs index 5c8a4f55a3..64d9d392b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs @@ -17,9 +17,9 @@ use crate::bindings::_helpers::cost_command::bounds::{parse_bound, parse_since}; use crate::bindings::_helpers::cost_command::refresh::refresh_pending_cost_run; use crate::bindings::_helpers::cost_command::refusal::cost_error; use crate::bindings::_helpers::cost_command::sorting::{sort_history, sort_resources}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; /// The request fields every cost view reads. struct CostRequest<'py> { @@ -67,9 +67,7 @@ impl<'py> CostRequest<'py> { if self.request.getattr("no_color")?.is_truthy()? { return Ok(false); } - imported(self.request.py(), COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy() + stdout_supports_color(self.request.py()) } /// Write the JSON file when asked, then the JSON or the rendered text to stdout. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs index dec9f2ffdf..a1ea62d802 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod conversion; pub(crate) mod durations; pub(crate) mod intervals; +pub(crate) mod overrides; pub(crate) mod registration; pub(crate) mod scalars; pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs new file mode 100644 index 0000000000..268983ed83 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/overrides.rs @@ -0,0 +1,59 @@ +//! Executor cursor bounds from `--start-cursor`/`--end-cursor` overrides. + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyInt, PyString, PyStringMethods}; +use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cursor::conversion::{MODELS_MODULE, algebra_error}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; + +const DECIMAL_MODULE: &str = "decimal"; +const INTEGER_AS_TIMESTAMP: &str = "an integer cursor cannot be used as a timestamp"; +const TIMESTAMP_AS_INTEGER: &str = "a timestamp cursor cannot be used as an integer"; + +fn instance_of(value: &Bound<'_, PyAny>, class: &str) -> PyResult { + value.is_instance(&imported(value.py(), MODELS_MODULE, class)?) +} + +/// A timestamp override as a `datetime`; dates start at midnight and text is ISO 8601. +pub(crate) fn cursor_timestamp<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, "TimestampValue")? { + return value.getattr("value"); + } + let datetime: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?; + if instance_of(value, "DateValue")? { + let midnight: Bound<'py, PyAny> = datetime.getattr("min")?.call_method0("time")?; + return datetime.call_method1("combine", (value.getattr("value")?, midnight)); + } + if instance_of(value, "IntegerValue")? { + return Err(algebra_error(py, INTEGER_AS_TIMESTAMP)); + } + datetime.call_method1("fromisoformat", (value,)) +} + +/// An integer override as an `int`; text is read the way `int(Decimal(text))` reads it. +pub(crate) fn cursor_integer<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if value.is_none() { + return Ok(value.clone()); + } + if instance_of(value, "IntegerValue")? { + return value.getattr("value"); + } + if instance_of(value, "DateValue")? || instance_of(value, "TimestampValue")? { + return Err(algebra_error(py, TIMESTAMP_AS_INTEGER)); + } + if let Ok(text) = value.downcast::() + && let Some(parsed) = parse_integer_text(text.to_str()?) + { + return Ok(parsed.into_pyobject(py)?.into_any()); + } + let decimal: Bound<'py, PyAny> = imported(py, DECIMAL_MODULE, "Decimal")?.call1((value,))?; + py.get_type::().call1((decimal,)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs index ea7f60a7c9..f92b519062 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs @@ -5,8 +5,7 @@ use pyo3::types::PyBool; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_driver::python_calls::imported; - -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; /// `sys.stdout` or `sys.stderr`, looked up when written. pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult> { @@ -15,9 +14,7 @@ pub(crate) fn stream<'py>(py: Python<'py>, name: &str) -> PyResult) -> PyResult { - imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy() + stdout_supports_color(py) } pub(crate) fn flag(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 069ca675fc..f1c48bf5dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -31,13 +31,13 @@ use crate::bindings::_helpers::command_runtime::warehouses::{ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( &[], &[ @@ -395,10 +395,7 @@ fn run_debug_command(py: Python<'_>, options: Option>) -> PyRe let text: String = if arguments.flag("json_output")? { format_debug_json(&report) + "\n" } else { - let use_color: bool = !arguments.flag("no_color")? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; format_debug_text(&report, CliStyle { use_color }) }; let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs index 644245c366..78fa537525 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/registration.rs @@ -18,10 +18,10 @@ use crate::bindings::_helpers::diff_report::command::{ }; use crate::bindings::_helpers::diff_report::facts::{diff_facts, json_of}; use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_cli::diff_report::types::PhaseSeconds; const DEFAULT_MAX_VALUE_LENGTH: usize = 160; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const TEXT_OPTIONS: (&[&str], &[&str]) = ( &[ "result", @@ -195,10 +195,7 @@ pub(crate) fn publish_diff_output( if request.getattr("json_output")?.is_truthy()? { return write_stdout(py, &json_text(models, &document)?); } - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(py)?; write_stdout( py, &render_diff_text( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index c32f91ad2d..d72d384f86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -25,9 +25,9 @@ use crate::bindings::_helpers::format_command::report::lint_report; use crate::bindings::_helpers::format_command::run::{FormatOutcome, FormatRequest, run_format}; use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_lint_inputs}; use crate::bindings::_helpers::frontier::encoder::emit_error; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; const START_MESSAGE: &str = "Formatting SQL START"; @@ -285,10 +285,7 @@ fn run_format_command<'py>(py: Python<'py>, options: Option>) } else { raw_project_dir }; - let use_color: bool = !arguments.flag("no_color")? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = !arguments.flag("no_color")? && stdout_supports_color(py)?; let flags: FormatFlags = FormatFlags { check: arguments.flag("check")?, diff: arguments.flag("diff")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index c344aa690c..28a40c7088 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -17,10 +17,10 @@ use sqlbuild_core::freshness_text::models::FreshnessSourceResult; use crate::bindings::_helpers::freshness::freshness_observe::{ Observer, PreviousState, observe_freshness, }; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; @@ -368,10 +368,8 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult if !json_output_path.is_none() { write_json_output(py, &payload, (false, &json_output_path))?; } - let use_color: bool = !request.getattr("no_color")?.extract::()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; write(py, &render_freshness_text(&results, CliStyle { use_color }))?; } Ok(freshness_exit_code( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs index 464164b09f..dad2d061af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -15,10 +15,10 @@ use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::init_command::scaffold::scaffold_blank_project; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const EXISTS_MESSAGE: &str = "sqlbuild_project.toml already exists in this directory"; fn exists_error(py: Python<'_>) -> PyErr { @@ -56,9 +56,7 @@ fn run_init_command(py: Python<'_>, project_dir: Option>) -> Py &[("project_dir", base_dir)], )?; let style: CliStyle = CliStyle { - use_color: imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?, + use_color: stdout_supports_color(py)?, }; py.import("sys")?.getattr("stdout")?.call_method1( "write", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index c41bf73def..578e2994a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -24,11 +24,11 @@ use crate::bindings::_helpers::janitor::janitor_phases::{ compile_janitor_project, connect_janitor_warehouse, execute_janitor_cleanup, }; use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janitor_result_counts}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; @@ -104,10 +104,8 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let options: Bound<'py, PyDict> = keywords( py, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 6f26ec444a..a7d518e291 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -36,10 +36,10 @@ use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views, lineage_keys}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; @@ -686,9 +686,7 @@ fn run_lineage_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult()? && !machine_output - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + && stdout_supports_color(py)?; let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if machine_output { "stderr" } else { "stdout" })?; @@ -356,13 +354,6 @@ fn prepare<'py>( }) } -fn cursor<'py>( - overrides: &Bound<'py, PyAny>, - (field, parser): (&str, &str), -) -> PyResult> { - imported(overrides.py(), CURSOR_PARSING_MODULE, parser)?.call1((overrides.getattr(field)?,)) -} - /// The executor's runtime parameters for this load. fn runtime_params<'py>( request: &Bound<'py, PyAny>, @@ -387,19 +378,19 @@ fn runtime_params<'py>( ("is_reload", request.getattr("reload")?), ( "start_cursor_ts", - cursor(overrides, ("start_ts", "parse_cursor_timestamp"))?, + cursor_timestamp(&overrides.getattr("start_ts")?)?, ), ( "end_cursor_ts", - cursor(overrides, ("end_ts", "parse_cursor_timestamp"))?, + cursor_timestamp(&overrides.getattr("end_ts")?)?, ), ( "start_cursor_int", - cursor(overrides, ("start_int", "parse_cursor_integer"))?, + cursor_integer(&overrides.getattr("start_int")?)?, ), ( "end_cursor_int", - cursor(overrides, ("end_int", "parse_cursor_integer"))?, + cursor_integer(&overrides.getattr("end_int")?)?, ), ( "use_color", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 6025a057f7..d49a517152 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -7,13 +7,13 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::build_command::support::keyword_relay; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const VERBOSE_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; @@ -90,11 +90,8 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult = py.import("sys")? .getattr(if json_output { "stderr" } else { "stdout" })?; @@ -285,12 +282,9 @@ fn run_plan_command(request: &Bound<'_, PyAny>) -> PyResult { invocation.stream.call_method1("write", ("\n",))?; invocation.stream.call_method0("flush")?; } - let verbose: Bound<'_, PyDict> = PyDict::new(py); - verbose.set_item("enabled", request.getattr("verbose")?)?; - let pipeline: Bound<'_, PyAny> = within( - &imported(py, VERBOSE_MODULE, "verbose_inspection_output")?.call((), Some(&verbose))?, - |_| compile_pipeline(request, &invocation), - )?; + let verbose: bool = request.getattr("verbose")?.is_truthy()?; + let pipeline: Bound<'_, PyAny> = + with_verbose_inspection(py, verbose, || compile_pipeline(request, &invocation))?; write_output(request, &invocation, &pipeline)?; Ok(0) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs index 519415c3d5..42554e58ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod document; pub(crate) mod functions; pub(crate) mod registration; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs index 999bafd567..fd3b43e9ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/registration.rs @@ -2,9 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::presentation_text::{document, functions}; +use crate::bindings::_helpers::presentation_text::{document, functions, terminal}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { document::register(module)?; - functions::register(module) + functions::register(module)?; + terminal::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs new file mode 100644 index 0000000000..369db30e07 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs @@ -0,0 +1,28 @@ +//! Terminal colour capability of the active output stream. + +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +const NO_COLOR_VARIABLE: &str = "NO_COLOR"; + +/// Whether `sys.stdout` is a terminal and `NO_COLOR` is unset. +pub(crate) fn stdout_supports_color(py: Python<'_>) -> PyResult { + if std::env::var_os(NO_COLOR_VARIABLE).is_some() { + return Ok(false); + } + let stdout = py.import("sys")?.getattr("stdout")?; + if !stdout.hasattr("isatty")? { + return Ok(false); + } + stdout.call_method0("isatty")?.is_truthy() +} + +/// Return whether the active output terminal supports color. +#[pyfunction(name = "supports_color")] +fn color_support(py: Python<'_>) -> PyResult { + stdout_supports_color(py) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(color_support, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 09dc56013d..9a3490c70d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -35,13 +35,13 @@ use sqlbuild_refactor::refactoring::types::Originals; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::lineage::lineage_index::lineage_key; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, }; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const COMPILE_MODULE: &str = "sqlbuild.cli.commands._helpers.refactor.compile_facts"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; @@ -417,10 +417,8 @@ fn run_refactor_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; let json_output: bool = request.getattr("json_output")?.extract()?; - let use_color: bool = !request.getattr("no_color")?.extract::()? - && imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .extract::()?; + let use_color: bool = + !request.getattr("no_color")?.extract::()? && stdout_supports_color(py)?; let command: Command<'_> = Command { request: request.clone(), project_dir: project_dir.call_method0("resolve")?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 821c8ef319..18831a2e29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -22,6 +22,7 @@ use sqlbuild_core::scenario_command::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; @@ -32,7 +33,6 @@ const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; const METADATA_MODULE: &str = "importlib.metadata"; const DISTRIBUTION_NAME: &str = "sqlbuild"; @@ -242,10 +242,8 @@ pub(crate) fn project_connection_config<'py>( /// `not no_color and supports_color()`. pub(crate) fn style(request: &Bound<'_, PyAny>) -> PyResult { - let use_color: bool = !request.getattr("no_color")?.is_truthy()? - && imported(request.py(), COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(request.py())?; Ok(CliStyle { use_color }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index 2087649059..a9a4e1b0a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -13,6 +13,7 @@ use sqlbuild_scopes::scope_text::main::render_scope_text::render_scope_text; use sqlbuild_scopes::scope_text::main::scope_result_failed::scope_result_failed; use sqlbuild_scopes::scope_text::models::ScopeTextRequest; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; @@ -25,7 +26,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ enums, constants, tests and audits with their prefix, for example macro:"; @@ -329,10 +329,7 @@ fn use_color( if output_stream.is_some() || request.getattr("no_color")?.extract::()? { return Ok(false); } - py.import(COLOR_MODULE)? - .getattr("supports_color")? - .call0()? - .extract() + stdout_supports_color(py) } /// Load offline compiler facts, run one pure query, and present its result. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index c7b51d761b..74c0708ad9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -38,7 +38,6 @@ const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; -const PROGRESS_MODELS: &str = "sqlbuild.cli.progress.models"; const EXECUTOR_PIPELINE_MODELS: &str = "sqlbuild.executor.pipeline.models"; const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; const RESOURCE_ID_MODULE: &str = "sqlbuild.executor.testing.main._resource_id"; @@ -154,7 +153,7 @@ fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { /// Nested `expect` rows of a completed test, one per step. fn expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = result.py(); - let row_class: Bound<'py, PyAny> = imported(py, PROGRESS_MODELS, "NestedProgressChildRow")?; + let row_class: Bound<'py, PyAny> = imported(py, COMMAND_MODELS, "NestedProgressChildRow")?; let name: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_name")?; let detail: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_detail")?; let mut rows: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index 28fd26df81..dfe56f585b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -20,12 +20,12 @@ use sqlbuild_config::workspace::models::{PlaygroundRequest, SkillInstallRequest} use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; const TEMPLATES_DIRECTORY: &str = "playground_templates"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const OUTPUT_MODELS: &str = "sqlbuild.cli.output.models"; -const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const SKILLS_OPTIONS: (&[&str], &[&str]) = (&["project_dir"], &["global_install", "targets", "force"]); const UPDATE_OPTIONS: (&[&str], &[&str]) = ( @@ -198,9 +198,7 @@ fn run_playground_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResul py, &skill_request(py, playground_project_dir(&target_dir, &template))?, )?; - let use_color: bool = imported(py, COLOR_MODULE, "supports_color")? - .call0()? - .is_truthy()?; + let use_color: bool = stdout_supports_color(py)?; print( py, &playground_summary(&display_path, &template, CliStyle { use_color }), diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index e2bc094ac3..e528e91df7 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -189,7 +189,6 @@ PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) PUBLIC_COLOR_ENTRY_PARTS: frozenset[tuple[str, ...]] = frozenset( { - ("src", "sqlbuild", "presentation", "main", "supports_color.py"), ("src", "sqlbuild", "presentation", "main", "terminal_columns.py"), } ) diff --git a/scripts/fensu_policy/rules/canonical_operations.py b/scripts/fensu_policy/rules/canonical_operations.py index b2e10b9933..8968f85d6d 100644 --- a/scripts/fensu_policy/rules/canonical_operations.py +++ b/scripts/fensu_policy/rules/canonical_operations.py @@ -30,8 +30,8 @@ code="XSB041", family=Family.CUSTOM, slug="color-capability-entry", - message="color capability imports must use the presentation main entry", - remediation="Import supports_color from sqlbuild.presentation.main.supports_color.", + message="terminal capability imports must use the presentation main entry", + remediation="Import terminal_columns from sqlbuild.presentation.main.terminal_columns.", ) def color_capability_entry(*, module: object, ctx: RuleContext) -> list[Fault]: del module diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9cb0c22209..a9a84415a4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -36,6 +36,7 @@ from sqlbuild.cli.commands.models import ( JanitorCommandRequest, LineageCommandRequest, LoadCommandRequest, + NestedProgressChildRow, PlanCommandRequest, PlanSafetyGates, PlaygroundCommandRequest, @@ -55,10 +56,10 @@ from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, CompileCommandRequest, CompileProfileFlags, + ParsedCliInvocation, ResolvedWarehouse, ) from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget -from sqlbuild.cli.progress.models import NestedProgressChildRow from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, MeasurementContract, @@ -2937,9 +2938,8 @@ class SqlReferenceScanner: def scope_expected_model_names( texts: list[tuple[str, str]], scenario: bool, syntax: dict[str, object] ) -> list[tuple[str | None, list[str]]]: ... -def parse_cli_arguments( - argv: list[str], defaults: list[tuple[str, object]], *, use_color: bool, columns: int -) -> tuple[list[tuple[str, object]] | None, int | None]: ... +def supports_color() -> bool: ... +def parse_cli_invocation(argv: Sequence[str], *, use_color: bool) -> ParsedCliInvocation: ... def run_cli(argv: Sequence[str] | None = None, handlers: object | None = None) -> int: ... def extract_sql_test_ctes( *, sql: str, file_label: str, syntax: SqlLexicalSyntax diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py index ccfc2f01f9..68d301eeb9 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/observability.py @@ -14,6 +14,7 @@ NativeProgressProjector, TerminalEventIndex, machine_output_mode, + supports_color, terminal_event_index_scope, ) from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace @@ -33,7 +34,6 @@ DiscoveredRuntimeExtensions, ) from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher from sqlbuild.runtime.event_exporting.constants import ( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py deleted file mode 100644 index a3bbfef78b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ /dev/null @@ -1,33 +0,0 @@ -"""CLI argument parsing through the native argv parser, with post-parse normalization.""" - -from __future__ import annotations - -import shutil -import sys -from collections.abc import Sequence - -import sqlbuild._native as _native -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace -from sqlbuild.cli.entry.models import ParsedCliInvocation - - -def parse_cli_invocation(*, argv: Sequence[str] | None, use_color: bool) -> ParsedCliInvocation: - """Parse CLI arguments into a namespace; help, version and errors print and exit.""" - - args: CliNamespace = CliNamespace() - raw_args: list[str] = list(sys.argv[1:] if argv is None else argv) - rows: list[tuple[str, object]] | None - exit_code: int | None - rows, exit_code = _native.parse_cli_arguments( - raw_args, - list(vars(args).items()), - use_color=use_color, - columns=shutil.get_terminal_size().columns, - ) - if rows is None: - return ParsedCliInvocation(args=None, exit_code=exit_code) - name: str - value: object - for name, value in rows: - setattr(args, name, value) - return ParsedCliInvocation(args=args, exit_code=None) diff --git a/src/sqlbuild/cli/commands/_helpers/input/__init__.py b/src/sqlbuild/cli/commands/_helpers/input/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/input/parsing.py b/src/sqlbuild/cli/commands/_helpers/input/parsing.py deleted file mode 100644 index f248468e33..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/input/parsing.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Neutral command input conversion.""" - -from __future__ import annotations - -from datetime import datetime -from decimal import Decimal - -from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import CursorScalar - - -def parse_cursor_timestamp(value: str | CursorScalar | None) -> datetime | None: - if value is None: - return None - if isinstance(value, TimestampValue): - return value.value - if isinstance(value, DateValue): - return datetime.combine(value.value, datetime.min.time()) - if isinstance(value, IntegerValue): - raise CursorAlgebraError("an integer cursor cannot be used as a timestamp") - return datetime.fromisoformat(value) - - -def parse_cursor_integer(value: str | CursorScalar | None) -> int | None: - if value is None: - return None - if isinstance(value, IntegerValue): - return value.value - if isinstance(value, DateValue | TimestampValue): - raise CursorAlgebraError("a timestamp cursor cannot be used as an integer") - return int(Decimal(value)) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py index 180f99d001..b9af715ee9 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py @@ -8,17 +8,17 @@ from pathlib import Path from typing import Protocol, TextIO -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import ( + ConnectionProgressReporter, + PlanningProgressReporter, + discover_project_inputs, + supports_color, +) from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) from sqlbuild.cli.commands.models import AdapterConnectionContext -from sqlbuild.cli.progress.main._build_command_progress_reporters import ( - build_command_progress_reporters, -) -from sqlbuild.cli.progress.models import CommandProgressReporters from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color class _CommandInvocationRequest(Protocol): @@ -79,11 +79,6 @@ def resolve_reported_command_invocation[InvocationT]( context: _CommandInvocationContext = _resolve_command_invocation_context( request=request, require_connection=require_connection ) - reporters: CommandProgressReporters = build_command_progress_reporters( - adapter_name=context.adapter_context.adapter_name, - stream=context.progress_stream, - use_color=context.use_color, - ) return invocation_type( effective_project_dir=context.effective_project_dir, discovered_inputs=context.discovered_inputs, @@ -92,8 +87,14 @@ def resolve_reported_command_invocation[InvocationT]( connection_config=context.adapter_context.connection_config, use_color=context.use_color, progress_stream=context.progress_stream, - connection_progress=reporters.connection, - planning_progress=reporters.planning, + connection_progress=ConnectionProgressReporter( + adapter_name=context.adapter_context.adapter_name, + stream=context.progress_stream, + use_color=context.use_color, + ), + planning_progress=PlanningProgressReporter( + stream=context.progress_stream, use_color=context.use_color + ), ) diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 6f7ff03df6..29671a69ed 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from pathlib import Path +from sqlbuild._native import supports_color from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( log_compute_capture_failure, ) @@ -13,7 +14,6 @@ from sqlbuild.diagnostics.classes.invocation_diagnostic_routing import InvocationDiagnosticRouting from sqlbuild.diagnostics.main.configure import configure_diagnostics from sqlbuild.diagnostics.models import DiagnosticRoutingOptions -from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.compute_logs.classes.local_filesystem_compute_log_storage import ( LocalFilesystemComputeLogStorage, ) diff --git a/src/sqlbuild/cli/commands/main/execution/__init__.py b/src/sqlbuild/cli/commands/main/execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/main/execution/connection_progress.py b/src/sqlbuild/cli/commands/main/execution/connection_progress.py deleted file mode 100644 index 3d6ae93e2a..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/connection_progress.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Connection progress public entry for cross-domain orchestration.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild._native import ConnectionProgressReporter - - -def build_connection_progress_reporter( - *, - adapter_name: str, - stream: TextIO, - blank_line_after_complete: bool = False, - use_color: bool = False, -) -> ConnectionProgressReporter: - """Build a connection progress reporter.""" - - return ConnectionProgressReporter( - adapter_name=adapter_name, - stream=stream, - blank_line_after_complete=blank_line_after_complete, - use_color=use_color, - ) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index ef924758ec..08aab9280d 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -51,6 +51,16 @@ from sqlbuild.compiler.refactoring.models import RefactorProject +@dataclass(frozen=True) +class NestedProgressChildRow: + """One child row rendered below a completed nested progress item.""" + + label: str + name: str + status_text: str + detail: str = "" + + @dataclass(frozen=True) class AuditCommandRequest: """CLI inputs for one audit command invocation.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py index abb322f09a..5c124e001a 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py @@ -9,7 +9,7 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import NativeReuseAttempt +from sqlbuild._native import NativeReuseAttempt, supports_color from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( invocation_identity, redirected_output_files, @@ -51,7 +51,6 @@ COMPILE_CACHE_DISABLE_VALUE, ) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.presentation.main.supports_color import supports_color _LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) _OUTCOMES: dict[str, CompileReuseOutcome] = { diff --git a/src/sqlbuild/cli/progress/__init__.py b/src/sqlbuild/cli/progress/__init__.py deleted file mode 100644 index c202eed76a..0000000000 --- a/src/sqlbuild/cli/progress/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI progress rendering.""" diff --git a/src/sqlbuild/cli/progress/main/__init__.py b/src/sqlbuild/cli/progress/main/__init__.py deleted file mode 100644 index afe461d975..0000000000 --- a/src/sqlbuild/cli/progress/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI progress entry functions.""" diff --git a/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py b/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py deleted file mode 100644 index 826778e69f..0000000000 --- a/src/sqlbuild/cli/progress/main/_build_command_progress_reporters.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Progress reporter construction for CLI commands.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter -from sqlbuild.cli.progress.models import CommandProgressReporters - - -def build_command_progress_reporters( - *, - adapter_name: str, - stream: TextIO, - use_color: bool, -) -> CommandProgressReporters: - """Build connection and planning progress reporters for one command.""" - - return CommandProgressReporters( - connection=ConnectionProgressReporter( - adapter_name=adapter_name, - stream=stream, - use_color=use_color, - ), - planning=PlanningProgressReporter( - stream=stream, - use_color=use_color, - ), - ) diff --git a/src/sqlbuild/cli/progress/main/verbose_inspection_output.py b/src/sqlbuild/cli/progress/main/verbose_inspection_output.py deleted file mode 100644 index 84d16e9534..0000000000 --- a/src/sqlbuild/cli/progress/main/verbose_inspection_output.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Verbose per-query warehouse inspection output on stderr.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager - -from sqlbuild._native import VerboseInspectionWriter -from sqlbuild.adapter.relations.main.inspection_query_sink import inspection_query_sink - - -@contextmanager -def verbose_inspection_output(*, enabled: bool) -> Iterator[None]: - """Write each inspection read, then a total, to stderr while ``enabled``.""" - - if not enabled: - yield - return - writer: VerboseInspectionWriter = VerboseInspectionWriter() - with inspection_query_sink(writer.write): - yield - writer.write_total() diff --git a/src/sqlbuild/cli/progress/models.py b/src/sqlbuild/cli/progress/models.py deleted file mode 100644 index 91c51b4c99..0000000000 --- a/src/sqlbuild/cli/progress/models.py +++ /dev/null @@ -1,25 +0,0 @@ -"""CLI progress models.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild._native import ConnectionProgressReporter, PlanningProgressReporter - - -@dataclass(frozen=True) -class CommandProgressReporters: - """Connection and planning progress reporters bound to one output stream.""" - - connection: ConnectionProgressReporter - planning: PlanningProgressReporter - - -@dataclass(frozen=True) -class NestedProgressChildRow: - """One child row rendered below a completed nested progress item.""" - - label: str - name: str - status_text: str - detail: str = "" diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py index 44a93ea5ed..b2c8f288f5 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/execution_phases.py @@ -13,9 +13,6 @@ resolve_target_config, ) from sqlbuild.cli.commands.main.dbt.dbt_sqlbuild_work import execute_dbt_sqlbuild_work -from sqlbuild.cli.commands.main.execution.connection_progress import ( - build_connection_progress_reporter, -) from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput @@ -145,7 +142,7 @@ def resolve_sqlbuild_execution_plan_output( return None invocation.output_stream.write("\n") invocation.output_stream.flush() - connection_progress: ConnectionProgressReporter = build_connection_progress_reporter( + connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( adapter_name=compiled.adapter_name, stream=invocation.output_stream, blank_line_after_complete=True, diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/plan.py b/src/sqlbuild/integrations/dbt/main/pipeline/plan.py index 35f46789c6..47b74d6a66 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/plan.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/plan.py @@ -7,9 +7,7 @@ from pathlib import Path from typing import Any, TextIO -from sqlbuild.cli.commands.main.execution.connection_progress import ( - build_connection_progress_reporter, -) +from sqlbuild._native import ConnectionProgressReporter from sqlbuild.integrations.dbt._helpers.pipeline.interop_prologue import ( compile_dbt_interop_project, load_compiled_dbt_manifest, @@ -75,7 +73,7 @@ def plan_dbt_interop_from_project( on_progress=on_progress, ) connection_progress: Any | None = ( - build_connection_progress_reporter( + ConnectionProgressReporter( adapter_name=compiled.adapter_name, stream=progress_stream, use_color=use_color, diff --git a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py b/src/sqlbuild/presentation/_helpers/terminal_capabilities.py index bc8cc09b94..ec3b5e3cfa 100644 --- a/src/sqlbuild/presentation/_helpers/terminal_capabilities.py +++ b/src/sqlbuild/presentation/_helpers/terminal_capabilities.py @@ -2,21 +2,11 @@ from __future__ import annotations -import os import shutil -import sys _DEFAULT_TERMINAL_COLUMNS: int = 120 -def supports_color() -> bool: - if os.environ.get("NO_COLOR") is not None: - return False - if not hasattr(sys.stdout, "isatty"): - return False - return sys.stdout.isatty() - - def terminal_columns() -> int: """Return the current terminal width in columns.""" diff --git a/src/sqlbuild/presentation/main/supports_color.py b/src/sqlbuild/presentation/main/supports_color.py deleted file mode 100644 index c98d0ad2eb..0000000000 --- a/src/sqlbuild/presentation/main/supports_color.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public terminal color capability entry.""" - -from __future__ import annotations - -from sqlbuild.presentation._helpers.terminal_capabilities import supports_color as _supports_color - - -def supports_color() -> bool: - """Return whether the active output terminal supports color.""" - - return _supports_color() diff --git a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py index 312bb5130e..491d20a309 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py +++ b/tests/unit/scripts/fensu_policy/rules/test_canonical_operations.py @@ -19,17 +19,17 @@ "test_case", [ CustomRuleTestCase( - description="raw color capability import faults", + description="raw terminal capability import faults", path="src/sqlbuild/example/main/build.py", source=( - "from sqlbuild.presentation._helpers.terminal_capabilities import supports_color\n" + "from sqlbuild.presentation._helpers.terminal_capabilities import terminal_columns\n" ), expected_fault_count=1, ), CustomRuleTestCase( - description="public color capability import passes", + description="public terminal capability import passes", path="src/sqlbuild/example/main/build.py", - source="from sqlbuild.presentation.main.supports_color import supports_color\n", + source="from sqlbuild.presentation.main.terminal_columns import terminal_columns\n", expected_fault_count=0, ), ], diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 2d8f5984f9..bb6f34880b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation +from sqlbuild._native import parse_cli_invocation from sqlbuild.cli.commands.types import CompileLineageMode, PlaygroundTemplate from sqlbuild.cli.entry.models import ( ParsedCliInvocation, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index efe524a245..870af56275 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -7,7 +7,7 @@ import pytest from sqlbuild._native import describe_connection_warehouse, resolve_project_connection_config -from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation +from sqlbuild._native import parse_cli_invocation from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py index c67d0fc59f..8c7f13ab9c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -7,7 +7,6 @@ import sqlbuild._native as dbt_init_module import sqlbuild.integrations.dbt.main.profile.profile_init as dbt_profile_init -import sqlbuild.presentation.main.supports_color as color_support from sqlbuild.cli.commands.models import DbtInitCommandRequest from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult @@ -16,6 +15,7 @@ DbtInitProjectDirDefaultTestCase, DbtInitValidationOrderTestCase, ) +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -173,7 +173,7 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: dry_run=False, expected_fragments=(), expected_color_fragments=( - "\033[2mInspecting dbt project and profile...\033[0m", + "\033[32mâ ‹\033[0m Inspecting dbt project and profile...", "\033[34m\033[1mSQLBuild project created\033[0m", "\033[1mSetup summary\033[0m:", "\033[34mProject", @@ -209,7 +209,7 @@ def run_dbt_profile_init(*, request: DbtInitRequest) -> DbtInitResult: toml='name = "analytics"\n', ) - monkeypatch.setattr(color_support, "supports_color", lambda: True) + use_color_terminal(monkeypatch) monkeypatch.setattr(dbt_profile_init, "run_dbt_profile_init", run_dbt_profile_init) monkeypatch.setattr( dbt_profile_init, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 13806edd53..ae59421a72 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -52,6 +52,7 @@ build_json_recording_handler, invalid_loader_destination_error, ) +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -2167,7 +2168,7 @@ def test_given_parser_error_and_color_support_when_running_main_then_it_colorize capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("sqlbuild.presentation.main.supports_color.supports_color", lambda: True) + use_color_terminal(monkeypatch) exit_code: int = main(test_case.argv) rendered_stderr: str = capsys.readouterr().err @@ -2193,7 +2194,7 @@ def test_given_parser_error_and_no_color_when_running_main_then_it_renders_plain capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("sqlbuild.presentation.main.supports_color.supports_color", lambda: True) + use_color_terminal(monkeypatch) exit_code: int = main(test_case.argv) rendered_stderr: str = capsys.readouterr().err @@ -2467,7 +2468,7 @@ def test_given_expected_cli_error_and_color_support_when_running_main_then_it_co capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr("sqlbuild.presentation.main.supports_color.supports_color", lambda: True) + use_color_terminal(monkeypatch) def run_query( project_dir: Path | None, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 6da7fb9048..431eb0306d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -17,8 +17,8 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module -from sqlbuild.presentation.main import supports_color as supports_color_module from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal def fail_build_discovery(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> Mock: @@ -57,7 +57,7 @@ def configure_native_scenario_command( local_config=LocalConfig(), ) monkeypatch.setattr(sys, "stdout", stream) - monkeypatch.setattr(supports_color_module, "supports_color", lambda: False) + use_plain_terminal(monkeypatch) monkeypatch.setattr(native, "discover_project_inputs", lambda **_: discovered_inputs) monkeypatch.setattr(native, "resolve_project_connection_config", lambda **_: {}) monkeypatch.setattr(native, "run_compile_only_pipeline", compile_project) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py index f2b1e72ffe..fb3e5b2524 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py @@ -6,9 +6,9 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.presentation.main.supports_color as color_module from sqlbuild._native import run_init_command from tests.unit.src.sqlbuild.cli.commands.main.init._test_types import InitScaffoldTestCase +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal, use_plain_terminal @pytest.mark.parametrize( @@ -87,7 +87,7 @@ def test_given_empty_directory_when_running_init_then_scaffolds_project_director project_dir: Path = tmp_path / test_case.project_dir_name project_dir.mkdir() monkeypatch.setattr(native_module, "update_sqlbuild_skills", lambda *, project_dir: None) - monkeypatch.setattr(color_module, "supports_color", lambda: False) + use_plain_terminal(monkeypatch) result: int = run_init_command(project_dir) @@ -131,7 +131,7 @@ def test_given_color_terminal_when_running_init_then_it_styles_heading( project_dir: Path = tmp_path / test_case.project_dir_name project_dir.mkdir() monkeypatch.setattr(native_module, "update_sqlbuild_skills", lambda *, project_dir: None) - monkeypatch.setattr(color_module, "supports_color", lambda: True) + use_color_terminal(monkeypatch) result: int = run_init_command(project_dir) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 026803509f..d4843a3838 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -9,12 +9,12 @@ from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.cli.output.models import SkillMaintenanceResult -from sqlbuild.presentation.main import supports_color as playground_completion_output from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, RunPlaygroundTestCase, WrapperPlaygroundSkillsTestCase, ) +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -399,7 +399,7 @@ def test_given_color_terminal_when_running_playground_command_then_it_styles_key capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(playground_completion_output, "supports_color", lambda: True) + use_color_terminal(monkeypatch) exit_code: int = run_playground_command( PlaygroundCommandRequest( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index f19b8b079b..ea6763184b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -11,9 +11,9 @@ from sqlbuild.cli.commands._helpers.runtime import adapter_context from sqlbuild.cli.commands.models import ScopeCommandRequest from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex -from sqlbuild.presentation.main import supports_color as color_support from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup +from tests.unit.src.sqlbuild.presentation.helpers import use_color_terminal @pytest.mark.parametrize( @@ -99,7 +99,7 @@ def test_given_no_color_flag_when_rendering_text_to_color_terminal_then_output_i capsys: pytest.CaptureFixture[str], test_case: ScopeCommandCase, ) -> None: - monkeypatch.setattr(color_support, "supports_color", lambda: True) + use_color_terminal(monkeypatch) index: ScopeIndex = report_scope_lookup().index outputs: list[str] = [] for no_color in (False, True): diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 0be484741e..77e77250e7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -18,7 +18,7 @@ ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.progress.models import NestedProgressChildRow +from sqlbuild.cli.commands.models import NestedProgressChildRow from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode diff --git a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py index 2986b6d893..806d07e06d 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/_test_types.py @@ -17,9 +17,8 @@ class VerboseInspectionFormatTestCase: @dataclass(frozen=True) class VerboseInspectionOutputTestCase: - """Inspection reads recorded inside and after a verbose output context.""" + """Inspection reads recorded inside and after a verbose writer sink.""" description: str - enabled: bool records: tuple[tuple[str, float, int], ...] expected_stderr_lines: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py index 775b1d4844..58f8dd94c1 100644 --- a/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py +++ b/tests/unit/src/sqlbuild/cli/progress/main/verbose_inspection_output/test_verbose_inspection_output.py @@ -6,9 +6,9 @@ from _pytest.capture import CaptureResult from sqlbuild._native import VerboseInspectionWriter +from sqlbuild.adapter.relations.main.inspection_query_sink import inspection_query_sink from sqlbuild.adapter.relations.main.record_inspection_query import record_inspection_query from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output from tests.unit.src.sqlbuild.cli.progress.main.verbose_inspection_output._test_types import ( VerboseInspectionFormatTestCase, VerboseInspectionOutputTestCase, @@ -74,8 +74,7 @@ def test_given_inspection_record_when_formatting_then_renders_one_bounded_line( "test_case", [ VerboseInspectionOutputTestCase( - description="enabled output writes each read and a total to stderr", - enabled=True, + description="sink writes each read and the total to stderr", records=(("SELECT 1", 0.5, 3), ("SELECT 2", 0.25, 0)), expected_stderr_lines=( " inspect 0.50s 3 rows SELECT 1", @@ -84,18 +83,18 @@ def test_given_inspection_record_when_formatting_then_renders_one_bounded_line( ), ), VerboseInspectionOutputTestCase( - description="disabled output writes nothing", - enabled=False, - records=(("SELECT 1", 0.5, 3),), + description="no reads write no total", + records=(), expected_stderr_lines=(), ), ], ids=lambda case: case.description, ) -def test_given_verbose_setting_when_recording_reads_then_writes_only_inside_context_to_stderr( +def test_given_writer_sink_when_recording_reads_then_writes_only_inside_sink_to_stderr( test_case: VerboseInspectionOutputTestCase, capsys: pytest.CaptureFixture[str] ) -> None: - with verbose_inspection_output(enabled=test_case.enabled): + writer: VerboseInspectionWriter = VerboseInspectionWriter() + with inspection_query_sink(writer.write): for sql, elapsed, rows in test_case.records: record_inspection_query( record=InspectionQueryRecord(sql=sql, elapsed_seconds=elapsed, row_count=rows) @@ -103,6 +102,7 @@ def test_given_verbose_setting_when_recording_reads_then_writes_only_inside_cont record_inspection_query( record=InspectionQueryRecord(sql="SELECT 3", elapsed_seconds=0.1, row_count=0) ) + writer.write_total() captured: CaptureResult[str] = capsys.readouterr() assert captured.out == "" diff --git a/tests/unit/src/sqlbuild/presentation/helpers.py b/tests/unit/src/sqlbuild/presentation/helpers.py new file mode 100644 index 0000000000..7c2cf34b10 --- /dev/null +++ b/tests/unit/src/sqlbuild/presentation/helpers.py @@ -0,0 +1,40 @@ +"""Terminal colour capability fixtures for CLI output tests.""" + +from __future__ import annotations + +import sys +from typing import TextIO + +import pytest + + +class ColorTerminalStream: + """A text stream that reports a terminal and writes through to the captured stream.""" + + def __init__(self, stream: TextIO) -> None: + self._stream: TextIO = stream + + def isatty(self) -> bool: + return True + + def write(self, text: str) -> int: + return self._stream.write(text) + + def flush(self) -> None: + self._stream.flush() + + def __getattr__(self, name: str) -> object: + return getattr(self._stream, name) + + +def use_color_terminal(monkeypatch: pytest.MonkeyPatch) -> None: + """Make `sys.stdout` a colour terminal that still writes to the captured output.""" + + monkeypatch.delenv("NO_COLOR", raising=False) + monkeypatch.setattr(sys, "stdout", ColorTerminalStream(sys.stdout)) + + +def use_plain_terminal(monkeypatch: pytest.MonkeyPatch) -> None: + """Disable terminal colour through `NO_COLOR`.""" + + monkeypatch.setenv("NO_COLOR", "1") From cba060961a6a4737a60bd1b0172bbee3d3299430 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:29:13 +0100 Subject: [PATCH 19/32] refactor: run the CLI observability scope natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 + .../src/dispatch/main/event_export_warning.rs | 38 ++ crates/sqlbuild-cli/src/dispatch/main/mod.rs | 1 + crates/sqlbuild-cli/src/dispatch/models.rs | 17 + crates/sqlbuild-cli/src/dispatch/tests.rs | 2 + .../src/dispatch/tests/helpers.rs | 21 +- .../tests/test_event_export_warning.rs | 56 ++ .../src/dispatch/tests/test_types.rs | 9 +- .../src/bindings/_helpers/cli/entry.rs | 17 +- .../src/bindings/_helpers/cli/mod.rs | 1 + .../bindings/_helpers/cli/observability.rs | 572 ++++++++++++++++++ .../src/bindings/_helpers/cli/registration.rs | 5 +- src/sqlbuild/_native.pyi | 7 + .../commands/_helpers/entry/observability.py | 305 ---------- ...pe.py => _event_exporter_command_scope.py} | 0 .../test_cli_concurrent_history.py | 2 +- .../_helpers/test_module_paths.py | 1 + .../commands/_helpers/entry/_test_types.py | 8 - .../cli/commands/_helpers/entry/helpers.py | 33 - .../entry/test_format_event_export_warning.py | 67 -- .../cli/commands/main/entry/helpers.py | 15 +- .../main/entry/test_observability_cleanup.py | 41 +- 22 files changed, 745 insertions(+), 476 deletions(-) create mode 100644 crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs create mode 100644 crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/entry/observability.py rename src/sqlbuild/runtime/event_exporting/main/{event_exporter_command_scope.py => _event_exporter_command_scope.py} (100%) delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py delete mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 24518a3c75..ad78844d34 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -799,3 +799,6 @@ pub(crate) const SCENARIO_CAPTURE_ROUTE: Route = routed( &SCENARIO_CAPTURE_ARGUMENTS, REQUEST_KEYWORD_SHAPE, ); + +/// The setting a user raises when lifecycle event export runs out of time at exit. +pub(crate) const SHUTDOWN_TIMEOUT_SETTING: &str = "sinks.lifecycle.shutdown_timeout"; diff --git a/crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs b/crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs new file mode 100644 index 0000000000..365528aa8f --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/main/event_export_warning.rs @@ -0,0 +1,38 @@ +//! The stderr warning for lifecycle events dropped or failed at command exit. + +use crate::dispatch::constants::SHUTDOWN_TIMEOUT_SETTING; +use crate::dispatch::models::{ExportSummary, ExporterLoss}; + +/// One warning line when lifecycle events were dropped or failed, else `None`. +#[must_use] +pub fn event_export_warning(summary: &ExportSummary, command_succeeded: bool) -> Option { + if summary.dropped == 0 && summary.failed == 0 { + return None; + } + let affected: Vec<&ExporterLoss> = summary + .per_exporter + .iter() + .filter(|exporter| exporter.dropped > 0 || exporter.failed > 0) + .collect(); + let names: String = affected + .iter() + .map(|exporter| format!("'{}'", exporter.exporter_name)) + .collect::>() + .join(", "); + let sink_label: &str = if affected.len() == 1 { "sink" } else { "sinks" }; + let unit: &str = if summary.per_exporter.len() == 1 { + "events" + } else { + "event deliveries" + }; + let prefix: &str = if command_succeeded { + "Warning: command completed successfully, but lifecycle event export was incomplete" + } else { + "Warning: lifecycle event export was incomplete" + }; + Some(format!( + "{prefix}: {} of {} {unit} dropped, {} failed ({sink_label} {names}). \ + Increase {SHUTDOWN_TIMEOUT_SETTING} or check sink health.", + summary.dropped, summary.accepted, summary.failed + )) +} diff --git a/crates/sqlbuild-cli/src/dispatch/main/mod.rs b/crates/sqlbuild-cli/src/dispatch/main/mod.rs index 73ddbae24f..d96f4ca211 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/mod.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/mod.rs @@ -1,4 +1,5 @@ pub mod command_dispatch; +pub mod event_export_warning; pub mod failure_report; pub mod invocation_policy; pub mod unavailable_handler; diff --git a/crates/sqlbuild-cli/src/dispatch/models.rs b/crates/sqlbuild-cli/src/dispatch/models.rs index eafaee9839..87230c9bf4 100644 --- a/crates/sqlbuild-cli/src/dispatch/models.rs +++ b/crates/sqlbuild-cli/src/dispatch/models.rs @@ -142,3 +142,20 @@ pub enum Dispatch { /// Commands with no handler exit successfully. Nothing, } + +/// Lifecycle event export accounting of one sink at command exit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExporterLoss { + pub exporter_name: String, + pub dropped: u64, + pub failed: u64, +} + +/// Lifecycle event export accounting of every sink at command exit. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExportSummary { + pub accepted: u64, + pub dropped: u64, + pub failed: u64, + pub per_exporter: Vec, +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests.rs b/crates/sqlbuild-cli/src/dispatch/tests.rs index a857d5eccd..2844cc4653 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_command_dispatch.rs"] mod command_dispatch; +#[path = "tests/test_event_export_warning.rs"] +mod event_export_warning; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_invocation_policy.rs"] diff --git a/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs b/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs index ac0c7c9e38..dddc39e942 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/helpers.rs @@ -1,4 +1,6 @@ -use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; +use crate::dispatch::models::{ + CommandFacts, Dispatch, DispatchFailure, ExportSummary, ExporterLoss, Route, +}; /// Facts of `command` and `subcommand` with every flag off and every required value present. pub(super) fn facts(command: &str, subcommand: Option<&str>) -> CommandFacts { @@ -25,3 +27,20 @@ pub(super) fn failure(message: &str, code: &'static str) -> DispatchFailure { help: None, } } + +/// An export summary of `(name, accepted, delivered, failed, dropped)` sinks. +pub(super) fn export_summary(exporters: &[(&str, u64, u64, u64, u64)]) -> ExportSummary { + ExportSummary { + accepted: exporters.iter().map(|exporter| exporter.1).sum(), + dropped: exporters.iter().map(|exporter| exporter.4).sum(), + failed: exporters.iter().map(|exporter| exporter.3).sum(), + per_exporter: exporters + .iter() + .map(|exporter| ExporterLoss { + exporter_name: exporter.0.to_owned(), + dropped: exporter.4, + failed: exporter.3, + }) + .collect(), + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs new file mode 100644 index 0000000000..c799b4b25a --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_event_export_warning.rs @@ -0,0 +1,56 @@ +use crate::dispatch::main::event_export_warning::event_export_warning; +use crate::dispatch::tests::helpers::export_summary; +use crate::dispatch::tests::test_types::EventExportWarningTestCase; + +#[test] +fn given_final_export_summary_when_formatting_warning_then_reports_loss_only() { + let test_cases = [ + EventExportWarningTestCase { + description: "all delivered", + summary: export_summary(&[("orders_events", 248, 248, 0, 0)]), + command_succeeded: true, + expected_warning: None, + }, + EventExportWarningTestCase { + description: "successful command with drops", + summary: export_summary(&[("orders_events", 248, 108, 0, 140)]), + command_succeeded: true, + expected_warning: Some( + "Warning: command completed successfully, but lifecycle event export was \ + incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). \ + Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + }, + EventExportWarningTestCase { + description: "failed command with failures", + summary: export_summary(&[("orders_events", 10, 9, 1, 0)]), + command_succeeded: false, + expected_warning: Some( + "Warning: lifecycle event export was incomplete: 0 of 10 events dropped, \ + 1 failed (sink 'orders_events'). \ + Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + }, + EventExportWarningTestCase { + description: "multiple sinks names only affected sinks", + summary: export_summary(&[ + ("customer_events", 20, 20, 0, 0), + ("orders_events", 20, 15, 1, 4), + ]), + command_succeeded: true, + expected_warning: Some( + "Warning: command completed successfully, but lifecycle event export was \ + incomplete: 4 of 40 event deliveries dropped, 1 failed (sink 'orders_events'). \ + Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + event_export_warning(&test_case.summary, test_case.command_succeeded).as_deref(), + test_case.expected_warning, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs index 2a20ac2140..a075ba8d89 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::dispatch::models::{CommandFacts, Dispatch, InvocationPolicy}; +use crate::dispatch::models::{CommandFacts, Dispatch, ExportSummary, InvocationPolicy}; pub(super) struct CommandDispatchTestCase { pub(super) description: &'static str, @@ -11,3 +11,10 @@ pub(super) struct InvocationPolicyTestCase { pub(super) facts: CommandFacts, pub(super) expected_policy: InvocationPolicy, } + +pub(super) struct EventExportWarningTestCase { + pub(super) description: &'static str, + pub(super) summary: ExportSummary, + pub(super) command_succeeded: bool, + pub(super) expected_warning: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 905fc37cee..d57da3604a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -14,6 +14,7 @@ use sqlbuild_cli::dispatch::models::{CommandFacts, CommandFailure, InvocationPol use crate::bindings::_helpers::cli::argv::parse_cli_invocation; use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; +use crate::bindings::_helpers::cli::observability::cli_observability_scope; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; @@ -23,7 +24,6 @@ const NO_COLOR_OPTION: &str = "--no-color"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMPUTE_LOGS_MODULE: &str = "sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"; -const OBSERVABILITY_MODULE: &str = "sqlbuild.cli.commands._helpers.entry.observability"; const LIFECYCLE_EVENT_MODULE: &str = "sqlbuild.runtime.observability.main.create_lifecycle_event"; const INVOCATION_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.invocation_scope"; const EXTERNAL_SCOPE_MODULE: &str = @@ -417,13 +417,14 @@ fn observed_dispatch<'py>( .unwrap_or(u64::MAX) .into_bound_py_any(py) }; - let scope: Bound<'py, PyAny> = call_with( - &imported(py, OBSERVABILITY_MODULE, "cli_observability_scope")?, - &[ - ("args", invocation.args.clone()), - ("project_dir", invocation.project_dir()?), - ], - )?; + let scope: Bound<'py, PyAny> = Bound::new( + py, + cli_observability_scope( + invocation.args.clone().unbind(), + invocation.project_dir()?.unbind(), + ), + )? + .into_any(); within(&scope, |dispatcher| { let publish = |event: Bound<'py, PyAny>| -> PyResult<()> { dispatcher.call_method1("publish_lifecycle", (event,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 4f49628d00..101172f2be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod argv; pub(crate) mod dispatch; pub(crate) mod entry; +pub(crate) mod observability; pub(crate) mod progress; pub(crate) mod progress_reporters; pub(crate) mod progress_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs new file mode 100644 index 0000000000..c51f83098a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -0,0 +1,572 @@ +//! The observed CLI invocation: lifecycle dispatcher, native progress, terminal index and sinks. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, MutexGuard}; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyString, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_cli::dispatch::main::event_export_warning::event_export_warning; +use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EVENT_DISPATCHER_MODULE: &str = "sqlbuild.runtime.observability.classes.event_dispatcher"; +const DISPATCHER_SCOPE_MODULE: &str = "sqlbuild.runtime.observability.main.dispatcher_scope"; +const IDENTITY_MODULE: &str = "sqlbuild.runtime.observability.main.current_execution_identity"; +const EXTENSIONS_MODULE: &str = "sqlbuild.compiler.discovery.main.runtime_extensions"; +const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; +const EXPORTER_DISPATCHER_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.dispatcher"; +const EXPORTER_SCOPE_CLASS_MODULE: &str = "sqlbuild.runtime.event_exporting.classes.command_scope"; +const EXPORTER_SCOPE_MODULE: &str = + "sqlbuild.runtime.event_exporting.main._event_exporter_command_scope"; +const EXPORTING_CONSTANTS_MODULE: &str = "sqlbuild.runtime.event_exporting.constants"; +const OUTPUT_CAPTURE_MODULE: &str = + "sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture"; +const DEBUG_EVENT_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const OBSERVABILITY_LOGGER: &str = "sqlbuild.cli.observability"; +const DEBUG_COMMAND: &str = "debug"; +const LINEAGE_COMMAND: &str = "lineage"; +const INVOCATION_COMPLETED: &str = "invocation_completed"; +const INVOCATION_FAILED: &str = "invocation_failed"; +const SUMMARY_FIELDS: [&str; 3] = ["queue_depth", "queue_capacity", "flush_complete"]; +const COUNT_FIELDS: [&str; 5] = ["accepted", "filtered", "delivered", "failed", "dropped"]; + +/// `log_debug_event(logger=, message=message, **fields)`. +fn log_debug<'py>( + py: Python<'py>, + message: &str, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult<()> { + let logger: Bound<'py, PyAny> = + imported(py, "logging", "getLogger")?.call1((OBSERVABILITY_LOGGER,))?; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("logger", logger), + ("message", PyString::new(py, message).into_any()), + ]; + pairs.extend(fields.iter().cloned()); + call_with( + &imported(py, DEBUG_EVENT_MODULE, "log_debug_event")?, + &pairs, + )?; + Ok(()) +} + +fn error_type_name<'py>(py: Python<'py>, error: &PyErr) -> PyResult> { + error.get_type(py).getattr("__name__") +} + +/// Log a failed lifecycle subscriber. +#[pyfunction] +fn log_dispatch_failure(failure: &Bound<'_, PyAny>) -> PyResult<()> { + log_debug( + failure.py(), + "Lifecycle event subscriber failed", + &[ + ("error_type", failure.getattr("error_type")?), + ("channel", failure.getattr("channel")?), + ("subscriber", failure.getattr("subscriber")?), + ], + ) +} + +/// Log a failed event exporter delivery. +#[pyfunction] +fn log_exporter_failure(failure: &Bound<'_, PyAny>) -> PyResult<()> { + log_debug( + failure.py(), + "Event exporter delivery failed", + &[ + ("exporter_name", failure.getattr("exporter_name")?), + ("error_type", failure.getattr("error_type")?), + ("event_kind", failure.getattr("event_kind")?), + ("event_severity", failure.getattr("event_severity")?), + ], + ) +} + +/// Log a failed command-output sink delivery. +#[pyfunction] +fn log_command_output_failure(error: &Bound<'_, PyAny>) -> PyResult<()> { + log_debug( + error.py(), + "Command-output sink delivery failed", + &[("error_type", error.get_type().getattr("__name__")?)], + ) +} + +/// Log the event exporter delivery summary and each exporter's accounting. +#[pyfunction] +fn log_exporter_summary(summary: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = summary.py(); + let mut fields: Vec<(&str, Bound<'_, PyAny>)> = Vec::new(); + for name in COUNT_FIELDS.into_iter().chain(SUMMARY_FIELDS) { + fields.push((name, summary.getattr(name)?)); + } + log_debug(py, "Event exporter delivery summary", &fields)?; + for exporter in summary.getattr("per_exporter")?.try_iter()? { + let exporter: Bound<'_, PyAny> = exporter?; + let counts: Bound<'_, PyAny> = exporter.getattr("counts")?; + let mut fields: Vec<(&str, Bound<'_, PyAny>)> = + vec![("exporter_name", exporter.getattr("exporter_name")?)]; + for name in COUNT_FIELDS { + fields.push((name, counts.getattr(name)?)); + } + for name in SUMMARY_FIELDS { + fields.push((name, summary.getattr(name)?)); + } + log_debug(py, "Event exporter accounting", &fields)?; + } + Ok(()) +} + +/// Records whether the invocation published a successful terminal fact. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct InvocationOutcome { + succeeded: AtomicBool, +} + +#[pymethods] +impl InvocationOutcome { + fn consume(&self, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event_type: String = event.getattr("event_type")?.extract()?; + if event_type == INVOCATION_COMPLETED || event_type == INVOCATION_FAILED { + self.succeeded + .store(event_type == INVOCATION_COMPLETED, Ordering::SeqCst); + } + Ok(()) + } +} + +/// What an observed invocation installed, released in reverse when it ends. +struct Observed { + dispatcher: Py, + projector: Py, + projector_token: Py, + unsubscribe_terminal_index: Py, + unsubscribe_progress: Py, + outcome: Py, + exporter_scope: Option>, + unsubscribe_exporters: Option>, + unsubscribe_outcome: Option>, + /// Context managers entered, innermost last. + entered: Vec>, +} + +/// Log a cleanup failure without letting logging fail the command. +fn report_cleanup_failure(py: Python<'_>, error: &PyErr, phase: &str) { + let reported = error_type_name(py, error).and_then(|error_type| { + log_debug( + py, + "CLI observability cleanup failed", + &[ + ("error_type", error_type), + ("phase", PyString::new(py, phase).into_any()), + ], + ) + }); + drop(reported); +} + +fn run_cleanup(py: Python<'_>, result: PyResult, phase: &str) { + if let Err(error) = result { + report_cleanup_failure(py, &error, phase); + } +} + +/// Native export accounting of a Python `EventExportSummary`. +fn export_summary(summary: &Bound<'_, PyAny>) -> PyResult { + let mut per_exporter: Vec = Vec::new(); + for exporter in summary.getattr("per_exporter")?.try_iter()? { + let exporter: Bound<'_, PyAny> = exporter?; + let counts: Bound<'_, PyAny> = exporter.getattr("counts")?; + per_exporter.push(ExporterLoss { + exporter_name: exporter.getattr("exporter_name")?.extract()?, + dropped: counts.getattr("dropped")?.extract()?, + failed: counts.getattr("failed")?.extract()?, + }); + } + Ok(ExportSummary { + accepted: summary.getattr("accepted")?.extract()?, + dropped: summary.getattr("dropped")?.extract()?, + failed: summary.getattr("failed")?.extract()?, + per_exporter, + }) +} + +/// Print the incomplete-export warning to stderr when events were dropped or failed. +fn warn_incomplete_export(summary: &Bound<'_, PyAny>, command_succeeded: bool) -> PyResult<()> { + let py: Python<'_> = summary.py(); + let Some(message) = event_export_warning(&export_summary(summary)?, command_succeeded) else { + return Ok(()); + }; + let options = pyo3::types::PyDict::new(py); + pyo3::types::PyDictMethods::set_item(&options, "file", py.import("sys")?.getattr("stderr")?)?; + pyo3::types::PyDictMethods::set_item(&options, "flush", true)?; + py.import("builtins")? + .getattr("print")? + .call((message,), Some(&options))?; + Ok(()) +} + +impl Observed { + /// Leave entered contexts innermost first; the exception raised by one, and whether any suppressed. + fn exit_contexts( + &mut self, + py: Python<'_>, + mut pending: Option, + ) -> (Option, bool) { + let mut suppressed: bool = false; + while let Some(manager) = self.entered.pop() { + let arguments: (Py, Py, Py) = pending.as_ref().map_or_else( + || (py.None(), py.None(), py.None()), + |error| { + ( + error.get_type(py).into_any().unbind(), + error.value(py).clone().into_any().unbind(), + error + .traceback(py) + .map_or_else(|| py.None(), |traceback| traceback.into_any().unbind()), + ) + }, + ); + match manager.bind(py).call_method1("__exit__", arguments) { + Ok(result) => { + if pending.is_some() && result.is_truthy().unwrap_or(false) { + pending = None; + suppressed = true; + } + } + Err(error) => { + pending = Some(error); + suppressed = false; + } + } + } + (pending, suppressed) + } + + /// Release exporters, progress and the terminal index, then warn about lost events. + fn release(self, py: Python<'_>) { + if let Some(unsubscribe) = &self.unsubscribe_exporters { + run_cleanup( + py, + unsubscribe.bind(py).call0(), + "event_exporter_unsubscribe", + ); + } + if let Some(unsubscribe) = &self.unsubscribe_outcome { + run_cleanup( + py, + unsubscribe.bind(py).call0(), + "invocation_outcome_unsubscribe", + ); + } + let summary: Option> = self.exporter_scope.as_ref().and_then(|scope| { + match scope.bind(py).call_method0("close") { + Ok(summary) => Some(summary), + Err(error) => { + report_cleanup_failure(py, &error, "event_exporter_shutdown"); + None + } + } + }); + run_cleanup( + py, + self.unsubscribe_terminal_index.bind(py).call0(), + "terminal_index_unsubscribe", + ); + run_cleanup( + py, + self.unsubscribe_progress.bind(py).call0(), + "progress_unsubscribe", + ); + let projector: &Bound<'_, PyAny> = self.projector.bind(py); + run_cleanup(py, projector.call_method0("close"), "progress_close"); + run_cleanup( + py, + projector.call_method1("restore", (&self.projector_token,)), + "progress_context_restore", + ); + if let Some(summary) = summary.filter(|summary| !summary.is_none()) { + let succeeded: bool = self.outcome.get().succeeded.load(Ordering::SeqCst); + run_cleanup( + py, + warn_incomplete_export(&summary, succeeded), + "event_exporter_warning", + ); + } + } + + fn subscribe(&self, py: Python<'_>, subscriber: Bound<'_, PyAny>) -> PyResult> { + Ok(call_with( + &self.dispatcher.bind(py).getattr("subscribe_lifecycle")?, + &[ + ("subscriber", subscriber), + ( + "accepts_opaque", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )? + .unbind()) + } + + fn enter(&mut self, manager: Bound<'_, PyAny>) -> PyResult<()> { + manager.call_method0("__enter__")?; + self.entered.push(manager.unbind()); + Ok(()) + } + + /// Bind configured project event exporters and command-output sinks, then install scopes. + fn install( + &mut self, + py: Python<'_>, + (args, project_dir): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + terminal_index: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let mut command_output_sinks: bool = false; + if args.getattr("command")?.ne(LINEAGE_COMMAND)? { + let extensions: Bound<'_, PyAny> = call_with( + &imported(py, EXTENSIONS_MODULE, "discover_runtime_extensions")?, + &[("project_dir", project_dir.clone())], + )?; + let event_exporters: Bound<'_, PyAny> = extensions.getattr("event_exporters")?; + let sinks: Bound<'_, PyAny> = extensions.getattr("command_output_sinks")?; + command_output_sinks = sinks.is_truthy()?; + if event_exporters.is_truthy()? || command_output_sinks { + imported(py, LAUNCH_RECORD_MODULE, "withhold_launch_record")?.call0()?; + let timeout: Bound<'_, PyAny> = + extensions.getattr("lifecycle_shutdown_timeout_seconds")?; + let timeout: Bound<'_, PyAny> = if timeout.is_none() { + imported( + py, + EXPORTING_CONSTANTS_MODULE, + "DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS", + )? + } else { + timeout.extract::()?.into_bound_py_any(py)? + }; + let delivery: Bound<'_, PyAny> = call_with( + &imported(py, EXPORTER_DISPATCHER_MODULE, "EventExporterDispatcher")?, + &[ + ("shutdown_timeout_seconds", timeout), + ( + "failure_callback", + wrap_pyfunction!(log_exporter_failure, py)?.into_any(), + ), + ( + "summary_callback", + wrap_pyfunction!(log_exporter_summary, py)?.into_any(), + ), + ], + )?; + let scope: Bound<'_, PyAny> = call_with( + &imported(py, EXPORTER_SCOPE_CLASS_MODULE, "EventExporterCommandScope")?, + &[("dispatcher", delivery.clone())], + )?; + self.exporter_scope = Some(scope.clone().unbind()); + call_with( + &scope.getattr("configure_extensions")?, + &[ + ("project_dir", project_dir.clone()), + ("providers", extensions.getattr("providers")?), + ("event_exporters", event_exporters), + ("command_output_sinks", sinks), + ], + )?; + self.unsubscribe_exporters = + Some(self.subscribe(py, delivery.getattr("enqueue")?)?); + let outcome: Bound<'_, PyAny> = self.outcome.bind(py).clone().into_any(); + self.unsubscribe_outcome = Some(self.subscribe(py, outcome.getattr("consume")?)?); + } + } + self.enter( + imported(py, DISPATCHER_SCOPE_MODULE, "dispatcher_scope")? + .call1((self.dispatcher.bind(py),))?, + )?; + self.enter( + imported(py, NATIVE_MODULE, "terminal_event_index_scope")?.call1((terminal_index,))?, + )?; + let Some(scope) = self + .exporter_scope + .as_ref() + .map(|scope| scope.bind(py).clone()) + else { + return Ok(()); + }; + self.enter( + imported(py, EXPORTER_SCOPE_MODULE, "event_exporter_command_scope")? + .call1((&scope,))?, + )?; + let identity: Bound<'_, PyAny> = + imported(py, IDENTITY_MODULE, "current_execution_identity")?.call0()?; + if !identity.is_none() && command_output_sinks { + self.enter(call_with( + &imported(py, OUTPUT_CAPTURE_MODULE, "configured_output_capture_scope")?, + &[ + ("exporter_scope", scope), + ("identity", identity), + ( + "failure_callback", + wrap_pyfunction!(log_command_output_failure, py)?.into_any(), + ), + ], + )?)?; + } + Ok(()) + } +} + +/// Installs native progress, terminal indexing and configured event exporters for one command. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CliObservabilityScope { + args: Py, + project_dir: Py, + observed: Mutex>, +} + +impl CliObservabilityScope { + fn locked(&self) -> PyResult>> { + self.observed + .lock() + .map_err(|_| PyRuntimeError::new_err("observability scope lock is poisoned")) + } +} + +#[pymethods] +impl CliObservabilityScope { + fn __enter__(&self, py: Python<'_>) -> PyResult> { + let args: &Bound<'_, PyAny> = self.args.bind(py); + let dispatcher: Bound<'_, PyAny> = call_with( + &imported(py, EVENT_DISPATCHER_MODULE, "EventDispatcher")?, + &[( + "health_callback", + wrap_pyfunction!(log_dispatch_failure, py)?.into_any(), + )], + )?; + let terminal_index: Bound<'_, PyAny> = + imported(py, NATIVE_MODULE, "TerminalEventIndex")?.call0()?; + let subscribe = |subscriber: Bound<'_, PyAny>| -> PyResult> { + Ok(call_with( + &dispatcher.getattr("subscribe_lifecycle")?, + &[ + ("subscriber", subscriber), + ( + "accepts_opaque", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )? + .unbind()) + }; + let unsubscribe_terminal_index: Py = subscribe(terminal_index.getattr("consume")?)?; + let machine_output: bool = imported(py, NATIVE_MODULE, "machine_output_mode")? + .call1((args,))? + .is_truthy()?; + let debug: Bound<'_, PyAny> = args.getattr("debug")?; + let progress_on_stderr: bool = + machine_output || debug.is_truthy()? || args.getattr("command")?.eq(DEBUG_COMMAND)?; + let use_color: bool = !machine_output + && !args.getattr("no_color")?.is_truthy()? + && stdout_supports_color(py)?; + let projector: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "NativeProgressProjector")?, + &[ + ( + "stream", + py.import("sys")?.getattr(if progress_on_stderr { + "stderr" + } else { + "stdout" + })?, + ), + ( + "use_color", + PyBool::new(py, use_color).to_owned().into_any(), + ), + ("debug", debug), + ], + )?; + let unsubscribe_progress: Py = subscribe(projector.getattr("consume")?)?; + let projector_token: Bound<'_, PyAny> = projector.call_method0("install")?; + let mut observed: Observed = Observed { + dispatcher: dispatcher.clone().unbind(), + projector: projector.unbind(), + projector_token: projector_token.unbind(), + unsubscribe_terminal_index, + unsubscribe_progress, + outcome: Py::new( + py, + InvocationOutcome { + succeeded: AtomicBool::new(false), + }, + )?, + exporter_scope: None, + unsubscribe_exporters: None, + unsubscribe_outcome: None, + entered: Vec::new(), + }; + if let Err(error) = observed.install(py, (args, self.project_dir.bind(py)), &terminal_index) + { + let (raised, _) = observed.exit_contexts(py, Some(error)); + observed.release(py); + return Err(raised + .unwrap_or_else(|| PyRuntimeError::new_err("observability scope setup failed"))); + } + *self.locked()? = Some(observed); + Ok(dispatcher.unbind()) + } + + #[pyo3(signature = (*exc_info))] + fn __exit__(&self, py: Python<'_>, exc_info: &Bound<'_, PyTuple>) -> PyResult { + let Some(mut observed) = self.locked()?.take() else { + return Ok(false); + }; + let value: Bound<'_, PyAny> = exc_info.get_item(1)?; + let pending: Option = if value.is_none() { + None + } else { + Some(PyErr::from_value(value)) + }; + let had_error: bool = pending.is_some(); + let (raised, suppressed) = observed.exit_contexts(py, pending); + observed.release(py); + match raised { + Some(error) if had_error => { + let original: Bound<'_, PyAny> = exc_info.get_item(1)?; + if error.value(py).is(&original) { + Ok(false) + } else { + Err(error) + } + } + Some(error) => Err(error), + None => Ok(suppressed), + } + } +} + +/// Install native progress, terminal indexing and configured event exporters for one command. +#[pyfunction] +#[pyo3(signature = (*, args, project_dir))] +pub(crate) fn cli_observability_scope( + args: Py, + project_dir: Py, +) -> CliObservabilityScope { + CliObservabilityScope { + args, + project_dir, + observed: Mutex::new(None), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(cli_observability_scope, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index a22b85a7c3..3bf2d20a93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -2,7 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; +use crate::bindings::_helpers::cli::{ + argv, entry, observability, progress, progress_reporters, progress_text, +}; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, check, clone, command_runtime, contract_command, cost_command, cost_report, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, @@ -14,6 +16,7 @@ use crate::bindings::_helpers::{ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { argv::register(module)?; entry::register(module)?; + observability::register(module)?; progress::register(module)?; progress_reporters::register(module)?; progress_text::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a9a84415a4..02c3bf0fef 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -351,6 +351,7 @@ from sqlbuild.rule_engine.models import ( ) from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.contracts.types import ExecutionResourceKind +from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( DbtConfig, @@ -1806,6 +1807,12 @@ def execute_query_diff( *, request: DiffCommandRequest, preparation: QueryDiffPreparation ) -> DiffExecutionResult: ... def machine_output_mode(args: CliNamespace, /) -> bool: ... + +class CliObservabilityScope: + def __enter__(self) -> EventDispatcher: ... + def __exit__(self, *exc_info: object) -> bool: ... + +def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> CliObservabilityScope: ... def current_native_progress_projector() -> NativeProgressProjector | None: ... class ConnectionProgressReporter: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py deleted file mode 100644 index 68d301eeb9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ /dev/null @@ -1,305 +0,0 @@ -"""CLI lifecycle dispatcher, native progress, and exporter scope.""" - -from __future__ import annotations - -import logging -import sys -from collections.abc import Callable, Iterator -from contextlib import ExitStack, contextmanager -from contextvars import Token -from functools import partial -from pathlib import Path - -from sqlbuild._native import ( - NativeProgressProjector, - TerminalEventIndex, - machine_output_mode, - supports_color, - terminal_event_index_scope, -) -from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace -from sqlbuild.cli.commands.constants import ( - DEBUG_COMMAND, - LINEAGE_COMMAND, -) -from sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture import ( - configured_output_capture_scope, -) -from sqlbuild.cli.entry.main._withhold_launch_record import withhold_launch_record -from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions -from sqlbuild.compiler.discovery.models import ( - DiscoveredCommandOutputSink, - DiscoveredEventExporter, - DiscoveredProvider, - DiscoveredRuntimeExtensions, -) -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope -from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher -from sqlbuild.runtime.event_exporting.constants import ( - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, - INVOCATION_TERMINAL_EVENT_TYPES, -) -from sqlbuild.runtime.event_exporting.main.event_exporter_command_scope import ( - event_exporter_command_scope, -) -from sqlbuild.runtime.event_exporting.models import ( - EventExporterAccounting, - EventExporterCounts, - EventExporterFailure, - EventExportSummary, -) -from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher -from sqlbuild.runtime.observability.main.current_execution_identity import ( - current_execution_identity, -) -from sqlbuild.runtime.observability.main.dispatcher_scope import dispatcher_scope -from sqlbuild.runtime.observability.models import ( - DispatchFailure, - ExecutionIdentity, - LifecycleEvent, -) -from sqlbuild.runtime.observability.types import Unsubscribe - -_LOGGER: logging.Logger = logging.getLogger("sqlbuild.cli.observability") -_INVOCATION_COMPLETED_EVENT_TYPE: str = "invocation_completed" -_SHUTDOWN_TIMEOUT_SETTING: str = "sinks.lifecycle.shutdown_timeout" - - -@contextmanager -def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterator[EventDispatcher]: - """Install native progress, terminal indexing, and configured event exporters.""" - - dispatcher: EventDispatcher = EventDispatcher(health_callback=_log_dispatch_failure) - terminal_index: TerminalEventIndex = TerminalEventIndex() - unsubscribe_terminal_index: Unsubscribe = dispatcher.subscribe_lifecycle( - subscriber=terminal_index.consume, - accepts_opaque=False, - ) - machine_output: bool = machine_output_mode(args) - projector: NativeProgressProjector = NativeProgressProjector( - stream=sys.stderr - if machine_output or args.debug or args.command == DEBUG_COMMAND - else sys.stdout, - use_color=not machine_output and not args.no_color and supports_color(), - debug=args.debug, - ) - unsubscribe_progress: Unsubscribe = dispatcher.subscribe_lifecycle( - subscriber=projector.consume, - accepts_opaque=False, - ) - projector_token: Token[NativeProgressProjector | None] = projector.install() - exporter_scope: EventExporterCommandScope | None = None - unsubscribe_exporters: Unsubscribe | None = None - unsubscribe_invocation_outcome: Unsubscribe | None = None - invocation_outcome: _InvocationOutcome = _InvocationOutcome() - final_export_summary: EventExportSummary | None = None - try: - extensions: DiscoveredRuntimeExtensions = ( - DiscoveredRuntimeExtensions() - if args.command == LINEAGE_COMMAND - else discover_runtime_extensions(project_dir=project_dir) - ) - providers: tuple[DiscoveredProvider, ...] = extensions.providers - event_exporters: tuple[DiscoveredEventExporter, ...] = extensions.event_exporters - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ( - extensions.command_output_sinks - ) - if event_exporters or command_output_sinks: - withhold_launch_record() - exporter_delivery: EventExporterDispatcher = EventExporterDispatcher( - shutdown_timeout_seconds=( - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS - if extensions.lifecycle_shutdown_timeout_seconds is None - else float(extensions.lifecycle_shutdown_timeout_seconds) - ), - failure_callback=_log_exporter_failure, - summary_callback=_log_exporter_summary, - ) - exporter_scope = EventExporterCommandScope(dispatcher=exporter_delivery) - exporter_scope.configure_extensions( - project_dir=project_dir, - providers=providers, - event_exporters=event_exporters, - command_output_sinks=command_output_sinks, - ) - unsubscribe_exporters = dispatcher.subscribe_lifecycle( - subscriber=exporter_delivery.enqueue, - accepts_opaque=False, - ) - unsubscribe_invocation_outcome = dispatcher.subscribe_lifecycle( - subscriber=invocation_outcome.consume, - accepts_opaque=False, - ) - with ExitStack() as stack: - _ = stack.enter_context(dispatcher_scope(dispatcher)) - _ = stack.enter_context(terminal_event_index_scope(terminal_index)) - if exporter_scope is not None: - _ = stack.enter_context(event_exporter_command_scope(exporter_scope)) - identity: ExecutionIdentity | None = current_execution_identity() - if identity is not None and command_output_sinks: - _ = stack.enter_context( - configured_output_capture_scope( - exporter_scope=exporter_scope, - identity=identity, - failure_callback=_log_command_output_failure, - ) - ) - yield dispatcher - finally: - if unsubscribe_exporters is not None: - _run_cleanup(action=unsubscribe_exporters, phase="event_exporter_unsubscribe") - if unsubscribe_invocation_outcome is not None: - _run_cleanup( - action=unsubscribe_invocation_outcome, phase="invocation_outcome_unsubscribe" - ) - if exporter_scope is not None: - final_export_summary = _close_exporter_scope(exporter_scope) - _run_cleanup(action=unsubscribe_terminal_index, phase="terminal_index_unsubscribe") - _run_cleanup(action=unsubscribe_progress, phase="progress_unsubscribe") - _run_cleanup(action=projector.close, phase="progress_close") - _run_cleanup( - action=lambda: projector.restore(projector_token), - phase="progress_context_restore", - ) - if final_export_summary is not None: - _run_cleanup( - action=partial( - _warn_incomplete_export, - summary=final_export_summary, - command_succeeded=invocation_outcome.succeeded, - ), - phase="event_exporter_warning", - ) - - -def format_event_export_warning( - *, summary: EventExportSummary, command_succeeded: bool -) -> str | None: - """Return one stderr warning line when lifecycle events were dropped or failed.""" - - if summary.dropped == 0 and summary.failed == 0: - return None - affected: tuple[EventExporterAccounting, ...] = tuple( - exporter - for exporter in summary.per_exporter - if exporter.counts.dropped > 0 or exporter.counts.failed > 0 - ) - names: str = ", ".join(f"'{exporter.exporter_name}'" for exporter in affected) - sink_label: str = "sink" if len(affected) == 1 else "sinks" - unit: str = "events" if len(summary.per_exporter) == 1 else "event deliveries" - prefix: str = ( - "Warning: command completed successfully, but lifecycle event export was incomplete" - if command_succeeded - else "Warning: lifecycle event export was incomplete" - ) - return ( - f"{prefix}: {summary.dropped} of {summary.accepted} {unit} dropped, " - f"{summary.failed} failed ({sink_label} {names}). " - f"Increase {_SHUTDOWN_TIMEOUT_SETTING} or check sink health." - ) - - -class _InvocationOutcome: - """Record whether the invocation published a successful terminal fact.""" - - def __init__(self) -> None: - self.succeeded: bool = False - - def consume(self, event: LifecycleEvent) -> None: - if event.event_type in INVOCATION_TERMINAL_EVENT_TYPES: - self.succeeded = event.event_type == _INVOCATION_COMPLETED_EVENT_TYPE - - -def _warn_incomplete_export(*, summary: EventExportSummary, command_succeeded: bool) -> None: - message: str | None = format_event_export_warning( - summary=summary, command_succeeded=command_succeeded - ) - if message is not None: - print(message, file=sys.stderr, flush=True) - - -def _close_exporter_scope(scope: EventExporterCommandScope) -> EventExportSummary | None: - try: - return scope.close() - except BaseException as error: - _report_cleanup_failure(error=error, phase="event_exporter_shutdown") - return None - - -def _run_cleanup(*, action: Callable[[], object], phase: str) -> None: - try: - _ = action() - except BaseException as error: - _report_cleanup_failure(error=error, phase=phase) - - -def _report_cleanup_failure(*, error: BaseException, phase: str) -> None: - try: - log_debug_event( - logger=_LOGGER, - message="CLI observability cleanup failed", - error_type=type(error).__name__, - phase=phase, - ) - except BaseException: - pass - - -def _log_dispatch_failure(failure: DispatchFailure) -> None: - log_debug_event( - logger=_LOGGER, - message="Lifecycle event subscriber failed", - error_type=failure.error_type, - channel=failure.channel, - subscriber=failure.subscriber, - ) - - -def _log_exporter_failure(failure: EventExporterFailure) -> None: - log_debug_event( - logger=_LOGGER, - message="Event exporter delivery failed", - exporter_name=failure.exporter_name, - error_type=failure.error_type, - event_kind=failure.event_kind, - event_severity=failure.event_severity, - ) - - -def _log_command_output_failure(error: BaseException) -> None: - log_debug_event( - logger=_LOGGER, - message="Command-output sink delivery failed", - error_type=type(error).__name__, - ) - - -def _log_exporter_summary(summary: EventExportSummary) -> None: - log_debug_event( - logger=_LOGGER, - message="Event exporter delivery summary", - accepted=summary.accepted, - filtered=summary.filtered, - delivered=summary.delivered, - failed=summary.failed, - dropped=summary.dropped, - queue_depth=summary.queue_depth, - queue_capacity=summary.queue_capacity, - flush_complete=summary.flush_complete, - ) - for exporter in summary.per_exporter: - counts: EventExporterCounts = exporter.counts - log_debug_event( - logger=_LOGGER, - message="Event exporter accounting", - exporter_name=exporter.exporter_name, - accepted=counts.accepted, - filtered=counts.filtered, - delivered=counts.delivered, - failed=counts.failed, - dropped=counts.dropped, - queue_depth=summary.queue_depth, - queue_capacity=summary.queue_capacity, - flush_complete=summary.flush_complete, - ) diff --git a/src/sqlbuild/runtime/event_exporting/main/event_exporter_command_scope.py b/src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py similarity index 100% rename from src/sqlbuild/runtime/event_exporting/main/event_exporter_command_scope.py rename to src/sqlbuild/runtime/event_exporting/main/_event_exporter_command_scope.py diff --git a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py index 2176a57cb3..dd1ac19762 100644 --- a/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py +++ b/tests/integration/src/sqlbuild/runtime/execution_history/test_cli_concurrent_history.py @@ -12,7 +12,7 @@ import pytest -from sqlbuild.cli.commands._helpers.entry.observability import cli_observability_scope +from sqlbuild._native import cli_observability_scope from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.execution_history import ( CanonicalLifecycleEvent, diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 36e7280746..57fb768f0a 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -112,6 +112,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli", "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", + "sqlbuild.cli.observability", "sqlbuild.compile", "sqlbuild.executor.runtime", "sqlbuild.fingerprints", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py index 65c5d922dc..e5edd3e8ac 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py @@ -37,14 +37,6 @@ class QueryDiffParsingTestCase: expected_unkeyed: bool -@dataclass(frozen=True) -class EventExportWarningTestCase: - description: str - exporter_counts: tuple[tuple[str, int, int, int, int], ...] - command_succeeded: bool - expected_message: str | None - - @dataclass(frozen=True) class GlobalFlagPlacementTestCase: """One placement of the flags every command accepts.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py deleted file mode 100644 index 422af92b1a..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Local helpers for CLI entry tests.""" - -from __future__ import annotations - -from sqlbuild.runtime.event_exporting.models import ( - EventExporterAccounting, - EventExporterCounts, - EventExportSummary, -) - - -def build_export_summary(counts: tuple[tuple[str, int, int, int, int], ...]) -> EventExportSummary: - per_exporter: tuple[EventExporterAccounting, ...] = tuple( - EventExporterAccounting( - exporter_name=name, - counts=EventExporterCounts( - accepted=accepted, delivered=delivered, failed=failed, dropped=dropped - ), - ) - for name, accepted, delivered, failed, dropped in counts - ) - return EventExportSummary( - aggregate=EventExporterCounts( - accepted=sum(item.counts.accepted for item in per_exporter), - delivered=sum(item.counts.delivered for item in per_exporter), - failed=sum(item.counts.failed for item in per_exporter), - dropped=sum(item.counts.dropped for item in per_exporter), - ), - per_exporter=per_exporter, - queue_depth=0, - queue_capacity=1024, - flush_complete=all(item.counts.dropped == 0 for item in per_exporter), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py deleted file mode 100644 index 2a445718e5..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Incomplete lifecycle-event export warning formatting.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.entry.observability import ( - format_event_export_warning, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( - EventExportWarningTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.entry.helpers import build_export_summary - - -@pytest.mark.parametrize( - "test_case", - ( - EventExportWarningTestCase( - "all delivered", - (("orders_events", 248, 248, 0, 0),), - True, - None, - ), - EventExportWarningTestCase( - "successful command with drops", - (("orders_events", 248, 108, 0, 140),), - True, - "Warning: command completed successfully, but lifecycle event export was " - "incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). " - "Increase sinks.lifecycle.shutdown_timeout or check sink health.", - ), - EventExportWarningTestCase( - "failed command with failures", - (("orders_events", 10, 9, 1, 0),), - False, - "Warning: lifecycle event export was incomplete: 0 of 10 events dropped, " - "1 failed (sink 'orders_events'). " - "Increase sinks.lifecycle.shutdown_timeout or check sink health.", - ), - EventExportWarningTestCase( - "multiple sinks names only affected sinks", - ( - ("customer_events", 20, 20, 0, 0), - ("orders_events", 20, 15, 1, 4), - ), - True, - "Warning: command completed successfully, but lifecycle event export was " - "incomplete: 4 of 40 event deliveries dropped, 1 failed (sink 'orders_events'). " - "Increase sinks.lifecycle.shutdown_timeout or check sink health.", - ), - ), - ids=lambda case: case.description, -) -def test_given_final_export_summary_when_formatting_warning_then_reports_loss_only( - test_case: EventExportWarningTestCase, -) -> None: - message: str | None = format_event_export_warning( - summary=build_export_summary(test_case.exporter_counts), - command_succeeded=test_case.command_succeeded, - ) - - assert message == test_case.expected_message - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index f5a4a72bee..e96da9a59c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any, Protocol, cast -from sqlbuild._native import NativeProgressProjector, loader_destination_parts +from sqlbuild._native import loader_destination_parts from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, ) @@ -88,16 +88,3 @@ def build_handlers(**overrides: Any) -> CliEntrypointHandlers: run_rules=overrides.get("run_rules", noop_handler), run_scope=overrides.get("run_scope", noop_handler), ) - - -class CloseFailingProjector: - """Delegate to a native progress projector whose close always fails.""" - - def __init__(self, **options: Any) -> None: - self._projector: NativeProgressProjector = NativeProgressProjector(**options) - - def __getattr__(self, name: str) -> object: - return getattr(self._projector, name) - - def close(self) -> None: - raise OSError("controlled projector close failure") diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py index 30e64d5637..10a3861cee 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_observability_cleanup.py @@ -5,16 +5,15 @@ from contextvars import Token from io import StringIO from pathlib import Path -from unittest.mock import patch import pytest from _pytest.capture import CaptureResult from sqlbuild._native import ( NativeProgressProjector, + cli_observability_scope, current_native_progress_projector, ) -from sqlbuild.cli.commands._helpers.entry.observability import cli_observability_scope from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.output.constants import INTEGRATION_RESULT_PATH_ENV from sqlbuild.observability import OperationLifecycle @@ -23,24 +22,19 @@ NoExporterFastPathTestCase, ObservabilityCleanupCase, ) -from tests.unit.src.sqlbuild.cli.commands.main.entry.helpers import CloseFailingProjector - -_PROJECTOR_FACTORY: str = ( - "sqlbuild.cli.commands._helpers.entry.observability.NativeProgressProjector" -) @pytest.mark.parametrize( "test_case", ( ObservabilityCleanupCase( - description="projector close failure preserves original exception and cleanup", + description="command failure propagates and restores the outer projector", expected_original_error="original command failure", ), ), ids=lambda case: case.description, ) -def test_given_projector_close_failure_when_command_fails_then_original_error_and_cleanup_survive( +def test_given_command_failure_when_scope_exits_then_original_error_and_projector_survive( tmp_path: Path, test_case: ObservabilityCleanupCase, ) -> None: @@ -48,10 +42,7 @@ def test_given_projector_close_failure_when_command_fails_then_original_error_an outer: NativeProgressProjector = NativeProgressProjector(stream=StringIO(), use_color=False) outer_token: Token[NativeProgressProjector | None] = outer.install() - with ( - patch(_PROJECTOR_FACTORY, CloseFailingProjector), - pytest.raises(ValueError, match=test_case.expected_original_error), - ): + with pytest.raises(ValueError, match=test_case.expected_original_error): with cli_observability_scope(args=args, project_dir=tmp_path): assert current_native_progress_projector() is not outer raise ValueError(test_case.expected_original_error) @@ -217,27 +208,3 @@ def test_given_integration_result_file_when_operation_runs_then_human_progress_u captured: CaptureResult[str] = capsys.readouterr() assert test_case.expected_stdout_fragment in captured.out assert captured.err == test_case.expected_stderr_fragment - - -@pytest.mark.parametrize( - "test_case", - ( - ObservabilityCleanupCase( - description="projector close failure preserves successful scope result", - expected_original_error="", - ), - ), - ids=lambda case: case.description, -) -def test_given_projector_close_failure_when_scope_succeeds_then_cleanup_does_not_raise( - tmp_path: Path, - test_case: ObservabilityCleanupCase, -) -> None: - args: CliNamespace = CliNamespace() - - with patch(_PROJECTOR_FACTORY, CloseFailingProjector): - with cli_observability_scope(args=args, project_dir=tmp_path): - result: str = "command result" - - assert result == "command result" - assert test_case.expected_original_error == "" From e83d6773f917ebc460696590fd68239d328cecb0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:29:18 +0100 Subject: [PATCH 20/32] style: sort test imports --- .../cli/commands/_helpers/runtime/test_warehouses.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index 870af56275..4c92b0b0f3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -6,8 +6,11 @@ import pytest -from sqlbuild._native import describe_connection_warehouse, resolve_project_connection_config -from sqlbuild._native import parse_cli_invocation +from sqlbuild._native import ( + describe_connection_warehouse, + parse_cli_invocation, + resolve_project_connection_config, +) from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError From 53a3f044ca558ce6a8ea7d7027c81c7a2a8babec Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:45:30 +0100 Subject: [PATCH 21/32] refactor: resolve CLI adapters, warehouse scopes and command invocations natively --- .../_helpers/audit_command/command.rs | 15 +- .../_helpers/build_command/invocation.rs | 10 +- .../bindings/_helpers/check/check_command.rs | 15 +- .../src/bindings/_helpers/cli/dispatch.rs | 19 ++- .../bindings/_helpers/clone/clone_command.rs | 10 +- .../_helpers/command_runtime/adapters.rs | 70 ++++++++ .../_helpers/command_runtime/invocation.rs | 155 ++++++++++++++++++ .../bindings/_helpers/command_runtime/mod.rs | 3 + .../_helpers/command_runtime/registration.rs | 6 +- .../command_runtime/warehouse_scope.rs | 117 +++++++++++++ .../_helpers/command_runtime/warehouses.rs | 5 +- .../_helpers/compile_driver/analysis.rs | 10 +- .../_helpers/contract_command/command.rs | 19 +-- .../bindings/_helpers/cost_command/refresh.rs | 10 +- .../_helpers/debug_command/run_debug.rs | 22 +-- .../bindings/_helpers/diff_report/query.rs | 7 +- .../_helpers/format_command/selection.rs | 10 +- .../_helpers/freshness/freshness_command.rs | 13 +- .../_helpers/janitor/janitor_phases.rs | 28 +--- .../_helpers/lineage/lineage_command.rs | 13 +- .../bindings/_helpers/load/load_command.rs | 10 +- .../bindings/_helpers/plan_command/plan.rs | 22 +-- .../bindings/_helpers/plan_command/rules.rs | 10 +- .../_helpers/query_command/project.rs | 10 +- .../_helpers/scenario_command/setup.rs | 10 +- .../bindings/_helpers/seed_command/command.rs | 14 +- .../_helpers/sql_test_command/test_command.rs | 19 +-- src/sqlbuild/_native.pyi | 16 +- .../_helpers/runtime/adapter_context.py | 43 ----- .../cli/commands/_helpers/runtime/adapters.py | 21 --- .../_helpers/runtime/command_invocation.py | 125 -------------- .../_helpers/runtime/warehouse_scope.py | 43 ----- .../cli/commands/_helpers/runtime/__init__.py | 1 - .../commands/_helpers/runtime/_test_types.py | 20 --- .../runtime/test_command_invocation.py | 103 ------------ .../commands/_helpers/cost/test_refresh.py | 6 +- .../commands/_helpers/debug/test_checks.py | 3 +- .../_helpers/runtime/test_warehouses.py | 2 +- .../commands/main/compile/test_adapters.py | 3 +- .../main/compile/test_compile_command.py | 2 +- .../cli/commands/main/dag/test_dag_command.py | 2 +- .../cli/commands/main/execution/helpers.py | 2 +- .../cli/commands/main/scope/test_scope.py | 6 +- .../main/run_rules/test_run_rules.py | 2 +- 44 files changed, 463 insertions(+), 589 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/runtime/adapters.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py delete mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py delete mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py delete mode 100644 tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index a438a7c984..67cdf32b3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -11,11 +11,11 @@ use sqlbuild_core::audit_command::main::render_audit_summary::render_audit_summa use sqlbuild_core::audit_command::models::AuditCounts; use sqlbuild_core::cli_text::models::CliStyle; +use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -482,15 +482,10 @@ fn complete( #[pyfunction] #[pyo3(signature = (*, request))] fn run_audit_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "AuditInvocation")?, - ), - ], + let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( + request, + &imported(py, COMMAND_MODELS, "AuditInvocation")?, + true, )?; let pipeline: Bound<'_, PyAny> = compile_audits(request, &invocation)?; let run: AuditRun<'_> = prepare_run(request, &invocation, &pipeline)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 50c7e75909..f50c97829f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -9,12 +9,12 @@ use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -147,13 +147,7 @@ pub(crate) fn resolve_build_context<'py>( ], )? .extract()?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", text(py, &adapter_name)), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&text(py, &adapter_name), &project_dir)?; let limits: Bound<'py, PyAny> = execution_limits(&discovered, &target_name, (&adapter_name, &adapter))?; let connection_config: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 4c27aa8493..09b1c82ae6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -15,6 +15,7 @@ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; +use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; use crate::bindings::_helpers::graph::python_selection::python_node_names; @@ -22,7 +23,6 @@ use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; @@ -681,15 +681,10 @@ fn complete<'py>( #[pyfunction] #[pyo3(signature = (*, request))] fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "CheckInvocation")?, - ), - ], + let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( + request, + &imported(py, COMMAND_MODELS, "CheckInvocation")?, + true, )?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; stream.call_method1("write", ("\n",))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 90df2f52a4..b20ec28c02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; +use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::diff_report::support::cli_error; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; @@ -21,7 +22,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; -const WAREHOUSE_SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; @@ -354,14 +354,15 @@ pub(crate) fn dispatch_command<'py>( ], )?; } - let scope: Bound<'py, PyAny> = call_with( - &imported(py, WAREHOUSE_SCOPE_MODULE, "command_warehouse_scope")?, - &[ - ("command", args.getattr("command")?), - ("cli_warehouse", warehouse), - ("cli_vars", args.getattr("vars")?), - ], - )?; + let scope: Bound<'py, PyAny> = Bound::new( + py, + command_warehouse_scope( + &args.getattr("command")?, + &warehouse, + Some(&args.getattr("vars")?), + )?, + )? + .into_any(); within(&scope, |_| routed_command(args, handlers.as_ref())) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 20fdc524b5..0a41321cf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -23,6 +23,7 @@ use crate::bindings::_helpers::clone::clone_execution::{ }; use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -32,7 +33,6 @@ const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; @@ -177,13 +177,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; Ok(Invocation { project_dir, discovered, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs new file mode 100644 index 0000000000..c1ba684318 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs @@ -0,0 +1,70 @@ +//! Adapter resolution for CLI commands, with adapter configuration failures as `C601`. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const REGISTRY_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; +const ADAPTER_ERRORS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; +const CLI_ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ADAPTER_CONFIGURATION_CODE: &str = "C601"; + +/// `CliUserError(error.message, code="C601", help=error.help)` raised from `error`. +fn cli_adapter_error(py: Python<'_>, error: &PyErr) -> PyResult { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", ADAPTER_CONFIGURATION_CODE)?; + keywords.set_item("help", value.getattr("help")?)?; + let raised: PyErr = PyErr::from_value( + imported(py, CLI_ERRORS_MODULE, "CliUserError")? + .call((value.getattr("message")?,), Some(&keywords))?, + ); + raised.set_cause(py, Some(error.clone_ref(py))); + Ok(raised) +} + +/// The registered adapter for `adapter_name`, resolved through the adapter registry. +pub(crate) fn resolve_cli_adapter<'py>( + adapter_name: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = adapter_name.py(); + let resolved: PyResult> = call_with( + &imported(py, REGISTRY_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name.clone()), + ("project_dir", project_dir.clone()), + ], + ); + match resolved { + Err(error) + if error.is_instance( + py, + &imported(py, ADAPTER_ERRORS_MODULE, "AdapterUserError")?, + ) => + { + Err(cli_adapter_error(py, &error)?) + } + resolved => resolved, + } +} + +/// Resolve an adapter name from project config to an adapter instance. +#[pyfunction] +#[pyo3(signature = (*, adapter_name, project_dir = None))] +fn resolve_adapter<'py>( + adapter_name: &Bound<'py, PyAny>, + project_dir: Option>, +) -> PyResult> { + let project_dir: Bound<'py, PyAny> = + project_dir.unwrap_or_else(|| adapter_name.py().None().into_bound(adapter_name.py())); + resolve_cli_adapter(adapter_name, &project_dir) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resolve_adapter, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs new file mode 100644 index 0000000000..83936d55a5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -0,0 +1,155 @@ +//! Discovery, adapter, connection and output context for project commands that connect. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; + +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; + +/// The project directory, discovered inputs, adapter context, colour and progress stream. +type InvocationContext<'py> = ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + bool, + Bound<'py, PyAny>, +); + +fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +/// The effective adapter and project connection configuration as an `AdapterConnectionContext`. +pub(crate) fn resolve_adapter_connection_context<'py>( + discovered: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + (selected_target, cli_vars): (Bound<'py, PyAny>, Bound<'py, PyAny>), + require_connection: bool, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, project_dir)?; + let connection_config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("project_dir", project_dir.clone()), + ("selected_target", selected_target), + ("cli_vars", cli_vars), + ("require_connection", boolean(py, require_connection)), + ], + )?; + call_with( + &imported(py, COMMAND_MODELS, "AdapterConnectionContext")?, + &[ + ("adapter_name", adapter_name), + ("adapter", adapter), + ("connection_config", connection_config), + ], + ) +} + +fn invocation_context<'py>( + request: &Bound<'py, PyAny>, + require_connection: bool, +) -> PyResult> { + let py: Python<'py> = request.py(); + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = if requested.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( + &discovered, + &project_dir, + ( + request.getattr("selected_target")?, + request.getattr("cli_vars")?, + ), + require_connection, + )?; + let machine_output: bool = request.getattr("json_output")?.is_truthy()?; + let use_color: bool = + !request.getattr("no_color")?.is_truthy()? && !machine_output && stdout_supports_color(py)?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if machine_output { "stderr" } else { "stdout" })?; + Ok((project_dir, discovered, adapter_context, use_color, stream)) +} + +fn context_fields<'py>( + context: &InvocationContext<'py>, +) -> PyResult)>> { + let (project_dir, discovered, adapter_context, use_color, stream) = context; + let py: Python<'py> = project_dir.py(); + Ok(vec![ + ("effective_project_dir", project_dir.clone()), + ("discovered_inputs", discovered.clone()), + ("adapter_name", adapter_context.getattr("adapter_name")?), + ("adapter", adapter_context.getattr("adapter")?), + ( + "connection_config", + adapter_context.getattr("connection_config")?, + ), + ("use_color", boolean(py, *use_color)), + ("progress_stream", stream.clone()), + ]) +} + +/// `invocation_type(...)` with discovery, adapter, connection and output context. +pub(crate) fn resolve_command_invocation<'py>( + request: &Bound<'py, PyAny>, + invocation_type: &Bound<'py, PyAny>, +) -> PyResult> { + let context: InvocationContext<'py> = invocation_context(request, true)?; + call_with(invocation_type, &context_fields(&context)?) +} + +/// `invocation_type(...)` with command context plus connection and planning progress reporters. +pub(crate) fn resolve_reported_command_invocation<'py>( + request: &Bound<'py, PyAny>, + invocation_type: &Bound<'py, PyAny>, + require_connection: bool, +) -> PyResult> { + let py: Python<'py> = request.py(); + let context: InvocationContext<'py> = invocation_context(request, require_connection)?; + let (_, _, adapter_context, use_color, stream) = &context; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = context_fields(&context)?; + fields.push(( + "connection_progress", + call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", adapter_context.getattr("adapter_name")?), + ("stream", stream.clone()), + ("use_color", boolean(py, *use_color)), + ], + )?, + )); + fields.push(( + "planning_progress", + call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", boolean(py, *use_color)), + ], + )?, + )); + call_with(invocation_type, &fields) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index df3675c30b..2d06334d13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,5 +1,8 @@ +pub(crate) mod adapters; pub(crate) mod connection; +pub(crate) mod invocation; pub(crate) mod preview_target; pub(crate) mod registration; pub(crate) mod runtime_errors; +pub(crate) mod warehouse_scope; pub(crate) mod warehouses; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs index 9aa9b25ee4..c6d17197ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/registration.rs @@ -2,9 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::command_runtime::{connection, preview_target, warehouses}; +use crate::bindings::_helpers::command_runtime::{ + adapters, connection, preview_target, warehouse_scope, warehouses, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + adapters::register(module)?; + warehouse_scope::register(module)?; warehouses::register(module)?; connection::register(module)?; preview_target::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs new file mode 100644 index 0000000000..ae3c52c241 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -0,0 +1,117 @@ +//! The active command's warehouse group, `--warehouse` override and CLI vars. + +use std::sync::Mutex; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const COMMAND_CONSTANTS: &str = "sqlbuild.cli.commands.constants"; +const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; +const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; +const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; + +fn context_variable(py: Python<'_>) -> PyResult> { + py.import(NATIVE_MODULE)?.getattr(CONTEXT_ATTRIBUTE) +} + +/// Installs one command's warehouse scope for the duration of a `with` block. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CommandWarehouseScopeInstall { + scope: Py, + token: Mutex>>, +} + +#[pymethods] +impl CommandWarehouseScopeInstall { + fn __enter__(&self, py: Python<'_>) -> PyResult> { + let token: Bound<'_, PyAny> = context_variable(py)?.call_method1("set", (&self.scope,))?; + *self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("warehouse scope lock is poisoned"))? = + Some(token.unbind()); + Ok(self.scope.clone_ref(py)) + } + + #[pyo3(signature = (*_exc_info))] + fn __exit__(&self, py: Python<'_>, _exc_info: &Bound<'_, PyTuple>) -> PyResult { + let token: Option> = self + .token + .lock() + .map_err(|_| PyRuntimeError::new_err("warehouse scope lock is poisoned"))? + .take(); + if let Some(token) = token { + context_variable(py)?.call_method1("reset", (token,))?; + } + Ok(false) + } +} + +/// Install the warehouse group, CLI override, and CLI vars for one command invocation. +#[pyfunction] +#[pyo3(signature = (*, command, cli_warehouse, cli_vars = None))] +pub(crate) fn command_warehouse_scope<'py>( + command: &Bound<'py, PyAny>, + cli_warehouse: &Bound<'py, PyAny>, + cli_vars: Option<&Bound<'py, PyAny>>, +) -> PyResult { + let py: Python<'py> = command.py(); + let group: Bound<'py, PyAny> = if command.is_none() { + py.None().into_bound(py) + } else { + imported(py, COMMAND_CONSTANTS, "COMMAND_WAREHOUSE_GROUPS")?.call_method1( + "get", + (imported(py, COMMAND_TYPES, "CliCommand")?.call1((command,))?,), + )? + }; + let vars: Bound<'py, PyAny> = match cli_vars.filter(|cli_vars| !cli_vars.is_none()) { + Some(cli_vars) => py.get_type::().call1((cli_vars,))?, + None => PyDict::new(py).into_any(), + }; + let scope: Bound<'py, PyAny> = call_with( + &imported(py, ENTRY_MODELS, "CommandWarehouseScope")?, + &[ + ("group", group), + ("cli_warehouse", cli_warehouse.clone()), + ("cli_vars", vars), + ], + )?; + Ok(CommandWarehouseScopeInstall { + scope: scope.unbind(), + token: Mutex::new(None), + }) +} + +/// Return the active command's warehouse scope, or an empty one outside the CLI. +#[pyfunction] +pub(crate) fn current_command_warehouse_scope(py: Python<'_>) -> PyResult> { + let scope: Bound<'_, PyAny> = context_variable(py)?.call_method0("get")?; + if !scope.is_none() { + return Ok(scope); + } + call_with( + &imported(py, ENTRY_MODELS, "CommandWarehouseScope")?, + &[("group", py.None().into_bound(py))], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("default", py.None())?; + module.add( + CONTEXT_ATTRIBUTE, + py.import("contextvars")? + .getattr("ContextVar")? + .call((CONTEXT_NAME,), Some(&keywords))?, + )?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(command_warehouse_scope, module)?)?; + module.add_function(wrap_pyfunction!(current_command_warehouse_scope, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index e02035283c..d9b469c8ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -21,6 +21,7 @@ use sqlbuild_model_config::templates::models::TemplateOptions; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, raised, }; +use crate::bindings::_helpers::command_runtime::warehouse_scope::current_command_warehouse_scope; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -28,7 +29,6 @@ use crate::bindings::_helpers::discovery::configuration::discover_project_config use crate::bindings::_helpers::model_config::template_data::expanded_template_data; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; -const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; @@ -86,8 +86,7 @@ fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { } pub(crate) fn command_scope(py: Python<'_>) -> PyResult> { - let scope: Bound<'_, PyAny> = - imported(py, SCOPE_MODULE, "current_command_warehouse_scope")?.call0()?; + let scope: Bound<'_, PyAny> = current_command_warehouse_scope(py)?; Ok(CommandScope { group: scope.getattr("group")?, cli_warehouse: optional_text(&scope.getattr("cli_warehouse")?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 632372481f..2deabc7365 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -6,6 +6,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; @@ -14,7 +15,6 @@ use crate::bindings::_helpers::rules::rules_config::load_rules_config; const DISCOVER_MODULE: &str = "sqlbuild._native"; const REJECT_SELECTORS_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; @@ -330,13 +330,7 @@ impl<'py> Driver<'_, 'py> { ("local_config", discovered.getattr("local_config")?), ], )?; - call_with( - &imported(self.py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", self.options.project_dir.clone()), - ], - ) + resolve_cli_adapter(&adapter_name, &self.options.project_dir) } /// Compile the project graph, starting the early SQL lint once inputs are ready. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index eb7a6439b0..78a94477a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -19,13 +19,13 @@ use sqlbuild_core::contract_command::models::{ }; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; -const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; @@ -397,18 +397,11 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< "C472", )); } - let context: Bound<'_, PyAny> = call_with( - &imported( - py, - ADAPTER_CONTEXT_MODULE, - "resolve_adapter_connection_context", - )?, - &[ - ("discovered_inputs", discovered.clone()), - ("effective_project_dir", project_dir.clone()), - ("selected_target", py.None().into_bound(py)), - ("cli_vars", request.getattr("cli_vars")?), - ], + let context: Bound<'_, PyAny> = resolve_adapter_connection_context( + &discovered, + &project_dir, + (py.None().into_bound(py), request.getattr("cli_vars")?), + true, )?; let selection: Selection<'_> = select_resources(request, &discovered, &context)?; let progress: Progress<'_> = progress_for(request)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 073bca79ba..cfee0e63c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -9,12 +9,12 @@ use sqlbuild_core::cost_command::constants::{ use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; use sqlbuild_core::cost_command::models::CostCoverage; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const COST_TYPES: &str = "sqlbuild.cost.types"; fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { @@ -120,13 +120,7 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'_, PyAny> = call_with( - &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; if !adapter.is_instance(&imported(py, COST_TYPES, "CostAwareAdapter")?)? { return Ok(()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index f1c48bf5dd..76a82c12f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -21,10 +21,12 @@ use sqlbuild_core::debug_report::models::{ DebugReport, DebugRuntime, WarehouseCheck, }; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::project_connection_config; use crate::bindings::_helpers::command_runtime::runtime_errors::{ expected_resolution_error, resolution_message, }; +use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; @@ -34,8 +36,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; -const SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; const DEBUG_OPTIONS: (&[&str], &[&str]) = ( @@ -272,13 +272,7 @@ fn build_report( ("local_config", discovered_inputs.getattr("local_config")?), ], )?; - let adapter: Bound<'_, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", project_dir_object.clone()), - ], - )?; + let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir_object)?; let context: DebugContext<'_> = DebugContext { project_dir: project_dir_object, discovered_inputs, @@ -286,13 +280,9 @@ fn build_report( adapter, }; let command_scope: CommandScope<'_> = command_scope(py)?; - let unscoped: Bound<'_, PyAny> = call_with( - &imported(py, SCOPE_MODULE, "command_warehouse_scope")?, - &[ - ("command", py.None().into_bound(py)), - ("cli_warehouse", py.None().into_bound(py)), - ], - )?; + let none: Bound<'_, PyAny> = py.None().into_bound(py); + let unscoped: Bound<'_, PyAny> = + Bound::new(py, command_warehouse_scope(&none, &none, None)?)?.into_any(); let base_connection: Bound<'_, PyAny> = within(&unscoped, |_| { project_connection_config( &context.discovered_inputs, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index ad318afc21..fb2d82986a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -12,6 +12,7 @@ use sqlbuild_cli::diff_report::main::read_only_query_problem::read_only_query_pr use sqlbuild_cli::diff_report::models::QueryShapeFacts; use sqlbuild_cli::diff_report::types::PhaseSeconds; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::diff_report::support::{ call, cli_error, error_text, none, object, progress_line, @@ -19,7 +20,6 @@ use crate::bindings::_helpers::diff_report::support::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const NAMESPACE_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODULE: &str = "sqlbuild._native"; const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; @@ -61,10 +61,7 @@ pub(crate) fn project_adapter<'py>( ("local_config", &discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[("adapter_name", &name), ("project_dir", project_dir)], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&name, project_dir)?; Ok((name.extract()?, adapter)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index facf69432e..e74a2b9a2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -14,6 +14,7 @@ use sqlbuild_core::format_command::main::resolve_lint_paths::resolve_lint_paths; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -23,7 +24,6 @@ const UNIT_TEST_SELECTORS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const SELECTION_CODE: &str = "S007"; /// The adapter, the selected files when a selector scopes the run, and the discovered inputs. @@ -83,13 +83,7 @@ fn discovered_adapter<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - call_with( - &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - ) + resolve_cli_adapter(&adapter_name, project_dir) } fn selectors<'py>(py: Python<'py>, values: &[String]) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 28a40c7088..a71149e344 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -14,6 +14,7 @@ use sqlbuild_core::freshness_text::main::freshness_json::freshness_json; use sqlbuild_core::freshness_text::main::render_freshness_text::render_freshness_text; use sqlbuild_core::freshness_text::models::FreshnessSourceResult; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::freshness::freshness_observe::{ Observer, PreviousState, observe_freshness, }; @@ -23,7 +24,6 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const CONNECTION_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; const UPSTREAM_MODULE: &str = "sqlbuild._native"; @@ -325,16 +325,7 @@ fn run_freshness_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult ], )?), )?; - let adapter: Bound<'_, PyAny> = imported(py, ADAPTERS_MODULE, "resolve_adapter")?.call( - (), - Some(&keywords( - py, - &[ - ("adapter_name", &adapter_name), - ("project_dir", &project_dir), - ], - )?), - )?; + let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; let graph: Bound<'_, PyAny> = imported(py, NATIVE_MODULE, "build_project_graph")?.call( (), Some(&keywords( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index eb305192e6..06deeaaf9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -6,10 +6,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const JANITOR_PLAN_MODULE: &str = "sqlbuild.executor.janitor.main.plan"; const JANITOR_EXECUTE_MODULE: &str = "sqlbuild.executor.janitor.main.execute"; @@ -54,24 +54,14 @@ pub(crate) fn compile_janitor_project<'py>( invocation: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = invocation.py(); - let context: Bound<'py, PyAny> = call_with( - &imported( - py, - ADAPTER_CONTEXT_MODULE, - "resolve_adapter_connection_context", - )?, - &[ - ( - "discovered_inputs", - invocation.getattr("discovered_inputs")?, - ), - ( - "effective_project_dir", - invocation.getattr("effective_project_dir")?, - ), - ("selected_target", invocation.getattr("selected_target")?), - ("cli_vars", py.None().into_bound(py)), - ], + let context: Bound<'py, PyAny> = resolve_adapter_connection_context( + &invocation.getattr("discovered_inputs")?, + &invocation.getattr("effective_project_dir")?, + ( + invocation.getattr("selected_target")?, + py.None().into_bound(py), + ), + true, )?; let started: Instant = Instant::now(); let status: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index a7d518e291..a8cfdc1f67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -32,6 +32,7 @@ use sqlbuild_analysis::lineage_command::models::{ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::build_progress::transient_status::maybe_status; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; @@ -40,7 +41,6 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; @@ -200,16 +200,7 @@ fn prepare_source<'py>( ], )?), )?; - let adapter: Bound<'py, PyAny> = imported(py, ADAPTERS_MODULE, "resolve_adapter")?.call( - (), - Some(&keywords( - py, - &[ - ("adapter_name", &adapter_name), - ("project_dir", project_dir), - ], - )?), - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, project_dir)?; let no_validation: bool = request.getattr("no_sql_validation")?.extract::()? || !compiled; let options: Bound<'py, PyDict> = keywords( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 45cbfc9b64..b477e79fbd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -16,6 +16,7 @@ use sqlbuild_core::load_command::main::render_load_plan::render_load_plan; use sqlbuild_core::load_command::main::render_load_ready::render_load_ready; use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::load::load_progress::{ @@ -33,7 +34,6 @@ const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_OVERRIDES_MODULE: &str = "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; @@ -284,13 +284,7 @@ fn prepare<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name.clone()), - ("project_dir", invocation.project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, &invocation.project_dir)?; let connection_config: Bound<'py, PyAny> = call_with( &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d49a517152..2f51f18c81 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -8,11 +8,11 @@ use pyo3::wrap_pyfunction; use crate::bindings::_helpers::build_command::support::keyword_relay; use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; +use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; @@ -76,18 +76,14 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult = call_with( - &imported( - py, - ADAPTER_CONTEXT_MODULE, - "resolve_adapter_connection_context", - )?, - &[ - ("discovered_inputs", discovered.clone()), - ("effective_project_dir", project_dir.clone()), - ("selected_target", request.getattr("selected_target")?), - ("cli_vars", request.getattr("cli_vars")?), - ], + let context: Bound<'py, PyAny> = resolve_adapter_connection_context( + &discovered, + &project_dir, + ( + request.getattr("selected_target")?, + request.getattr("cli_vars")?, + ), + true, )?; let json_output: bool = request.getattr("json_output")?.is_truthy()?; let use_color: bool = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index bc90ce8511..bfb27544ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -6,10 +6,10 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::wrap_pyfunction; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RULE_MODELS: &str = "sqlbuild.rule_engine.models"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; @@ -176,13 +176,7 @@ fn rule_inputs<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, project_dir)?; let graph: Bound<'py, PyAny> = call_with( &imported(py, NATIVE_MODULE, "build_project_graph")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 01c92d2a40..2368f1e8c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -2,11 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; -const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; /// The project directory, defaulting to the working directory, with its inputs and adapter. pub(crate) struct ProjectAdapter<'py> { @@ -34,13 +34,7 @@ pub(crate) fn project_adapter<'py>( ("local_config", discovered.getattr("local_config")?), ], )?; - let adapter: Bound<'py, PyAny> = call_with( - &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; + let adapter: Bound<'py, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; Ok(ProjectAdapter { project_dir, discovered, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index 18831a2e29..5395052fe9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -20,6 +20,7 @@ use sqlbuild_core::scenario_command::models::{ SnapshotLimitInputs, SnapshotLimits, SqlAnalysisCommand, }; +use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -29,7 +30,6 @@ const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; -const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; @@ -208,13 +208,7 @@ pub(crate) fn resolve_adapter<'py>( adapter_name: &str, ) -> PyResult> { let py: Python<'py> = project.discovered.py(); - call_with( - &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, - &[ - ("adapter_name", text(py, adapter_name)), - ("project_dir", project.project_dir.clone()), - ], - ) + resolve_cli_adapter(&text(py, adapter_name), &project.project_dir) } /// The adapter's SQL analysis dialect, which snapshots require. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 02231a9701..375a2c3dad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -14,11 +14,11 @@ use sqlbuild_core::seed_command::main::render_seed_row::render_seed_row; use sqlbuild_core::seed_command::main::seed_exit_code::seed_exit_code; use sqlbuild_core::seed_command::models::SeedRow; +use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -421,16 +421,8 @@ fn load_seeds<'py>( #[pyfunction] #[pyo3(signature = (*, request))] fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "SeedInvocation")?, - ), - ], - )?; + let invocation: Bound<'_, PyAny> = + resolve_command_invocation(request, &imported(py, COMMAND_MODELS, "SeedInvocation")?)?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; let concurrency: i64 = if requested.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 74c0708ad9..9baf8c6a1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -24,6 +24,7 @@ use sqlbuild_core::test_command::models::{ }; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -32,7 +33,6 @@ use crate::bindings::_helpers::result_output::integration_values::json_value; const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; -const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -634,19 +634,10 @@ fn complete( #[pyo3(signature = (*, request))] fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; - let invocation: Bound<'_, PyAny> = call_with( - &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, - &[ - ("request", request.clone()), - ( - "invocation_type", - imported(py, COMMAND_MODELS, "TestInvocation")?, - ), - ( - "require_connection", - PyBool::new(py, !inspect_only).to_owned().into_any(), - ), - ], + let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( + request, + &imported(py, COMMAND_MODELS, "TestInvocation")?, + !inspect_only, )?; let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 02c3bf0fef..81ec2e423b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -49,12 +49,13 @@ from sqlbuild.cli.commands.models import ( SeedCommandRequest, TestCommandRequest, ) -from sqlbuild.cli.commands.types import CompileLineageMode, ScopeIndexLoader +from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, CompileCommandRequest, + CommandWarehouseScope, CompileProfileFlags, ParsedCliInvocation, ResolvedWarehouse, @@ -2688,6 +2689,19 @@ def resolve_target_name( def resolve_target_config( *, project_config: ProjectConfig, local_config: LocalConfig, target_name: str ) -> TargetConfig: ... +def resolve_adapter(*, adapter_name: str, project_dir: Path | None = None) -> BaseAdapter: ... + +class CommandWarehouseScopeInstall: + def __enter__(self) -> CommandWarehouseScope: ... + def __exit__(self, *exc_info: object) -> bool: ... + +def command_warehouse_scope( + *, + command: CliCommand | str | None, + cli_warehouse: str | None, + cli_vars: Mapping[str, object] | None = None, +) -> CommandWarehouseScopeInstall: ... +def current_command_warehouse_scope() -> CommandWarehouseScope: ... def resolve_effective_adapter_name( *, project_config: ProjectConfig, local_config: LocalConfig ) -> str: ... diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py deleted file mode 100644 index 155eaae005..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapter_context.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Adapter and connection-context resolution for CLI commands.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import resolve_effective_adapter_name, resolve_project_connection_config -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.models import AdapterConnectionContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def resolve_adapter_connection_context( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_project_dir: Path, - selected_target: str | None, - cli_vars: dict[str, object] | None, - require_connection: bool = True, -) -> AdapterConnectionContext: - """Resolve the effective adapter and project connection configuration.""" - - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, - project_dir=effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - selected_target=selected_target, - cli_vars=cli_vars, - require_connection=require_connection, - ) - return AdapterConnectionContext( - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/adapters.py b/src/sqlbuild/cli/commands/_helpers/runtime/adapters.py deleted file mode 100644 index 24c2b6be1f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/adapters.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Adapter resolution from project configuration.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.discovery.main.resolve_adapter import ( - resolve_adapter as resolve_registered_adapter, -) -from sqlbuild.cli.commands.exceptions import CliUserError - - -def resolve_adapter(*, adapter_name: str, project_dir: Path | None = None) -> BaseAdapter: - """Resolve an adapter name from project config to an adapter instance.""" - - try: - return resolve_registered_adapter(adapter_name=adapter_name, project_dir=project_dir) - except AdapterUserError as error: - raise CliUserError(error.message, code="C601", help=error.help) from error diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py b/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py deleted file mode 100644 index b9af715ee9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/command_invocation.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Shared invocation resolution for project commands that connect to one target.""" - -from __future__ import annotations - -import sys -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path -from typing import Protocol, TextIO - -from sqlbuild._native import ( - ConnectionProgressReporter, - PlanningProgressReporter, - discover_project_inputs, - supports_color, -) -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.models import AdapterConnectionContext -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -class _CommandInvocationRequest(Protocol): - """CLI request fields needed to resolve a project command invocation.""" - - @property - def project_dir(self) -> Path | None: ... - - @property - def selected_target(self) -> str | None: ... - - @property - def cli_vars(self) -> dict[str, object] | None: ... - - @property - def json_output(self) -> bool: ... - - @property - def no_color(self) -> bool: ... - - -@dataclass(frozen=True) -class _CommandInvocationContext: - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_context: AdapterConnectionContext - use_color: bool - progress_stream: TextIO - - -def resolve_command_invocation[InvocationT]( - *, request: _CommandInvocationRequest, invocation_type: Callable[..., InvocationT] -) -> InvocationT: - """Resolve discovery, adapter, connection, and output context for one command.""" - - context: _CommandInvocationContext = _resolve_command_invocation_context( - request=request, require_connection=True - ) - return invocation_type( - effective_project_dir=context.effective_project_dir, - discovered_inputs=context.discovered_inputs, - adapter_name=context.adapter_context.adapter_name, - adapter=context.adapter_context.adapter, - connection_config=context.adapter_context.connection_config, - use_color=context.use_color, - progress_stream=context.progress_stream, - ) - - -def resolve_reported_command_invocation[InvocationT]( - *, - request: _CommandInvocationRequest, - invocation_type: Callable[..., InvocationT], - require_connection: bool = True, -) -> InvocationT: - """Resolve command context plus shared connection and planning progress reporters.""" - - context: _CommandInvocationContext = _resolve_command_invocation_context( - request=request, require_connection=require_connection - ) - return invocation_type( - effective_project_dir=context.effective_project_dir, - discovered_inputs=context.discovered_inputs, - adapter_name=context.adapter_context.adapter_name, - adapter=context.adapter_context.adapter, - connection_config=context.adapter_context.connection_config, - use_color=context.use_color, - progress_stream=context.progress_stream, - connection_progress=ConnectionProgressReporter( - adapter_name=context.adapter_context.adapter_name, - stream=context.progress_stream, - use_color=context.use_color, - ), - planning_progress=PlanningProgressReporter( - stream=context.progress_stream, use_color=context.use_color - ), - ) - - -def _resolve_command_invocation_context( - *, request: _CommandInvocationRequest, require_connection: bool -) -> _CommandInvocationContext: - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=discovered_inputs, - effective_project_dir=effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - require_connection=require_connection, - ) - machine_output: bool = request.json_output - use_color: bool = not request.no_color and not machine_output and supports_color() - return _CommandInvocationContext( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter_context=adapter_context, - use_color=use_color, - progress_stream=sys.stderr if machine_output else sys.stdout, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py b/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py deleted file mode 100644 index a068f97c5f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Active command warehouse scope for CLI warehouse connections.""" - -from __future__ import annotations - -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token - -from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS -from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.cli.entry.models import CommandWarehouseScope - -_CURRENT_SCOPE: ContextVar[CommandWarehouseScope | None] = ContextVar( - "sqlbuild_command_warehouse_scope", default=None -) - - -@contextmanager -def command_warehouse_scope( - *, - command: CliCommand | str | None, - cli_warehouse: str | None, - cli_vars: dict[str, object] | None = None, -) -> Iterator[CommandWarehouseScope]: - """Install the warehouse group, CLI override, and CLI vars for one command invocation.""" - - scope: CommandWarehouseScope = CommandWarehouseScope( - group=None if command is None else COMMAND_WAREHOUSE_GROUPS.get(CliCommand(command)), - cli_warehouse=cli_warehouse, - cli_vars={} if cli_vars is None else dict(cli_vars), - ) - token: Token[CommandWarehouseScope | None] = _CURRENT_SCOPE.set(scope) - try: - yield scope - finally: - _CURRENT_SCOPE.reset(token) - - -def current_command_warehouse_scope() -> CommandWarehouseScope: - """Return the active command's warehouse scope, or an empty one outside the CLI.""" - - scope: CommandWarehouseScope | None = _CURRENT_SCOPE.get() - return scope if scope is not None else CommandWarehouseScope(group=None) diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py deleted file mode 100644 index e0de81c887..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Integration tests for shared CLI command runtime helpers.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py deleted file mode 100644 index 8d5e37e5b1..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ /dev/null @@ -1,20 +0,0 @@ -from collections.abc import Callable -from dataclasses import dataclass - -from sqlbuild.cli.commands.models import ( - AuditInvocation, - CheckInvocation, - SeedInvocation, - TestInvocation, -) - - -@dataclass(frozen=True) -class CommandInvocationTestCase: - description: str - resolve: Callable[..., AuditInvocation | CheckInvocation | SeedInvocation | TestInvocation] - request_type: Callable[..., object] - json_output: bool - expected_invocation_type_name: str - expected_stream_name: str - expected_has_progress_reporters: bool diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py deleted file mode 100644 index f2845910f7..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Integration tests for resolving CLI command invocations against a real DuckDB project.""" - -from __future__ import annotations - -import sys -from functools import partial -from pathlib import Path - -import pytest - -from sqlbuild.cli.commands import models as command_models -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_command_invocation, - resolve_reported_command_invocation, -) -from tests.integration.src.sqlbuild.cli.commands._helpers.runtime._test_types import ( - CommandInvocationTestCase, -) - -_PROJECT_TOML: str = ( - 'name = "orders_shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' -) - - -@pytest.mark.parametrize( - "test_case", - ( - CommandInvocationTestCase( - description="audit json output reports progress on stderr", - resolve=partial( - resolve_reported_command_invocation, - invocation_type=command_models.AuditInvocation, - ), - request_type=command_models.AuditCommandRequest, - json_output=True, - expected_invocation_type_name="AuditInvocation", - expected_stream_name="stderr", - expected_has_progress_reporters=True, - ), - CommandInvocationTestCase( - description="check text output reports progress on stdout", - resolve=partial( - resolve_reported_command_invocation, - invocation_type=command_models.CheckInvocation, - ), - request_type=command_models.CheckCommandRequest, - json_output=False, - expected_invocation_type_name="CheckInvocation", - expected_stream_name="stdout", - expected_has_progress_reporters=True, - ), - CommandInvocationTestCase( - description="test json output reports progress on stderr", - resolve=partial( - resolve_reported_command_invocation, - invocation_type=command_models.TestInvocation, - require_connection=True, - ), - request_type=command_models.TestCommandRequest, - json_output=True, - expected_invocation_type_name="TestInvocation", - expected_stream_name="stderr", - expected_has_progress_reporters=True, - ), - CommandInvocationTestCase( - description="seed text output omits shared progress reporters", - resolve=partial( - resolve_command_invocation, - invocation_type=command_models.SeedInvocation, - ), - request_type=command_models.SeedCommandRequest, - json_output=False, - expected_invocation_type_name="SeedInvocation", - expected_stream_name="stdout", - expected_has_progress_reporters=False, - ), - ), - ids=lambda case: case.description, -) -def test_given_duckdb_project_when_resolving_command_invocation_then_returns_command_context( - tmp_path: Path, - test_case: CommandInvocationTestCase, -) -> None: - _ = (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") - request: object = test_case.request_type( - project_dir=tmp_path, no_color=True, json_output=test_case.json_output - ) - - invocation: ( - command_models.AuditInvocation - | command_models.CheckInvocation - | command_models.SeedInvocation - | command_models.TestInvocation - ) = test_case.resolve(request=request) - - assert type(invocation).__name__ == test_case.expected_invocation_type_name - assert invocation.effective_project_dir == tmp_path - assert invocation.adapter_name == "duckdb" - assert invocation.discovered_inputs.project_config.name == "orders_shop" - assert invocation.use_color is False - assert invocation.progress_stream is getattr(sys, test_case.expected_stream_name) - assert hasattr(invocation, "connection_progress") is test_case.expected_has_progress_reporters - assert hasattr(invocation, "planning_progress") is test_case.expected_has_progress_reporters diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index f310432175..b7c2fc1f1d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -91,7 +91,7 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", lambda **kwargs: _RefreshingCostAdapter(), ) monkeypatch.setattr( @@ -195,7 +195,7 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", lambda **kwargs: _StaticCostAdapter(candidate), ) monkeypatch.setattr( @@ -278,7 +278,7 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", lambda **kwargs: _StaticCostAdapter(candidate), ) monkeypatch.setattr( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py index 42fbaaa224..3fe992d79a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import run_debug_command -from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope +from sqlbuild._native import command_warehouse_scope, run_debug_command from tests.unit.src.sqlbuild.cli.commands._helpers.debug._test_types import ( DebugWarehouseVarsTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index 4c92b0b0f3..ef9873a880 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -7,11 +7,11 @@ import pytest from sqlbuild._native import ( + command_warehouse_scope, describe_connection_warehouse, parse_cli_invocation, resolve_project_connection_config, ) -from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index 34f8c5f0c9..a1ca166e58 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import resolve_effective_adapter_name +from sqlbuild._native import resolve_adapter, resolve_effective_adapter_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 4b6351bb72..394f533461 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -11,7 +11,7 @@ import pytest import sqlbuild._native as discovery_entry -from sqlbuild.cli.commands._helpers.runtime import adapters as runtime_adapters +from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 6ee8bb077c..77823c58a4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -6,7 +6,7 @@ import pytest from sqlbuild._native import run_dag_command -from sqlbuild.cli.commands._helpers.runtime import adapters as adapter_resolution +from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_resolution from tests.unit.src.sqlbuild.cli.commands.main.dag._test_types import ( DagCommandTestCase, PythonDagCommandTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 431eb0306d..0d1820b0af 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -10,8 +10,8 @@ import pytest from sqlbuild import _native as native +from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module from sqlbuild.cli.commands._helpers.planning import external_refs as external_refs_module -from sqlbuild.cli.commands._helpers.runtime import adapters as adapters_module from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index ea6763184b..913badbca7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -7,8 +7,9 @@ import pytest +from sqlbuild import _native as native from sqlbuild._native import run_scope_command -from sqlbuild.cli.commands._helpers.runtime import adapter_context +from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_registry from sqlbuild.cli.commands.models import ScopeCommandRequest from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase @@ -25,7 +26,8 @@ def test_given_offline_index_loader_when_running_then_connection_runtime_is_not_ def fail(**_kwargs: object) -> None: pytest.fail("scope must not resolve adapters or warehouse credentials") - monkeypatch.setattr(adapter_context, "resolve_adapter_connection_context", fail) + monkeypatch.setattr(adapter_registry, "resolve_adapter", fail) + monkeypatch.setattr(native, "resolve_project_connection_config", fail) exit_code: int = run_scope_command( request=ScopeCommandRequest(target="model:orders", json_output=True), load_scope_index=lambda **_kwargs: report_scope_lookup().index, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py index d6f68e22ac..0a2994db2d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/run_rules/test_run_rules.py @@ -8,10 +8,10 @@ build_project_graph, discover_project_inputs, load_rules_config, + resolve_adapter, run_rules, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.rule_engine.models import RulesRunResult From 40593fc43d2a1dbeab8603f5cdbb00dfa63dc060 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:45:33 +0100 Subject: [PATCH 22/32] style: sort native stub imports --- src/sqlbuild/_native.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 81ec2e423b..5fc520b43f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -54,8 +54,8 @@ from sqlbuild.cli.compile.models import CompileAnalysis from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, - CompileCommandRequest, CommandWarehouseScope, + CompileCommandRequest, CompileProfileFlags, ParsedCliInvocation, ResolvedWarehouse, From 88d6986f43647dc1dde10d9bc99472e2622a0827 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:10:15 +0100 Subject: [PATCH 23/32] refactor: check, replay and record compile reuse natively --- .../sqlbuild-cli/src/replay/_helpers/files.rs | 61 +++ .../sqlbuild-cli/src/replay/_helpers/mod.rs | 1 + crates/sqlbuild-cli/src/replay/constants.rs | 2 + crates/sqlbuild-cli/src/replay/main/mod.rs | 1 + .../src/replay/main/write_launch_record.rs | 15 + crates/sqlbuild-cli/src/replay/tests.rs | 4 + .../sqlbuild-cli/src/replay/tests/helpers.rs | 32 ++ .../src/replay/tests/test_types.rs | 7 + .../replay/tests/test_write_launch_record.rs | 45 ++ .../bindings/_helpers/compile_command/mod.rs | 1 + .../_helpers/compile_command/registration.rs | 3 +- .../_helpers/compile_command/reuse_attempt.rs | 454 ++++++++++++++++++ .../compile_command/reuse_recording.rs | 13 +- .../_helpers/compile_outputs/json_report.rs | 13 + fensu.toml | 5 + src/sqlbuild/_native.pyi | 3 +- src/sqlbuild/cli/compile/main/run.py | 2 +- .../cli/compile_reuse/_helpers/attempt.py | 26 - .../compile_reuse/_helpers/launch_records.py | 118 ----- .../compile_reuse/_helpers/native_reuse.py | 162 ------- .../cli/compile_reuse/_helpers/timings.py | 31 -- .../cli/compile_reuse/main/__init__.py | 0 .../main/_attempt_compile_reuse.py | 12 - .../cli/commands/main/compile/helpers.py | 6 +- .../commands/main/compile_outputs/helpers.py | 7 +- 25 files changed, 656 insertions(+), 368 deletions(-) create mode 100644 crates/sqlbuild-cli/src/replay/_helpers/files.rs create mode 100644 crates/sqlbuild-cli/src/replay/main/write_launch_record.rs create mode 100644 crates/sqlbuild-cli/src/replay/tests/helpers.rs create mode 100644 crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs delete mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/attempt.py delete mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py delete mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py delete mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/timings.py delete mode 100644 src/sqlbuild/cli/compile_reuse/main/__init__.py delete mode 100644 src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py diff --git a/crates/sqlbuild-cli/src/replay/_helpers/files.rs b/crates/sqlbuild-cli/src/replay/_helpers/files.rs new file mode 100644 index 0000000000..2f3dca3d61 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/_helpers/files.rs @@ -0,0 +1,61 @@ +//! Atomic launch record writes and pruning of the oldest records. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::replay::constants::{LAUNCH_RECORD_LIMIT, LAUNCH_RECORD_SUFFIX}; + +/// 32 random hex digits, as Python's `uuid.uuid4().hex`. +fn uuid4_hex() -> io::Result { + let mut bytes = [0_u8; 16]; + getrandom::fill(&mut bytes).map_err(|error| io::Error::other(error.to_string()))?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Ok(bytes.iter().map(|byte| format!("{byte:02x}")).collect()) +} + +/// Write `text` to `path` through a hidden temporary sibling replaced into place. +pub(crate) fn replace_file(path: &Path, text: &str) -> io::Result<()> { + let Some(parent) = path.parent() else { + return Err(io::Error::other("a launch record path has no directory")); + }; + fs::create_dir_all(parent)?; + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let temporary: PathBuf = path.with_file_name(format!(".{name}.{}.tmp", uuid4_hex()?)); + let written: io::Result<()> = + fs::write(&temporary, text.as_bytes()).and_then(|()| fs::rename(&temporary, path)); + let _ = fs::remove_file(&temporary); + written +} + +/// Delete all but the newest `LAUNCH_RECORD_LIMIT` records in `directory`. +pub(crate) fn prune_records(directory: &Path) -> io::Result<()> { + let mut records: Vec<(i128, String, PathBuf)> = Vec::new(); + for entry in fs::read_dir(directory)? { + let entry: fs::DirEntry = entry?; + let name: String = entry.file_name().to_string_lossy().into_owned(); + if !name.ends_with(LAUNCH_RECORD_SUFFIX) { + continue; + } + let metadata: fs::Metadata = fs::metadata(entry.path())?; + records.push((modified_ns(&metadata)?, name, entry.path())); + } + records.sort_by(|left, right| right.cmp(left)); + for (_, _, stale) in records.into_iter().skip(LAUNCH_RECORD_LIMIT) { + let _ = fs::remove_file(stale); + } + Ok(()) +} + +/// `os.stat(path).st_mtime_ns`. +fn modified_ns(metadata: &fs::Metadata) -> io::Result { + let modified: std::time::SystemTime = metadata.modified()?; + Ok(match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(after) => i128::try_from(after.as_nanos()).unwrap_or(i128::MAX), + Err(before) => -i128::try_from(before.duration().as_nanos()).unwrap_or(i128::MAX), + }) +} diff --git a/crates/sqlbuild-cli/src/replay/_helpers/mod.rs b/crates/sqlbuild-cli/src/replay/_helpers/mod.rs index 07f5c4f195..97182491a2 100644 --- a/crates/sqlbuild-cli/src/replay/_helpers/mod.rs +++ b/crates/sqlbuild-cli/src/replay/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod console; +pub(crate) mod files; pub(crate) mod records; pub(crate) mod serving; diff --git a/crates/sqlbuild-cli/src/replay/constants.rs b/crates/sqlbuild-cli/src/replay/constants.rs index 2ee2e8da61..7c34dbf8a1 100644 --- a/crates/sqlbuild-cli/src/replay/constants.rs +++ b/crates/sqlbuild-cli/src/replay/constants.rs @@ -6,3 +6,5 @@ pub const LAUNCH_RECORD_FORMAT: i64 = 1; pub const LAUNCH_RECORD_DIRECTORY: [&str; 3] = ["target", "cache", "launcher-v1"]; pub const LAUNCH_RECORD_SUFFIX: &str = ".json"; pub const OUTPUT_FAILURE_EXIT_CODE: i64 = 1; +/// How many launch records one project keeps; older ones are pruned after each write. +pub const LAUNCH_RECORD_LIMIT: usize = 32; diff --git a/crates/sqlbuild-cli/src/replay/main/mod.rs b/crates/sqlbuild-cli/src/replay/main/mod.rs index 6f3a35f1ad..f377c949dc 100644 --- a/crates/sqlbuild-cli/src/replay/main/mod.rs +++ b/crates/sqlbuild-cli/src/replay/main/mod.rs @@ -1 +1,2 @@ pub mod serve_reused_compile; +pub mod write_launch_record; diff --git a/crates/sqlbuild-cli/src/replay/main/write_launch_record.rs b/crates/sqlbuild-cli/src/replay/main/write_launch_record.rs new file mode 100644 index 0000000000..4eba983e99 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/main/write_launch_record.rs @@ -0,0 +1,15 @@ +//! Leave the launch record a later `sqb` reads to serve an unchanged compile natively. + +use std::io; +use std::path::Path; + +use crate::replay::_helpers::files::{prune_records, replace_file}; + +/// Atomically replace `path` with the record `text`, then prune the oldest records beside it. +pub fn write_launch_record(path: &Path, text: &str) -> io::Result<()> { + replace_file(path, text)?; + match path.parent() { + Some(directory) => prune_records(directory), + None => Ok(()), + } +} diff --git a/crates/sqlbuild-cli/src/replay/tests.rs b/crates/sqlbuild-cli/src/replay/tests.rs index 46c3c1617c..c91f6b3796 100644 --- a/crates/sqlbuild-cli/src/replay/tests.rs +++ b/crates/sqlbuild-cli/src/replay/tests.rs @@ -1,4 +1,8 @@ #[path = "tests/test_console.rs"] mod console; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_write_launch_record.rs"] +mod write_launch_record; diff --git a/crates/sqlbuild-cli/src/replay/tests/helpers.rs b/crates/sqlbuild-cli/src/replay/tests/helpers.rs new file mode 100644 index 0000000000..fffc994443 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests/helpers.rs @@ -0,0 +1,32 @@ +use std::fs; +use std::path::Path; +use std::time::{Duration, SystemTime}; + +/// `count` records named `000.json`, `001.json`, ... with increasing modification times. +pub(super) fn existing_records(directory: &Path, count: usize) { + fs::create_dir_all(directory).expect("record directory"); + let start: SystemTime = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + for index in 0..count { + let path = directory.join(format!("{index:03}.json")); + fs::write(&path, b"{}").expect("record"); + let file = fs::File::options().write(true).open(&path).expect("open"); + file.set_modified(start + Duration::from_secs(index as u64)) + .expect("mtime"); + } +} + +/// The file names in `directory`, sorted. +pub(super) fn names(directory: &Path) -> Vec { + let mut names: Vec = fs::read_dir(directory) + .expect("listing") + .map(|entry| { + entry + .expect("entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + names.sort(); + names +} diff --git a/crates/sqlbuild-cli/src/replay/tests/test_types.rs b/crates/sqlbuild-cli/src/replay/tests/test_types.rs index b8145c3236..72ea3d4416 100644 --- a/crates/sqlbuild-cli/src/replay/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/replay/tests/test_types.rs @@ -4,3 +4,10 @@ pub(crate) struct TenthsTestCase { pub(crate) milliseconds: i64, pub(crate) expected_text: &'static str, } + +/// Existing records in a launch record directory and which survive one more write. +pub(crate) struct LaunchRecordWriteTestCase { + pub(crate) description: &'static str, + pub(crate) existing_records: usize, + pub(crate) expected_names: Vec, +} diff --git a/crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs b/crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs new file mode 100644 index 0000000000..88d4caf8eb --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests/test_write_launch_record.rs @@ -0,0 +1,45 @@ +use std::fs; + +use crate::replay::main::write_launch_record::write_launch_record; +use crate::replay::tests::helpers::{existing_records, names}; +use crate::replay::tests::test_types::LaunchRecordWriteTestCase; + +#[test] +fn given_record_directory_when_writing_then_record_replaces_atomically_and_oldest_are_pruned() { + let test_cases = [ + LaunchRecordWriteTestCase { + description: "first record creates its directory", + existing_records: 0, + expected_names: vec!["new.json".to_owned()], + }, + LaunchRecordWriteTestCase { + description: "the oldest record beyond the limit is pruned", + existing_records: 32, + expected_names: (1..32) + .map(|index| format!("{index:03}.json")) + .chain(["new.json".to_owned()]) + .collect(), + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let directory = root.path().join("target/cache/launcher-v1"); + existing_records(&directory, test_case.existing_records); + let path = directory.join("new.json"); + + write_launch_record(&path, "{\"format\": 1}").expect("write"); + + assert_eq!( + fs::read_to_string(&path).expect("read"), + "{\"format\": 1}", + "{}", + test_case.description + ); + assert_eq!( + names(&directory), + test_case.expected_names, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs index eb1ab460a8..a89f776184 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod artifact_batch; pub(crate) mod prepared_artifacts; pub(crate) mod registration; pub(crate) mod report; +pub(crate) mod reuse_attempt; pub(crate) mod reuse_recording; pub(crate) mod surrogates; pub(crate) mod target_writer; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs index afaea3f886..dfc638391e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::compile_command::{ - prepared_artifacts, report, reuse_recording, target_writer, + prepared_artifacts, report, reuse_attempt, reuse_recording, target_writer, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { report::register(module)?; + reuse_attempt::register(module)?; reuse_recording::register(module)?; target_writer::register(module)?; prepared_artifacts::register(module) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs new file mode 100644 index 0000000000..2d8b81a517 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -0,0 +1,454 @@ +//! Check whether a compile can replay the stored result, replay it on a hit and leave a launch record. + +use std::path::PathBuf; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file; + +use crate::bindings::_helpers::compile_driver::phases::{elapsed_ms, monotonic}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_outputs::json_report::plain_json; +use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const REUSE_CONSTANTS: &str = "sqlbuild.cli.compile_reuse.constants"; +const REUSE_MODELS: &str = "sqlbuild.cli.compile_reuse.models"; +const REUSE_TYPES: &str = "sqlbuild.cli.compile_reuse.types"; +const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; +const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; +const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; +const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; +const LAUNCH_RECORD_PATH_MODULE: &str = "sqlbuild.cli.entry.main._launch_record_path"; +const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; +const OUTPUT_CAPTURE_CONSTANTS: &str = "sqlbuild.runtime.output_capture.constants"; +const SECONDS_FIELD: &str = "{seconds:.1f}"; +const HIT: &str = "HIT"; +const CHECKED_HIT: &str = "hit"; +const CHECKED_BYPASS: &str = "bypass"; +const MISS: &str = "MISS"; +const BYPASS: &str = "BYPASS"; + +fn constant<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, REUSE_CONSTANTS, name) +} + +fn sorted<'py>(value: Bound<'py, PyAny>) -> PyResult> { + imported(value.py(), "builtins", "sorted")?.call1((value,)) +} + +fn listed<'py>(value: Bound<'py, PyAny>) -> PyResult> { + Ok(PyList::new(value.py(), value.try_iter()?.collect::>>()?)?.into_any()) +} + +fn reuse_logger(py: Python<'_>) -> PyResult> { + imported(py, "logging", "getLogger")?.call1((constant(py, "REUSE_LOGGER_NAME")?,)) +} + +/// The reuse constants the native check, the stored compile and the launch record share. +pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult> { + let rules: Bound<'_, PyDict> = PyDict::new(py); + rules.set_item("excluded", sorted(constant(py, "EXCLUDED_DIRECTORIES")?)?)?; + rules.set_item( + "excluded_root", + sorted(constant(py, "EXCLUDED_ROOT_DIRECTORIES")?)?, + )?; + rules.set_item( + "presence_suffixes", + listed(constant(py, "PRESENCE_ONLY_FILE_SUFFIXES")?)?, + )?; + let output_files: Bound<'_, PyAny> = + imported(py, RUNTIME_IDENTITY_MODULE, "redirected_output_files")?.call0()?; + if with_output_files { + rules.set_item("output_files", output_files)?; + } + rules.set_item( + "tracked_prefixes", + listed(constant(py, "TRACKED_ENVIRONMENT_PREFIXES")?)?, + )?; + rules.set_item( + "untracked_names", + sorted(constant(py, "UNTRACKED_ENVIRONMENT_NAMES")?)?, + )?; + rules.set_item( + "disabling_variables", + PyList::new( + py, + [ + ( + imported(py, COMPILE_CONSTANTS, "COMPILE_CACHE_DISABLE_ENV_VAR")?, + imported(py, COMPILE_CONSTANTS, "COMPILE_CACHE_DISABLE_VALUE")?, + ), + ( + constant(py, "REUSE_DISABLE_ENV_VAR")?, + constant(py, "REUSE_DISABLE_VALUE")?, + ), + ], + )?, + )?; + rules.set_item( + "config_filenames", + listed(constant(py, "PROJECT_CONFIG_FILENAMES")?)?, + )?; + for (key, name) in [ + ("racy_window_ns", "RACY_WINDOW_NS"), + ("max_stored_entries", "REUSE_MAX_STORED_ENTRIES"), + ("max_entry_bytes", "REUSE_MAX_ENTRY_BYTES"), + ("missing_environment_value", "MISSING_ENVIRONMENT_VALUE"), + ("missing_file_digest", "MISSING_FILE_DIGEST"), + ("missing_path_mtime_ns", "MISSING_PATH_MTIME_NS"), + ("project_root_path_mtime_ns", "PROJECT_ROOT_PATH_MTIME_NS"), + ] { + rules.set_item(key, constant(py, name)?)?; + } + rules.set_item( + "compiled_root", + imported(py, "os.path", "join")?.call1(( + constant(py, "TARGET_DIRECTORY_NAME")?, + constant(py, "COMPILED_DIRECTORY_NAME")?, + ))?, + )?; + rules.set_item( + "retired_directories", + PyList::new(py, [constant(py, "RETIRED_REUSE_DIRECTORY_NAME")?])?, + )?; + Ok(rules) +} + +/// The launch record `sqb` replays this reusable compile from, as a JSON object. +fn launch_payload<'py>( + attempt: &Bound<'py, PyAny>, + native: &Bound<'py, PyAny>, + key: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = attempt.py(); + let (invocation_digest, runtime, search_path): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = native.call_method0("launch_identity")?.extract()?; + let project_dir: Bound<'py, PyAny> = attempt.getattr("project_dir")?; + let resolved: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; + let capture: Bound<'py, PyAny> = if py + .import("sys")? + .getattr("stdout")? + .is_instance(&imported(py, COMPUTE_LOG_TEE_MODULE, "TextComputeLogTee")?)? + { + let logs: Bound<'py, PyAny> = resolved + .call_method1( + "__truediv__", + (constant(py, "COMPUTE_LOG_DIRECTORY_NAME")?,), + )? + .call_method0("resolve")?; + let capture: Bound<'py, PyDict> = PyDict::new(py); + capture.set_item("root", logs.str()?)?; + capture.set_item("command", constant(py, "COMPILE_COMMAND_NAME")?)?; + capture.set_item("project_dir", resolved.str()?)?; + capture.set_item("target", attempt.getattr("selected_target")?)?; + capture.into_any() + } else { + py.None().into_bound(py) + }; + let hit_message: Bound<'py, PyAny> = + constant(py, "REUSE_HIT_MESSAGE")?.call_method1("split", (SECONDS_FIELD,))?; + let timings: Bound<'py, PyDict> = PyDict::new(py); + for (key, name) in [ + ("hit", "REUSE_HIT_TIMING"), + ("miss", "REUSE_MISS_TIMING"), + ("bypass", "REUSE_BYPASS_TIMING"), + ("check", "REUSE_CHECK_TIMING"), + ("total", "TOTAL_TIMING"), + ] { + timings.set_item(key, constant(py, name)?)?; + } + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("format", constant(py, "LAUNCH_RECORD_FORMAT")?)?; + payload.set_item("key", key)?; + payload.set_item("project_dir", project_dir.str()?)?; + payload.set_item("store_path", native.call_method0("store_path")?)?; + payload.set_item("invocation_digest", invocation_digest)?; + payload.set_item("runtime", runtime)?; + payload.set_item("search_path", search_path)?; + payload.set_item( + "startup_inputs", + imported(py, STARTUP_INPUTS_MODULE, "startup_inputs")?.call0()?, + )?; + payload.set_item("json_output", attempt.getattr("json_output")?)?; + payload.set_item("rules", reuse_rules(py, false)?)?; + payload.set_item("capture", capture)?; + payload.set_item("hit_message", listed(hit_message)?)?; + payload.set_item("timings", timings)?; + Ok(payload) +} + +fn write_record( + attempt: &Bound<'_, PyAny>, + native: &Bound<'_, PyAny>, + path: &Bound<'_, PyAny>, +) -> PyResult<()> { + let payload: Bound<'_, PyDict> = launch_payload(attempt, native, path.getattr("stem")?)?; + let text: String = plain_json(payload.as_any())?; + let path: PathBuf = path.extract()?; + write_record_file(&path, &text).map_err(PyErr::from) +} + +/// Record this reusable compile for `sqb` when nothing it cannot replay took part. +pub(crate) fn write_launch_record(attempt: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = attempt.py(); + let path: Bound<'_, PyAny> = + imported(py, LAUNCH_RECORD_PATH_MODULE, "launch_record_path")?.call0()?; + let native: Bound<'_, PyAny> = attempt.getattr("native")?; + if path.is_none() || native.is_none() { + return Ok(()); + } + let environment: Bound<'_, PyAny> = imported(py, "os", "environ")?; + if environment.contains(imported( + py, + OUTPUT_CAPTURE_CONSTANTS, + "INVOCATION_CONTEXT_ENV", + )?)? { + return Ok(()); + } + let mut directory: Bound<'_, PyAny> = attempt.getattr("project_dir")?; + for part in constant(py, "LAUNCH_RECORD_DIRECTORY")?.try_iter()? { + directory = directory.call_method1("__truediv__", (part?,))?; + } + if path + .getattr("parent")? + .call_method0("resolve")? + .ne(directory.call_method0("resolve")?)? + { + return Ok(()); + } + match write_record(attempt, &native, &path) { + Err(error) if error.is_instance_of::(py) => { + reuse_logger(py)?.call_method( + "debug", + ("Compile reuse did not leave a launch record",), + Some(&keywords( + py, + &[("exc_info", error.value(py).clone().into_any())], + )?), + )?; + Ok(()) + } + written => written, + } +} + +fn outcome<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, REUSE_TYPES, "CompileReuseOutcome")?.getattr(name) +} + +/// A `CompileReuseAttempt` with `fields`. +fn reuse_attempt<'py>( + py: Python<'py>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + call_with(&imported(py, REUSE_MODELS, "CompileReuseAttempt")?, fields) +} + +/// The native reuse check request for this invocation. +fn reuse_check<'py>( + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let flag = |name: &str| -> PyResult { request.getattr(name)?.is_truthy() }; + let use_color: bool = !flag("no_color")? && stdout_supports_color(py)?; + let check: Bound<'py, PyDict> = PyDict::new(py); + check.set_item("project_dir", project_dir)?; + check.set_item( + "store_directory", + imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")? + .call1((imported(py, "pathlib", "Path")?.call1((project_dir,))?,))? + .call_method1( + "__truediv__", + (constant(py, "NATIVE_REUSE_DIRECTORY_NAME")?,), + )?, + )?; + check.set_item("selected_target", request.getattr("selected_target")?)?; + check.set_item( + "invocation", + call_with( + &imported(py, RUNTIME_IDENTITY_MODULE, "invocation_identity")?, + &[ + ("request", request.clone()), + ("project_dir", project_dir.clone()), + ("use_color", use_color.into_bound_py_any(py)?), + ], + )?, + )?; + check.set_item( + "runtime", + imported(py, RUNTIME_IDENTITY_MODULE, "runtime_identity")?.call0()?, + )?; + check.set_item("search_path", listed(py.import("sys")?.getattr("path")?)?)?; + check.set_item( + "bypass_requested", + flag("no_cache")? || flag("manifest")? || flag("profiling")? || flag("debug")?, + )?; + check.set_item("json_output", request.getattr("json_output")?)?; + check.set_item("rules", reuse_rules(py, true)?)?; + Ok(check) +} + +/// `print(text, file=sys.stderr)`. +fn print_stderr(py: Python<'_>, text: &Bound<'_, PyAny>) -> PyResult<()> { + py.import("builtins")?.getattr("print")?.call( + (text,), + Some(&keywords( + py, + &[("file", py.import("sys")?.getattr("stderr")?)], + )?), + )?; + Ok(()) +} + +/// `fields` with each override replacing the field of the same name, as `dataclasses.replace`. +fn overridden<'py>( + mut fields: Vec<(&'static str, Bound<'py, PyAny>)>, + overrides: Vec<(&'static str, Bound<'py, PyAny>)>, +) -> Vec<(&'static str, Bound<'py, PyAny>)> { + for (name, value) in overrides { + match fields.iter_mut().find(|field| field.0 == name) { + Some(field) => field.1 = value, + None => fields.push((name, value)), + } + } + fields +} + +/// Replay the stored compile on a hit; the missed attempt when it cannot be replayed. +fn replayed<'py>( + native: &Bound<'py, PyAny>, + missed: Vec<(&'static str, Bound<'py, PyAny>)>, + (started, json_output): (f64, bool), +) -> PyResult> { + let py: Python<'py> = native.py(); + let check_ms: i64 = elapsed_ms(py, started)?; + let mut timings: Vec<(Bound<'py, PyAny>, i64)> = Vec::new(); + for (name, counted) in [ + ("REUSE_HIT_TIMING", 1), + ("REUSE_MISS_TIMING", 0), + ("REUSE_BYPASS_TIMING", 0), + ("REUSE_CHECK_TIMING", check_ms), + ] { + timings.push((constant(py, name)?, counted)); + } + timings.push((constant(py, "TOTAL_TIMING")?, elapsed_ms(py, started)?)); + let rows: Bound<'py, PyList> = if json_output { + PyList::new(py, timings)? + } else { + PyList::empty(py) + }; + let replay: Bound<'py, PyAny> = native.call_method1("replay", (rows,))?; + if replay.is_none() { + return reuse_attempt( + py, + &overridden(missed, vec![("check_ms", 0_i64.into_bound_py_any(py)?)]), + ); + } + let (stdout, stderr_lines, exit_code): ( + Bound<'py, PyString>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = replay.extract()?; + let seconds: Bound<'py, PyAny> = check_ms + .into_bound_py_any(py)? + .call_method1("__truediv__", (constant(py, "MILLISECONDS_PER_SECOND")?,))?; + print_stderr( + py, + &constant(py, "REUSE_HIT_MESSAGE")?.call_method( + "format", + (), + Some(&keywords(py, &[("seconds", seconds)])?), + )?, + )?; + for line in stderr_lines.try_iter()? { + print_stderr(py, &line?)?; + } + let output: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; + output.call_method1("write", (stdout,))?; + output.call_method0("flush")?; + reuse_attempt( + py, + &overridden( + missed, + vec![ + ("outcome", outcome(py, HIT)?), + ("check_ms", check_ms.into_bound_py_any(py)?), + ("exit_code", exit_code), + ], + ), + ) +} + +/// Replay the stored compile when every input matches, otherwise prepare to store one. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn attempt_compile_reuse<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let started: f64 = monotonic(py)?; + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = + imported(py, "os.path", "abspath")?.call1((if requested.is_none() { + imported(py, "os", "getcwd")?.call0()? + } else { + requested + },))?; + let native: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "check_compile_reuse")? + .call1((reuse_check(request, &project_dir)?,))?; + let checked: String = native.call_method0("outcome")?.extract()?; + let json_output: Bound<'py, PyAny> = request.getattr("json_output")?; + let mut base: Vec<(&'static str, Bound<'py, PyAny>)> = vec![ + ( + "project_dir", + imported(py, "pathlib", "Path")?.call1((&project_dir,))?, + ), + ("started", started.into_bound_py_any(py)?), + ("selected_target", request.getattr("selected_target")?), + ("json_output", json_output.clone()), + ]; + if checked == CHECKED_BYPASS { + base.push(("outcome", outcome(py, BYPASS)?)); + return reuse_attempt(py, &base); + } + reuse_logger(py)?.call_method1( + "debug", + ( + "Compile reuse check digested %d project files", + native.call_method0("digested_files")?, + ), + )?; + let check_ms: i64 = if native.call_method0("replay_failed")?.is_truthy()? { + 0 + } else { + elapsed_ms(py, started)? + }; + base.extend([ + ("outcome", outcome(py, MISS)?), + ( + "entry_path", + imported(py, "pathlib", "Path")?.call1((native.call_method0("store_path")?,))?, + ), + ("native", native.clone()), + ("check_ms", check_ms.into_bound_py_any(py)?), + ]); + if checked != CHECKED_HIT { + return reuse_attempt(py, &base); + } + let attempt: Bound<'py, PyAny> = replayed(&native, base, (started, json_output.is_truthy()?))?; + if attempt.getattr("outcome")?.eq(outcome(py, HIT)?)? { + write_launch_record(&attempt)?; + } + Ok(attempt) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(attempt_compile_reuse, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index aa64f0de47..9908bd9144 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -8,15 +8,14 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::bindings::_helpers::compile_command::reuse_attempt::{reuse_rules, write_launch_record}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; use crate::bindings::_helpers::compile_outputs::reuse::artifact_digest; const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; -const NATIVE_REUSE_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.native_reuse"; const PROVIDER_SETTINGS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.provider_settings"; -const LAUNCH_RECORDS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.launch_records"; const MACRO_STORE_PATHS_MODULE: &str = "sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"; const MISS_OUTCOME: &str = "miss"; @@ -466,10 +465,7 @@ fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { ), ("dag_path", dag_path), ("json_output", facts.json_output.clone()), - ( - "rules", - imported(py, NATIVE_REUSE_MODULE, "reuse_rules")?.call0()?, - ), + ("rules", reuse_rules(py, true)?.into_any()), ], )?; let (outcome, digested): (String, Bound<'_, PyAny>) = @@ -562,10 +558,7 @@ pub(crate) fn write_reusable_compile( Err(error) => return Err(error), }; if stored { - let _ = call_with( - &imported(py, LAUNCH_RECORDS_MODULE, "write_launch_record")?, - &[("attempt", attempt)], - )?; + write_launch_record(&attempt)?; } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index 1e499b85fe..b1280c561e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -104,6 +104,19 @@ pub(crate) fn indented_json(value: &Bound<'_, PyAny>) -> PyResult { ) } +/// `json.dumps(value)`, raising what `json.dumps` raises. +pub(crate) fn plain_json(value: &Bound<'_, PyAny>) -> PyResult { + let mut conversion: Conversion = Conversion::default(); + let converted: JsonValue = conversion.json_value(value)?; + if let Some(error) = conversion.stdlib_error { + return Err(error); + } + emitted( + &converted, + &JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + ) +} + /// `json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True)`. pub(crate) fn compact_sorted_json(value: &Bound<'_, PyAny>) -> PyResult { let mut conversion: Conversion = Conversion::default(); diff --git a/fensu.toml b/fensu.toml index 918a9a4fd8..f0f89d3c32 100644 --- a/fensu.toml +++ b/fensu.toml @@ -33,6 +33,11 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/constants.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/cli/compile_reuse/__init__.py" +reason = "Native bindings own the compile reuse check, replay and launch records; this package declares the reuse models and constants and keeps the provider settings, runtime identity and startup inputs the native code reads from the interpreter." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cursor_algebra/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5fc520b43f..3b126e7915 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -51,7 +51,7 @@ from sqlbuild.cli.commands.models import ( ) from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, CommandWarehouseScope, @@ -1304,6 +1304,7 @@ class NativeReuseAttempt: def record(self, record: dict[str, object]) -> tuple[str, int]: ... def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... +def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ... class CompileArtifactWrites: @property diff --git a/src/sqlbuild/cli/compile/main/run.py b/src/sqlbuild/cli/compile/main/run.py index 8dda17e2f1..52f9307154 100644 --- a/src/sqlbuild/cli/compile/main/run.py +++ b/src/sqlbuild/cli/compile/main/run.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.cli.compile_reuse.main._attempt_compile_reuse import attempt_compile_reuse +from sqlbuild._native import attempt_compile_reuse from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py deleted file mode 100644 index 90bc5d2537..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Decide whether a compile can replay the stored result, and replay it on a hit.""" - -from __future__ import annotations - -import os -import time - -from sqlbuild.cli.compile_reuse._helpers.launch_records import write_launch_record -from sqlbuild.cli.compile_reuse._helpers.native_reuse import native_attempt -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome - - -def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: - """Replay a stored compile when every input matches, otherwise prepare to store one.""" - - started: float = time.monotonic() - project_dir: str = os.path.abspath( - os.getcwd() if request.project_dir is None else request.project_dir - ) - attempt: CompileReuseAttempt = native_attempt( - request=request, project_dir=project_dir, started=started - ) - if attempt.outcome is CompileReuseOutcome.HIT: - write_launch_record(attempt=attempt) - return attempt diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py b/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py deleted file mode 100644 index 32fd04594f..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Leave the native `sqb` executable what it needs to serve the next unchanged compile itself.""" - -from __future__ import annotations - -import contextlib -import json -import logging -import os -import sys -import uuid -from pathlib import Path - -from sqlbuild._native import NativeReuseAttempt -from sqlbuild.cli.compile_reuse._helpers.native_reuse import reuse_rules -from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs -from sqlbuild.cli.compile_reuse.constants import ( - COMPILE_COMMAND_NAME, - COMPUTE_LOG_DIRECTORY_NAME, - LAUNCH_RECORD_DIRECTORY, - LAUNCH_RECORD_FORMAT, - LAUNCH_RECORD_LIMIT, - REUSE_BYPASS_TIMING, - REUSE_CHECK_TIMING, - REUSE_HIT_MESSAGE, - REUSE_HIT_TIMING, - REUSE_LOGGER_NAME, - REUSE_MISS_TIMING, - TOTAL_TIMING, -) -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.entry.main._launch_record_path import launch_record_path -from sqlbuild.runtime.compute_logs.classes.text_tee import TextComputeLogTee -from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV - -_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) -_SECONDS_FIELD: str = "{seconds:.1f}" -_OUTPUT_FILES_RULE: str = "output_files" - - -def write_launch_record(*, attempt: CompileReuseAttempt) -> None: - """Record this reusable compile for `sqb` when nothing it cannot replay took part.""" - - path: Path | None = launch_record_path() - native: NativeReuseAttempt | None = attempt.native - if path is None or native is None or not _replayable_natively(): - return - directory: Path = attempt.project_dir.joinpath(*LAUNCH_RECORD_DIRECTORY) - if path.parent.resolve() != directory.resolve(): - return - try: - _write(path=path, payload=_payload(attempt=attempt, native=native, key=path.stem)) - _prune(directory=directory) - except OSError: - _LOGGER.debug("Compile reuse did not leave a launch record", exc_info=True) - - -def _replayable_natively() -> bool: - return INVOCATION_CONTEXT_ENV not in os.environ - - -def _payload( - *, attempt: CompileReuseAttempt, native: NativeReuseAttempt, key: str -) -> dict[str, object]: - invocation_digest, runtime, search_path = native.launch_identity() - rules: dict[str, object] = { - name: value for name, value in reuse_rules().items() if name != _OUTPUT_FILES_RULE - } - project_dir: Path = attempt.project_dir.resolve() - prefix, suffix = REUSE_HIT_MESSAGE.split(_SECONDS_FIELD) - return { - "format": LAUNCH_RECORD_FORMAT, - "key": key, - "project_dir": str(attempt.project_dir), - "store_path": native.store_path(), - "invocation_digest": invocation_digest, - "runtime": runtime, - "search_path": search_path, - "startup_inputs": startup_inputs(), - "json_output": attempt.json_output, - "rules": rules, - "capture": { - "root": str((project_dir / COMPUTE_LOG_DIRECTORY_NAME).resolve()), - "command": COMPILE_COMMAND_NAME, - "project_dir": str(project_dir), - "target": attempt.selected_target, - } - if isinstance(sys.stdout, TextComputeLogTee) - else None, - "hit_message": [prefix, suffix], - "timings": { - "hit": REUSE_HIT_TIMING, - "miss": REUSE_MISS_TIMING, - "bypass": REUSE_BYPASS_TIMING, - "check": REUSE_CHECK_TIMING, - "total": TOTAL_TIMING, - }, - } - - -def _write(*, path: Path, payload: dict[str, object]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - try: - _ = temporary.write_text(json.dumps(payload), encoding="utf-8") - os.replace(temporary, path) - finally: - with contextlib.suppress(OSError): - temporary.unlink(missing_ok=True) - - -def _prune(*, directory: Path) -> None: - records: list[tuple[int, Path]] = [ - (entry.stat().st_mtime_ns, entry) for entry in directory.glob("*.json") - ] - records.sort(reverse=True) - for _, stale in records[LAUNCH_RECORD_LIMIT:]: - with contextlib.suppress(OSError): - stale.unlink() diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py deleted file mode 100644 index 5c124e001a..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py +++ /dev/null @@ -1,162 +0,0 @@ -"""Check, replay and record whole-project reuse through the native store.""" - -from __future__ import annotations - -import logging -import os -import sys -from dataclasses import replace -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import NativeReuseAttempt, supports_color -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( - invocation_identity, - redirected_output_files, - runtime_identity, -) -from sqlbuild.cli.compile_reuse._helpers.timings import elapsed_ms, reuse_timings -from sqlbuild.cli.compile_reuse.constants import ( - COMPILED_DIRECTORY_NAME, - EXCLUDED_DIRECTORIES, - EXCLUDED_ROOT_DIRECTORIES, - MILLISECONDS_PER_SECOND, - MISSING_ENVIRONMENT_VALUE, - MISSING_FILE_DIGEST, - MISSING_PATH_MTIME_NS, - NATIVE_REUSE_DIRECTORY_NAME, - PRESENCE_ONLY_FILE_SUFFIXES, - PROJECT_CONFIG_FILENAMES, - PROJECT_ROOT_PATH_MTIME_NS, - RACY_WINDOW_NS, - RETIRED_REUSE_DIRECTORY_NAME, - REUSE_DISABLE_ENV_VAR, - REUSE_DISABLE_VALUE, - REUSE_HIT_MESSAGE, - REUSE_LOGGER_NAME, - REUSE_MAX_ENTRY_BYTES, - REUSE_MAX_STORED_ENTRIES, - TARGET_DIRECTORY_NAME, - TOTAL_TIMING, - TRACKED_ENVIRONMENT_PREFIXES, - UNTRACKED_ENVIRONMENT_NAMES, -) -from sqlbuild.cli.compile_reuse.models import ( - CompileReuseAttempt, - CompileReuseRequest, -) -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory - -_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) -_OUTCOMES: dict[str, CompileReuseOutcome] = { - outcome.value: outcome for outcome in CompileReuseOutcome -} -_STORED: str = "stored" -_SKIPPED: str = "skipped" - - -def native_attempt( - *, request: CompileReuseRequest, project_dir: str, started: float -) -> CompileReuseAttempt: - """Check reuse natively, replaying on a hit.""" - - attempt: NativeReuseAttempt = _native.check_compile_reuse( - { - "project_dir": project_dir, - "store_directory": compiler_cache_directory(Path(project_dir)) - / NATIVE_REUSE_DIRECTORY_NAME, - "selected_target": request.selected_target, - "invocation": invocation_identity( - request=request, - project_dir=project_dir, - use_color=(not request.no_color) and supports_color(), - ), - "runtime": runtime_identity(), - "search_path": list(sys.path), - "bypass_requested": request.no_cache - or request.manifest - or request.profiling - or request.debug, - "json_output": request.json_output, - "rules": reuse_rules(), - } - ) - outcome: CompileReuseOutcome = _OUTCOMES[attempt.outcome()] - base: CompileReuseAttempt = CompileReuseAttempt( - outcome=outcome, - project_dir=Path(project_dir), - started=started, - selected_target=request.selected_target, - json_output=request.json_output, - ) - if outcome is CompileReuseOutcome.BYPASS: - return base - _LOGGER.debug("Compile reuse check digested %d project files", attempt.digested_files()) - missed: CompileReuseAttempt = replace( - base, - outcome=CompileReuseOutcome.MISS, - entry_path=Path(attempt.store_path()), - native=attempt, - check_ms=0 if attempt.replay_failed() else elapsed_ms(started=started), - ) - if outcome is CompileReuseOutcome.HIT: - return _replayed(attempt=attempt, missed=missed, started=started, json=request.json_output) - return missed - - -def _replayed( - *, - attempt: NativeReuseAttempt, - missed: CompileReuseAttempt, - started: float, - json: bool, -) -> CompileReuseAttempt: - check_ms: int = elapsed_ms(started=started) - timings: dict[str, int] = { - **reuse_timings(outcome=CompileReuseOutcome.HIT, check_ms=check_ms), - TOTAL_TIMING: elapsed_ms(started=started), - } - replay: tuple[str, list[str], int] | None = attempt.replay( - list(timings.items()) if json else [] - ) - if replay is None: - return replace(missed, check_ms=0) - stdout, stderr_lines, exit_code = replay - print(REUSE_HIT_MESSAGE.format(seconds=check_ms / MILLISECONDS_PER_SECOND), file=sys.stderr) - for line in stderr_lines: - print(line, file=sys.stderr) - _ = sys.stdout.write(stdout) - sys.stdout.flush() - return replace(missed, outcome=CompileReuseOutcome.HIT, check_ms=check_ms, exit_code=exit_code) - - -def reuse_rules() -> dict[str, object]: - """The reuse constants the native check and record share with the Python implementation.""" - - return { - "excluded": sorted(EXCLUDED_DIRECTORIES), - "excluded_root": sorted(EXCLUDED_ROOT_DIRECTORIES), - "presence_suffixes": list(PRESENCE_ONLY_FILE_SUFFIXES), - "output_files": redirected_output_files(), - "tracked_prefixes": list(TRACKED_ENVIRONMENT_PREFIXES), - "untracked_names": sorted(UNTRACKED_ENVIRONMENT_NAMES), - "disabling_variables": [ - (COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE), - (REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE), - ], - "config_filenames": list(PROJECT_CONFIG_FILENAMES), - "racy_window_ns": RACY_WINDOW_NS, - "max_stored_entries": REUSE_MAX_STORED_ENTRIES, - "max_entry_bytes": REUSE_MAX_ENTRY_BYTES, - "missing_environment_value": MISSING_ENVIRONMENT_VALUE, - "missing_file_digest": MISSING_FILE_DIGEST, - "missing_path_mtime_ns": MISSING_PATH_MTIME_NS, - "project_root_path_mtime_ns": PROJECT_ROOT_PATH_MTIME_NS, - "compiled_root": os.path.join(TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME), - "retired_directories": [RETIRED_REUSE_DIRECTORY_NAME], - } diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py b/src/sqlbuild/cli/compile_reuse/_helpers/timings.py deleted file mode 100644 index e33f7438b9..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Compile reuse counters and elapsed time reported in compile_timings.""" - -from __future__ import annotations - -import time - -from sqlbuild.cli.compile_reuse.constants import ( - MILLISECONDS_PER_SECOND, - REUSE_BYPASS_TIMING, - REUSE_CHECK_TIMING, - REUSE_HIT_TIMING, - REUSE_MISS_TIMING, -) -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome - - -def reuse_timings(*, outcome: CompileReuseOutcome, check_ms: int) -> dict[str, int]: - """Return the reuse counters and check time reported in compile_timings.""" - - return { - REUSE_HIT_TIMING: int(outcome is CompileReuseOutcome.HIT), - REUSE_MISS_TIMING: int(outcome is CompileReuseOutcome.MISS), - REUSE_BYPASS_TIMING: int(outcome is CompileReuseOutcome.BYPASS), - REUSE_CHECK_TIMING: check_ms, - } - - -def elapsed_ms(*, started: float) -> int: - """Return whole milliseconds elapsed since a monotonic start time.""" - - return int((time.monotonic() - started) * MILLISECONDS_PER_SECOND) diff --git a/src/sqlbuild/cli/compile_reuse/main/__init__.py b/src/sqlbuild/cli/compile_reuse/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py b/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py deleted file mode 100644 index 587fd42003..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public entry for replaying the previous compile when no compile input changed.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.attempt import attempt_reuse -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest - - -def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: - """Replay the stored compile on a hit, otherwise return the state a full compile stores.""" - - return attempt_reuse(request=request) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 70f7e9f404..ad3bd2d6d9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -31,8 +31,8 @@ import sqlbuild._native as native_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization -import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings +import sqlbuild.cli.compile_reuse.constants as reuse_constants import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class @@ -3397,7 +3397,7 @@ def record_digested_files(*, monkeypatch: pytest.MonkeyPatch) -> list[int]: def settle_racy_window(*, monkeypatch: pytest.MonkeyPatch) -> None: """Treat files written before a compile starts as settled instead of waiting two seconds.""" - monkeypatch.setattr(native_reuse, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(reuse_constants, "RACY_WINDOW_NS", 0) def write_large_file(*, project_dir: Path, relative_path: str, size_bytes: int) -> Path: @@ -4221,7 +4221,7 @@ def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: """Make compile reuse stop watching the models folder, so a model edit goes unseen.""" monkeypatch.setattr( - native_reuse, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} + reuse_constants, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index 037911a191..e24bb68e25 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -13,8 +13,9 @@ import sqlbuild._native as native_module from sqlbuild._native import PreparedCompileArtifacts from sqlbuild._native import run_cli as main -from sqlbuild.cli.compile_reuse._helpers import native_reuse as native_reuse_module +from sqlbuild.cli.compile_reuse import constants as reuse_constants_module from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module +from sqlbuild.cli.compile_reuse._helpers import runtime_identity as runtime_identity_module from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity from sqlbuild.cli.compile_reuse.constants import ( NATIVE_REUSE_DIRECTORY_NAME, @@ -109,7 +110,7 @@ def prepare_output_compiles(*, monkeypatch: pytest.MonkeyPatch, reuse_disabled: monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, reuse_disabled) monkeypatch.setattr(native_module, "PreparedCompileArtifacts", _StagingEveryProject) - monkeypatch.setattr(native_reuse_module, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(reuse_constants_module, "RACY_WINDOW_NS", 0) def main_compile_json(*, project_dir: Path) -> int: @@ -210,7 +211,7 @@ def change_runtime(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: def changed() -> dict[str, str]: return {**runtime_identity(), "python": "3.0.0 (simulated)"} - monkeypatch.setattr(native_reuse_module, "runtime_identity", changed) + monkeypatch.setattr(runtime_identity_module, "runtime_identity", changed) def change_native_build(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: From aa3464b09630868c71865dafecc879697658a232 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:10:39 +0100 Subject: [PATCH 24/32] chore: drop the launch record limit now owned by the native writer --- src/sqlbuild/cli/compile_reuse/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 0980a46ebe..a6c9d8cca0 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -76,6 +76,5 @@ ) LAUNCH_RECORD_FORMAT: int = 1 LAUNCH_RECORD_DIRECTORY: tuple[str, ...] = (TARGET_DIRECTORY_NAME, "cache", "launcher-v1") -LAUNCH_RECORD_LIMIT: int = 32 COMPUTE_LOG_DIRECTORY_NAME: str = "logs" COMPILE_COMMAND_NAME: str = "compile" From 39c78eb330b95dea4eee86604367a0d8ee3d03b6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:30:12 +0100 Subject: [PATCH 25/32] fix: use native colour detection in CLI previews --- python-boundary.toml | 2 +- scripts/cli_preview/_helpers/workflow.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python-boundary.toml b/python-boundary.toml index 8629e933a4..8872cc876d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 45121 +remaining_lines = 44051 [[stays]] path = "executor" diff --git a/scripts/cli_preview/_helpers/workflow.py b/scripts/cli_preview/_helpers/workflow.py index b4a678c918..47c4d61aa0 100644 --- a/scripts/cli_preview/_helpers/workflow.py +++ b/scripts/cli_preview/_helpers/workflow.py @@ -13,7 +13,7 @@ from scripts.cli_preview.exceptions import PreviewSetupError from scripts.cli_preview.models import PreviewScene -from sqlbuild.presentation.main.supports_color import supports_color +from sqlbuild._native import supports_color _PAYMENTS_MODEL_PATH: Path = Path("models/staging/stg_payments.sql") _QUERY_MARKER: str = " amount_cents,\n" From b659c3e941dcf82350f0593e6b50cb19ea6e09d9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:29:41 +0100 Subject: [PATCH 26/32] fix: stop native progress spinner threads before interpreter finalization --- .../_helpers/build_progress/registration.rs | 3 +- .../_helpers/build_progress/terminal.rs | 42 ++++++++++++++++++- .../bindings/_helpers/load/load_progress.rs | 26 ++++-------- src/sqlbuild/_native.pyi | 1 + .../presentation/classes/_test_types.py | 1 + .../sqlbuild/presentation/classes/helpers.py | 21 +++++++++- .../presentation/classes/test_status.py | 9 +++- 7 files changed, 80 insertions(+), 23 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs index 591e3c48e3..27abb010e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{ - audit_progress, build_callbacks, footer, nested_progress, transient_status, + audit_progress, build_callbacks, footer, nested_progress, terminal, transient_status, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -11,5 +11,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { build_callbacks::register(module)?; footer::register(module)?; nested_progress::register(module)?; + terminal::register(module)?; transient_status::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs index 65a1bb16ac..c0ddf2193d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/terminal.rs @@ -1,13 +1,17 @@ //! Terminal width, the shared transient line coordinator and spinner threads of progress reporters. -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const COLUMNS_MODULE: &str = "sqlbuild.presentation.main.terminal_columns"; const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; const SPINNER_JOIN_SECONDS: f64 = 0.2; +const EXIT_JOIN_SECONDS: f64 = 1.0; +const SPINNER_THREAD_NAME: &str = "sqlbuild-progress-spinner"; +const STOP_EVENT_ATTRIBUTE: &str = "_sqlbuild_stop_event"; pub(crate) const SPINNER_TICK_SECONDS: f64 = 0.1; pub(crate) const CLEAR_LINE: &str = "\r\u{1b}[K"; pub(crate) const HIDE_CURSOR: &str = "\u{1b}[?25l"; @@ -89,6 +93,32 @@ pub(crate) fn flush(stream: &Bound<'_, PyAny>) -> PyResult<()> { Ok(()) } +/// Stop and join live spinners at exit: finalization ending one inside its native wait aborts. +#[pyfunction] +fn stop_progress_spinners(py: Python<'_>) -> PyResult<()> { + let mut spinners: Vec> = Vec::new(); + for thread in imported(py, "threading", "enumerate")? + .call0()? + .try_iter()? + { + let thread: Bound<'_, PyAny> = thread?; + if thread.getattr("name")?.eq(SPINNER_THREAD_NAME)? + && thread.hasattr(STOP_EVENT_ATTRIBUTE)? + { + thread.getattr(STOP_EVENT_ATTRIBUTE)?.call_method0("set")?; + spinners.push(thread); + } + } + for thread in spinners { + if thread.call_method0("is_alive")?.is_truthy()? { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("timeout", EXIT_JOIN_SECONDS)?; + thread.call_method("join", (), Some(&options))?; + } + } + Ok(()) +} + /// Start a daemon thread running `owner._spin_until_stopped(event)`; return the event and thread. pub(crate) fn start_spinner<'py>( owner: &Bound<'py, PyAny>, @@ -101,8 +131,10 @@ pub(crate) fn start_spinner<'py>( ("target", owner.getattr("_spin_until_stopped")?), ("args", PyTuple::new(py, [&event])?.into_any()), ("daemon", PyBool::new(py, true).to_owned().into_any()), + ("name", SPINNER_THREAD_NAME.into_bound_py_any(py)?), ], )?; + thread.setattr(STOP_EVENT_ATTRIBUTE, &event)?; thread.call_method0("start")?; Ok((event, thread)) } @@ -116,3 +148,11 @@ pub(crate) fn join_spinner(thread: &Bound<'_, PyAny>) -> PyResult<()> { } Ok(()) } + +/// Add `stop_progress_spinners` and run it at exit, before finalization ends daemon threads. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let hook = wrap_pyfunction!(stop_progress_spinners, module)?; + module.add_function(hook.clone())?; + imported(module.py(), "atexit", "register")?.call1((hook,))?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs index 021b9ce147..506e5e996c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs @@ -4,16 +4,15 @@ use std::collections::HashMap; use std::sync::{Mutex, MutexGuard, PoisonError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use pyo3::{pyclass, pymethods}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::load_command::main::render_load_result::render_load_result; use sqlbuild_core::load_command::main::render_load_spinner_line::render_load_spinner_line; use sqlbuild_core::load_command::models::{LoadResultText, LoadSpinnerRow, LoadStatus}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, within, -}; +use crate::bindings::_helpers::build_progress::terminal::start_spinner; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; const LINES_MODULE: &str = "sqlbuild.presentation.main.transient_line_coordinator"; @@ -177,21 +176,10 @@ impl LoadProgressReporter { fn start_spinner(slf: &Bound<'_, Self>) -> PyResult<()> { let py: Python<'_> = slf.py(); slf.get().stop_spinner(py)?; - let event: Bound<'_, PyAny> = imported(py, "threading", "Event")?.call0()?; - let thread: Bound<'_, PyAny> = call_with( - &imported(py, "threading", "Thread")?, - &[ - ("target", slf.getattr("_spin_until_stopped")?), - ("args", PyTuple::new(py, [&event])?.into_any()), - ("daemon", PyBool::new(py, true).to_owned().into_any()), - ], - )?; - { - let mut state = slf.get().state(); - state.stop_event = Some(event.unbind()); - state.thread = Some(thread.clone().unbind()); - } - thread.call_method0("start")?; + let (event, thread) = start_spinner(slf.as_any())?; + let mut state = slf.get().state(); + state.stop_event = Some(event.unbind()); + state.thread = Some(thread.unbind()); Ok(()) } } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3b126e7915..0360bb0f30 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3488,6 +3488,7 @@ def write_runtime_target( ) -> None: ... def run_plan_command(*, request: PlanCommandRequest) -> int: ... def run_rules_command(*, request: RulesCommandRequest) -> int: ... +def stop_progress_spinners() -> None: ... class TransientStatusReporter: def __init__( diff --git a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py index 89c32a2216..068f788a87 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/presentation/classes/_test_types.py @@ -58,4 +58,5 @@ class StatusReporterOutputTestCase: @dataclass(frozen=True) class StatusReporterExitTestCase: description: str + script: str expected_returncode: int diff --git a/tests/unit/src/sqlbuild/presentation/classes/helpers.py b/tests/unit/src/sqlbuild/presentation/classes/helpers.py index ebb758f669..829f388973 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/helpers.py +++ b/tests/unit/src/sqlbuild/presentation/classes/helpers.py @@ -80,6 +80,14 @@ def persistent_lines(text: str) -> tuple[str, ...]: return tuple(line.rsplit("\x1b[K", 1)[-1] for line in without_cursor.split("\n")) +SLOW_FINALIZATION_SCRIPT: str = ( + "class SlowFinalizer:\n" + " def __init__(self):\n" + " self.cycle = self\n" + " def __del__(self):\n" + " time.sleep(0.5)\n" + "SlowFinalizer()\n" +) OPEN_SPINNER_SCRIPT: str = ( "import io, time\n" "from sqlbuild._native import TransientStatusReporter\n" @@ -89,7 +97,18 @@ def persistent_lines(text: str) -> tuple[str, ...]: "reporter = TransientStatusReporter(stream=Tty())\n" "reporter.start('Working...')\n" "time.sleep(0.35)\n" -) +) + SLOW_FINALIZATION_SCRIPT +OPEN_NESTED_SPINNER_SCRIPT: str = ( + "import io, time\n" + "from sqlbuild._native import NestedCommandProgressCallbacks\n" + "class Tty(io.StringIO):\n" + " def isatty(self):\n" + " return True\n" + "reporter = NestedCommandProgressCallbacks(\n" + " total=1, label='audits', stream=Tty(), use_color=False\n" + ")\n" + "reporter.on_item_start(group_name='orders', item_name='not_null')\n" +) + SLOW_FINALIZATION_SCRIPT def without_no_progress_env() -> dict[str, str]: diff --git a/tests/unit/src/sqlbuild/presentation/classes/test_status.py b/tests/unit/src/sqlbuild/presentation/classes/test_status.py index a9051dc84f..d69ab52cc5 100644 --- a/tests/unit/src/sqlbuild/presentation/classes/test_status.py +++ b/tests/unit/src/sqlbuild/presentation/classes/test_status.py @@ -13,6 +13,7 @@ StatusReporterOutputTestCase, ) from tests.unit.src.sqlbuild.presentation.classes.helpers import ( + OPEN_NESTED_SPINNER_SCRIPT, OPEN_SPINNER_SCRIPT, RecordingStream, apply_no_progress_env, @@ -140,6 +141,12 @@ def test_given_status_sequence_when_reporting_then_writes_expected_persistent_li ( StatusReporterExitTestCase( description="an open spinner is stopped before the interpreter exits", + script=OPEN_SPINNER_SCRIPT, + expected_returncode=0, + ), + StatusReporterExitTestCase( + description="an unclosed nested progress spinner is stopped before finalization", + script=OPEN_NESTED_SPINNER_SCRIPT, expected_returncode=0, ), ), @@ -149,7 +156,7 @@ def test_given_open_spinner_when_process_exits_then_exits_cleanly( test_case: StatusReporterExitTestCase, ) -> None: completed: subprocess.CompletedProcess[str] = subprocess.run( - (sys.executable, "-c", OPEN_SPINNER_SCRIPT), + (sys.executable, "-c", test_case.script), capture_output=True, text=True, check=False, From 012cfb244ec2a550914b70a50e6d283197a190f3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:00:34 +0100 Subject: [PATCH 27/32] refactor: build, evaluate and finalize native rules requests natively --- .../src/bindings/_helpers/rules/rows.rs | 290 +++++++++++++----- .../src/bindings/_helpers/rules/rule_run.rs | 28 +- src/sqlbuild/_native.pyi | 25 +- .../_helpers/engine/custom_dispatch.py | 15 +- .../rule_engine/_helpers/run/native_rows.py | 143 --------- .../compiler/compiled_project/helpers.py | 6 +- .../rule_engine/_helpers/engine/helpers.py | 27 +- .../rule_engine/_helpers/run/helpers.py | 11 +- 8 files changed, 272 insertions(+), 273 deletions(-) delete mode 100644 src/sqlbuild/rule_engine/_helpers/run/native_rows.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 54a905591f..8163395c66 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -4,12 +4,16 @@ use std::collections::HashSet; use std::path::PathBuf; use std::sync::Mutex; -use pyo3::exceptions::PyOSError; +use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods}; -use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use pyo3::{ + FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction, +}; +use serde_json::{Value, json}; +use sqlbuild_rules::constants::API_VERSION; use sqlbuild_rules::engine::main::build_models::build_models; use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; @@ -19,26 +23,29 @@ use sqlbuild_rules::models::{ Declaration, DeclarationMember, EvaluateRequest, Fault, Model, ModelRow, ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, }; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; +use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::records::declaration_rows; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; +use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; use crate::bindings::types::CompilerDetach; const LEFT_ARGUMENT: &str = "left"; const RIGHT_ARGUMENT: &str = "right"; - -type EvaluationRow<'py> = ( - Vec>, - Vec, - u64, - u64, - u64, - u64, - bool, -); +const DIALECT_ARGUMENT: &str = "dialect"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; +const TYPES_EQUAL_MODULE: &str = "sqlbuild.adapter.type_system.main.types_equal"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const NATIVE_RULES_CACHE_FILE: &str = "native.json"; +const NATIVE_RULES_MEMO_FILE: &str = "native-response.json"; /// Name, relative path, members, value, value type and rendering of one declaration. #[derive(FromPyObject)] @@ -52,10 +59,7 @@ struct DeclarationRowPy<'py>( ); /// The request's project facts, which models, tests and scenarios are built beside. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct ProjectRowsPy<'py> { - header_json: Bound<'py, PyBytes>, +struct ProjectRows<'py> { memo: Option<(String, String)>, sql_analysis_enabled: bool, include_type_proof: bool, @@ -64,6 +68,7 @@ struct ProjectRowsPy<'py> { public_constants: Vec>, initial_findings: Vec>, types_equal: Bound<'py, PyAny>, + dialect: Bound<'py, PyAny>, } /// A natively built rules request held until it is evaluated once. @@ -91,16 +96,97 @@ impl NativeRulesRequest { } } +/// Build the built-in rules request natively from the compiled project's rows. #[pyfunction] -fn build_rules_request<'py>( - mut project: ProjectRowsPy<'py>, - models: (PyRef<'py, NativeCompiledProject>, Vec), - sql_tests: Vec>, - scenarios: Vec>, -) -> PyResult { +#[pyo3(signature = (**options))] +fn build_rules_request(options: Option<&Bound<'_, PyDict>>) -> PyResult { + let request = Request::new("build_rules_request", options)?; + let py: Python<'_> = request.py(); + as_rules_error(py, built_request(&request)) +} + +fn built_request(options: &Request<'_, '_>) -> PyResult { + let py: Python<'_> = options.py(); + let project: Bound<'_, PyAny> = options.get("project")?; + let config: Bound<'_, PyAny> = options.get("config")?; + let project_dir: Bound<'_, PyAny> = options.get("project_dir")?; + let dialect: Bound<'_, PyAny> = options.get("dialect")?; + let resolved: Bound<'_, PyAny> = project_dir.call_method0("resolve")?; + let resolved_path: PathBuf = resolved.extract()?; + let header: Value = json!({ + "version": API_VERSION, + "project_dir": resolved.str()?.to_str()?, + "dialect": dialect.extract::()?, + "config": config_value(&config)?, + "scope_index": plain_value(&call_with( + &imported(py, NATIVE_MODULE, "scope_metadata_projection")?, + &[("index", project.getattr("scope_index")?)], + )?)?, + "defer_suppressions": true, + "custom_rules": plain_value(&options.get("custom_payloads")?)?, + "rules_cache_path": path_text(py, rules_bulk_cache_path(&resolved_path, NATIVE_RULES_CACHE_FILE))?, + }); + let (public_enums, public_constants) = declaration_rows( + PyTuple::new(py, mapping_values(&project.getattr("public_enums")?)?)?.as_any(), + PyTuple::new(py, mapping_values(&project.getattr("public_constants")?)?)?.as_any(), + )?; + let enabled: bool = config.getattr("cache")?.getattr("enabled")?.is_truthy()?; + let memo: Option<(String, String)> = if enabled { + let unresolved: PathBuf = project_dir.extract()?; + Some(( + imported(py, CODE_IDENTITY_MODULE, "compiled_code_identity")? + .call0()? + .extract()?, + path_text( + py, + rules_bulk_cache_path(&unresolved, NATIVE_RULES_MEMO_FILE), + )?, + )) + } else { + None + }; + let mut attached_audit_targets: Vec = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + let target: Bound<'_, PyAny> = audit?.getattr("attached_target_name")?; + if !target.is_none() { + attached_audit_targets.push(target.extract()?); + } + } + let mut rows = ProjectRows { + memo, + sql_analysis_enabled: project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()?, + include_type_proof: options.get("include_type_proof")?.is_truthy()?, + attached_audit_targets, + public_enums: public_enums.extract()?, + public_constants: public_constants.extract()?, + initial_findings: options + .get("initial_findings")? + .try_iter()? + .collect::>()?, + types_equal: imported(py, TYPES_EQUAL_MODULE, "types_equal")?, + dialect, + }; + let facts: Bound<'_, NativeCompiledProject> = + imported(py, NATIVE_MODULE, "compiled_project_facts")? + .call1((&project,))? + .downcast_into()?; + let mut names: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + names.push(model?.getattr("name")?.extract()?); + } + let sql_tests: Vec> = project + .getattr("sql_tests")? + .try_iter()? + .collect::>()?; + let scenarios: Vec> = project + .getattr("sql_scenarios")? + .try_iter()? + .collect::>()?; compiler_guard(|| { - let mut request: EvaluateRequest = - serde_json::from_slice(project.header_json.as_bytes()).map_err(value_error)?; + let mut request: EvaluateRequest = serde_json::from_value(header).map_err(value_error)?; request.sql_tests = sql_tests .iter() .map(sql_test_fact) @@ -109,17 +195,17 @@ fn build_rules_request<'py>( .iter() .map(scenario_fact) .collect::>()?; - request.public_enums = declarations(std::mem::take(&mut project.public_enums))?; - request.public_constants = declarations(std::mem::take(&mut project.public_constants))?; - request.initial_findings = project + request.public_enums = declarations(std::mem::take(&mut rows.public_enums))?; + request.public_constants = declarations(std::mem::take(&mut rows.public_constants))?; + request.initial_findings = rows .initial_findings .iter() .map(finding_fault) .collect::>()?; - let rows: Vec = - project_model_rows(models.0.facts(), &models.1).map_err(value_error)?; - request.models = built_models(&project, &request, rows)?; - let memo: Option<(String, PathBuf)> = project + let model_rows: Vec = + project_model_rows(facts.borrow().facts(), &names).map_err(value_error)?; + request.models = built_models(&rows, &request, model_rows)?; + let memo: Option<(String, PathBuf)> = rows .memo .take() .map(|(identity, path)| (identity, PathBuf::from(path))); @@ -129,11 +215,19 @@ fn build_rules_request<'py>( }) } +/// Answer the request from the response memo or evaluate it, decoding findings natively. #[pyfunction] fn evaluate_rules_request<'py>( py: Python<'py>, request: &Bound<'py, NativeRulesRequest>, -) -> PyResult> { +) -> PyResult> { + as_rules_error(py, evaluated(py, request)) +} + +fn evaluated<'py>( + py: Python<'py>, + request: &Bound<'py, NativeRulesRequest>, +) -> PyResult> { let request: RowsRequest = request .borrow() .request @@ -149,44 +243,63 @@ fn evaluate_rules_request<'py>( RowsError::Io(error) => PyOSError::new_err(error.to_string()), })?; let classes = FindingClasses::new(py)?; - Ok(( - evaluation - .faults - .into_iter() - .map(|fault| classes.finding_of_fault(fault, false)) - .collect::>()?, - evaluation.selected_codes, - evaluation.evaluated_models, - evaluation.cache_hits, - evaluation.cache_misses, - evaluation.built_in_ms, - evaluation.reused, - )) + let findings: Vec> = evaluation + .faults + .into_iter() + .map(|fault| classes.finding_of_fault(fault, false)) + .collect::>()?; + call_with( + &imported(py, MODELS_MODULE, "NativeRulesEvaluation")?, + &[ + ("findings", PyTuple::new(py, findings)?.into_any()), + ( + "selected_codes", + PyTuple::new(py, evaluation.selected_codes)?.into_any(), + ), + ( + "evaluated_models", + evaluation.evaluated_models.into_bound_py_any(py)?, + ), + ("cache_hits", evaluation.cache_hits.into_bound_py_any(py)?), + ( + "cache_misses", + evaluation.cache_misses.into_bound_py_any(py)?, + ), + ("built_in_ms", evaluation.built_in_ms.into_bound_py_any(py)?), + ("reused", evaluation.reused.into_bound_py_any(py)?), + ], + ) } -/// The findings one finalisation applies the exception policy to. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct FinalizeRowsPy<'py> { - project_dir: String, - config_json: Bound<'py, PyBytes>, +/// Apply the exception policy natively, keeping each original finding's fixable mark. +#[pyfunction] +#[pyo3(signature = (*, config, project_dir, evaluated_codes, findings))] +pub(crate) fn finalize_rule_findings_rows<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, evaluated_codes: Vec, findings: Vec>, +) -> PyResult> { + let py: Python<'py> = config.py(); + as_rules_error( + py, + finalized(config, project_dir, (evaluated_codes, findings)), + ) } -#[pyfunction] -fn finalize_rule_findings_rows<'py>( - py: Python<'py>, - request: FinalizeRowsPy<'py>, -) -> PyResult>> { - let config: RulesConfig = serde_json::from_slice(request.config_json.as_bytes()) +fn finalized<'py>( + config: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, + (evaluated_codes, findings): (Vec, Vec>), +) -> PyResult> { + let py: Python<'py> = config.py(); + let config: RulesConfig = serde_json::from_value(config_value(config)?) .map_err(|error| value_error(format!("invalid findings request: {error}")))?; - let FinalizeRowsPy { - project_dir, - evaluated_codes, - findings, - .. - } = request; + let project_dir: String = project_dir + .call_method0("resolve")? + .str()? + .to_str()? + .to_owned(); let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); let mut faults: Vec = Vec::with_capacity(findings.len()); for finding in &findings { @@ -200,31 +313,63 @@ fn finalize_rule_findings_rows<'py>( .compiler_detach(|| finalize_rows(project_dir, config, &evaluated_codes, faults)) .map_err(value_error)?; let classes = FindingClasses::new(py)?; - finalized + let rows: Vec> = finalized .into_iter() .map(|fault| { let is_fixable: bool = fixable.contains(&fault_identity(&fault)); classes.finding_of_fault(fault, is_fixable) }) - .collect() + .collect::>()?; + PyTuple::new(py, rows) +} + +/// `asdict(config)` as orjson with `default=str` encodes it. +fn config_value(config: &Bound<'_, PyAny>) -> PyResult { + plain_value(&imported(config.py(), DATACLASSES_MODULE, "asdict")?.call1((config,))?) +} + +/// `str(path)` of the Python path built from `path`. +fn path_text(py: Python<'_>, path: PathBuf) -> PyResult { + Ok(path.into_pyobject(py)?.str()?.to_str()?.to_owned()) +} + +fn mapping_values<'py>(mapping: &Bound<'py, PyAny>) -> PyResult>> { + mapping.call_method0("values")?.try_iter()?.collect() +} + +/// Value and type errors of the request as `RulesError`, as the rules API reports them. +fn as_rules_error(py: Python<'_>, result: PyResult) -> PyResult { + result.map_err(|error| { + if error.is_instance_of::(py) || error.is_instance_of::(py) { + let message: String = error.value(py).to_string(); + rules_error(py, &message, Some(error)) + } else { + error + } + }) } -/// `left=` and `right=`: the callback is `types_equal`, whose arguments are keyword-only. -fn type_arguments<'py>(py: Python<'py>, left: &str, right: &str) -> PyResult> { - let arguments: Bound<'py, PyDict> = PyDict::new(py); +/// `left=`, `right=` and `dialect=`: `types_equal` takes keyword-only arguments. +fn type_arguments<'py>( + dialect: &Bound<'py, PyAny>, + left: &str, + right: &str, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(dialect.py()); arguments.set_item(LEFT_ARGUMENT, left)?; arguments.set_item(RIGHT_ARGUMENT, right)?; + arguments.set_item(DIALECT_ARGUMENT, dialect)?; Ok(arguments) } fn built_models( - project: &ProjectRowsPy<'_>, + project: &ProjectRows<'_>, request: &EvaluateRequest, rows: Vec, ) -> PyResult> { let mut python_error: Option = None; let fallback = |left: &str, right: &str| -> Result { - match type_arguments(project.types_equal.py(), left, right) + match type_arguments(&project.dialect, left, right) .and_then(|arguments| project.types_equal.call((), Some(&arguments))) .and_then(|equal| equal.extract::()) { @@ -286,7 +431,6 @@ fn fault_identity(fault: &Fault) -> (String, String, u64, u64, String) { ) } -/// A plain Python value as the rules request's JSON value, as orjson with `default=str` encodes it. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(build_rules_request, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index 6797c7f30c..c46b29a069 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -4,7 +4,7 @@ use std::collections::BTreeSet; use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path as bulk_cache_path; use sqlbuild_rules::rule_results::main::group_unevaluated::group_unevaluated; @@ -13,9 +13,9 @@ use sqlbuild_rules::rule_results::models::RuleFinding; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::rules::results::rule_finding; +use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; -const NATIVE_ROWS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.native_rows"; const MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const PATHLIB_MODULE: &str = "pathlib"; const CTYPES_MODULE: &str = "ctypes"; @@ -56,28 +56,22 @@ fn finish_rule_run<'py>( result: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = plan.py(); - let mut evaluated_codes: Vec> = Vec::new(); + let mut evaluated_codes: Vec = Vec::new(); for rule in plan.getattr("selected")?.try_iter()? { - evaluated_codes.push(rule?.getattr("code")?); + evaluated_codes.push(rule?.getattr("code")?.extract()?); } let mut combined: Vec> = Vec::new(); for findings in [sql_result.getattr("findings")?, result.getattr("findings")?] { combined.extend(findings.try_iter()?.collect::>>()?); } - let finalized: Vec = call_with( - &imported(py, NATIVE_ROWS_MODULE, "finalize_rules_rows")?, - &[ - ("config", plan.getattr("config")?), - ("project_dir", plan.getattr("project_dir")?), - ( - "evaluated_codes", - PyTuple::new(py, evaluated_codes)?.into_any(), - ), - ("findings", PyTuple::new(py, combined)?.into_any()), - ], + let finalized: Vec = finalize_rule_findings_rows( + &plan.getattr("config")?, + &plan.getattr("project_dir")?, + evaluated_codes, + combined, )? - .try_iter()? - .map(|finding| rule_finding(&finding?)) + .iter() + .map(|finding| rule_finding(&finding)) .collect::>()?; let grouped: Vec = compiler_guard(|| Ok(group_unevaluated(finalized)))?; let unevaluated: BTreeSet = grouped diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0360bb0f30..83547ed762 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -341,6 +341,7 @@ from sqlbuild.presentation.types import CompletionState from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import ( Finding, + NativeRulesEvaluation, PreparedSqlLint, Rule, RuleRunPlan, @@ -419,14 +420,16 @@ class NativeRulesRequest: def sql_facts_json(self) -> str: ... def build_rules_request( - project: dict[str, object], - models: tuple[NativeCompiledProject, list[str]], - sql_tests: list[CompiledSqlTest], - scenarios: list[CompiledSqlScenario], + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + dialect: str, + initial_findings: tuple[Finding, ...], + custom_payloads: list[dict[str, object]], + include_type_proof: bool, ) -> NativeRulesRequest: ... -def evaluate_rules_request( - request: NativeRulesRequest, -) -> tuple[list[Finding], list[str], int, int, int, int, bool]: ... +def evaluate_rules_request(request: NativeRulesRequest) -> NativeRulesEvaluation: ... def emit_orjson_report(report: object) -> str: ... def emit_json_report(report: object) -> str: ... @@ -2041,7 +2044,13 @@ class PreparedCompileArtifacts: ) -> WrittenTarget | None: ... def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: ... -def finalize_rule_findings_rows(request: dict[str, object]) -> list[Finding]: ... +def finalize_rule_findings_rows( + *, + config: RulesConfig, + project_dir: Path, + evaluated_codes: Sequence[str], + findings: Sequence[Finding], +) -> tuple[Finding, ...]: ... def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py index c017406120..ae211e78c6 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_dispatch.py @@ -8,16 +8,17 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import rule_selected, selected_rule_codes +from sqlbuild._native import ( + build_rules_request, + evaluate_rules_request, + finalize_rule_findings_rows, + rule_selected, + selected_rule_codes, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.run.native_rows import ( - build_rules_request, - evaluate_rules_request, - finalize_rules_rows, -) from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.models import ( CustomRulesOutcome, @@ -110,7 +111,7 @@ def _evaluate_native_rows( *custom.findings, ) if not defer_suppressions: - findings = finalize_rules_rows( + findings = finalize_rule_findings_rows( config=config, project_dir=project_dir, evaluated_codes=native.selected_codes, diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py deleted file mode 100644 index 704ed1f96e..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Hand the native rules request builder rows read from the Python compiled project.""" - -from __future__ import annotations - -from dataclasses import asdict -from functools import partial -from pathlib import Path - -import orjson - -import sqlbuild._native as _native -from sqlbuild._native import ( - compiled_project_facts, - declaration_rows, - rules_bulk_cache_path, - scope_metadata_projection, -) -from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine.constants import ( - NATIVE_RULES_CACHE_FILE, - NATIVE_RULES_MEMO_FILE, - RULES_NATIVE_API_VERSION, -) -from sqlbuild.rule_engine.exceptions import RulesError -from sqlbuild.rule_engine.models import Finding, NativeRulesEvaluation, RulesConfig - - -def build_rules_request( - *, - project: CompiledProjectView, - config: RulesConfig, - project_dir: Path, - dialect: str, - initial_findings: tuple[Finding, ...], - custom_payloads: list[dict[str, object]], - include_type_proof: bool, -) -> _native.NativeRulesRequest: - """Build the built-in rules request natively from the compiled project's rows.""" - - resolved: Path = project_dir.resolve() - header: dict[str, object] = { - "version": RULES_NATIVE_API_VERSION, - "project_dir": str(resolved), - "dialect": dialect, - "config": asdict(config), - "scope_index": scope_metadata_projection(index=project.scope_index), - "defer_suppressions": True, - "custom_rules": custom_payloads, - "rules_cache_path": str( - rules_bulk_cache_path(project_dir=resolved, file_name=NATIVE_RULES_CACHE_FILE) - ), - } - public_enums, public_constants = declaration_rows( - enums=tuple(project.public_enums.values()), - constants=tuple(project.public_constants.values()), - ) - project_rows: dict[str, object] = { - "header_json": _encoded(header), - "memo": ( - ( - compiled_code_identity(), - str( - rules_bulk_cache_path(project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE) - ), - ) - if config.cache.enabled - else None - ), - "sql_analysis_enabled": project.settings.sql_analysis, - "include_type_proof": include_type_proof, - "attached_audit_targets": [ - audit.attached_target_name - for audit in project.audits - if audit.attached_target_name is not None - ], - "public_enums": public_enums, - "public_constants": public_constants, - "initial_findings": list(initial_findings), - "types_equal": partial(types_equal, dialect=dialect), - } - try: - request: _native.NativeRulesRequest - request = _native.build_rules_request( - project_rows, - (compiled_project_facts(project), [model.name for model in project.models]), - list(project.sql_tests), - list(project.sql_scenarios), - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - return request - - -def evaluate_rules_request(request: _native.NativeRulesRequest) -> NativeRulesEvaluation: - """Answer the request from the response memo or evaluate it, decoding findings natively.""" - - try: - faults, selected_codes, evaluated, hits, misses, built_in_ms, reused = ( - _native.evaluate_rules_request(request) - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - return NativeRulesEvaluation( - findings=tuple(faults), - selected_codes=tuple(selected_codes), - evaluated_models=evaluated, - cache_hits=hits, - cache_misses=misses, - built_in_ms=built_in_ms, - reused=reused, - ) - - -def finalize_rules_rows( - *, - config: RulesConfig, - project_dir: Path, - evaluated_codes: tuple[str, ...], - findings: tuple[Finding, ...], -) -> tuple[Finding, ...]: - """Apply the exception policy natively, keeping each original finding's fixable mark.""" - - try: - finalized: list[Finding] = _native.finalize_rule_findings_rows( - { - "project_dir": str(project_dir.resolve()), - "config_json": _encoded(asdict(config)), - "evaluated_codes": list(evaluated_codes), - "findings": list(findings), - } - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - return tuple(finalized) - - -def _encoded(value: object) -> bytes: - try: - return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=str) - except orjson.JSONEncodeError as error: - raise RulesError(str(error)) from error diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 8a61ffa85e..9fc9989506 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -42,7 +42,6 @@ ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.rule_engine._helpers.run import native_rows as native_rows_module from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -67,10 +66,7 @@ } -_FACTS_CONSUMERS: tuple[ModuleType, ...] = ( - native_rows_module, - _native, -) +_FACTS_CONSUMERS: tuple[ModuleType, ...] = (_native,) def forbid_hand_built_projects(*, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 9b054b6833..39123d7c0d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -9,6 +9,7 @@ import pytest +from sqlbuild import _native from sqlbuild._native import ( NativeRulesRequest, build_rule_catalogue, @@ -28,7 +29,6 @@ from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.run import native_rows from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES, RULES_SKILL_PATHS from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( @@ -48,7 +48,8 @@ _MODEL_SQL: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final" -_BUILD_RULES_REQUEST: Callable[..., NativeRulesRequest] = native_rows._native.build_rules_request +_BUILD_RULES_REQUEST: Callable[..., NativeRulesRequest] = _native.build_rules_request +_SCOPE_PROJECTION: Callable[..., dict[str, Any]] = _native.scope_metadata_projection def project_with_scope(*, index: ScopeIndex) -> CompiledProjectView: @@ -70,17 +71,14 @@ def captured_native_request( captured: dict[str, Any] = {} - def build_rules_request( - project_rows: dict[str, Any], - models: tuple[object, list[str]], - sql_tests: list[object], - sql_scenarios: list[object], - ) -> object: - request: NativeRulesRequest = _BUILD_RULES_REQUEST( - project_rows, models, sql_tests, sql_scenarios - ) - captured.update(cast(dict[str, Any], json.loads(project_rows["header_json"]))) - captured.update(models=models, **json.loads(request.sql_facts_json())) + def scope_metadata_projection(**kwargs: Any) -> dict[str, Any]: + projection: dict[str, Any] = _SCOPE_PROJECTION(**kwargs) + captured.update(scope_index=projection) + return projection + + def build_rules_request(**kwargs: Any) -> object: + request: NativeRulesRequest = _BUILD_RULES_REQUEST(**kwargs) + captured.update(cast(dict[str, Any], json.loads(request.sql_facts_json()))) return request def evaluate_rules_request(request: object) -> NativeRulesEvaluation: @@ -94,7 +92,8 @@ def evaluate_rules_request(request: object) -> NativeRulesEvaluation: reused=False, ) - monkeypatch.setattr(native_rows._native, "build_rules_request", build_rules_request) + monkeypatch.setattr(_native, "scope_metadata_projection", scope_metadata_projection) + monkeypatch.setattr(native, "build_rules_request", build_rules_request) monkeypatch.setattr(native, "evaluate_rules_request", evaluate_rules_request) native.evaluate_native( project=project, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py index 141ef13ceb..4a74d73192 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py @@ -2,9 +2,10 @@ import pytest +from sqlbuild import _native from sqlbuild._native import NativeRulesRequest +from sqlbuild.compiler.frontier.main import compiled_code_identity as code_identity from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native -from sqlbuild.rule_engine._helpers.run import native_rows from sqlbuild.rule_engine.models import NativeRulesEvaluation @@ -24,9 +25,7 @@ def record_reuse(*, monkeypatch: pytest.MonkeyPatch) -> list[bool]: """Record whether each natively evaluated request was answered from the response memo.""" reused: list[bool] = [] - evaluate: Callable[[NativeRulesRequest], NativeRulesEvaluation] = ( - native_rows.evaluate_rules_request - ) + evaluate: Callable[[NativeRulesRequest], NativeRulesEvaluation] = _native.evaluate_rules_request def recording(request: NativeRulesRequest) -> NativeRulesEvaluation: evaluation: NativeRulesEvaluation = evaluate(request) @@ -40,5 +39,5 @@ def recording(request: NativeRulesRequest) -> NativeRulesEvaluation: def rebuild_identity(*, monkeypatch: pytest.MonkeyPatch) -> None: """Pretend the compiled code changed, as after installing another build.""" - identity: str = native_rows.compiled_code_identity() - monkeypatch.setattr(native_rows, "compiled_code_identity", lambda: f"{identity}-rebuilt") + identity: str = code_identity.compiled_code_identity() + monkeypatch.setattr(code_identity, "compiled_code_identity", lambda: f"{identity}-rebuilt") From 23ba51e3a3747315e944e859d49ff7be44ce5313 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:22:47 +0100 Subject: [PATCH 28/32] refactor: plan rules runs, run SQL rules and early SQL lint natively --- .../_helpers/compile_driver/analysis.rs | 8 +- .../bindings/_helpers/rules/early_sql_lint.rs | 313 +++++++++ .../src/bindings/_helpers/rules/mod.rs | 3 + .../bindings/_helpers/rules/registration.rs | 9 +- .../src/bindings/_helpers/rules/rule_plan.rs | 228 ++++++ .../src/bindings/_helpers/rules/rule_run.rs | 7 +- .../src/bindings/_helpers/rules/sql_rules.rs | 662 ++++++++++++++++++ src/sqlbuild/_native.pyi | 40 +- .../rule_engine/_helpers/run/rule_threads.py | 16 +- .../rule_engine/_helpers/run/rules.py | 659 ----------------- .../rule_engine/classes/early_sql_lint.py | 59 -- .../sqlbuild/cli/commands/main/test_rules.py | 9 +- 12 files changed, 1268 insertions(+), 745 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs delete mode 100644 src/sqlbuild/rule_engine/_helpers/run/rules.py delete mode 100644 src/sqlbuild/rule_engine/classes/early_sql_lint.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 2deabc7365..b39370a296 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rules_config::load_rules_config; @@ -30,7 +31,6 @@ const SELECTION_MODULE: &str = "sqlbuild._native"; const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const RULE_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; -const EARLY_LINT_MODULE: &str = "sqlbuild.rule_engine.classes.early_sql_lint"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild.cli.compile.models"; @@ -141,11 +141,7 @@ pub(crate) fn analyze<'py>( let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(selection.getattr("select")?.is_truthy()? || selection.getattr("exclude")?.is_truthy()?); - let early_lint_class: Bound<'py, PyAny> = imported(py, EARLY_LINT_MODULE, "EarlySqlLint")?; - let manager: Bound<'py, PyAny> = call_with( - &early_lint_class, - &[("enabled", PyBool::new(py, enabled).to_owned().into_any())], - )?; + let manager: Bound<'py, PyAny> = Bound::new(py, EarlySqlLint::new(py, enabled)?)?.into_any(); within(&manager, |early_lint| analyzed(py, &options, early_lint)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs new file mode 100644 index 0000000000..975e1a0262 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -0,0 +1,313 @@ +//! Speculative SQL checks started once compile inputs are ready, and the lint settings they use. + +use std::path::PathBuf; +use std::time::Instant; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_threads"; +const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_CONSTANTS_MODULE: &str = "sqlbuild.lint.constants"; +const FUTURES_MODULE: &str = "concurrent.futures"; +const THREADING_MODULE: &str = "threading"; +const LINT_THREAD_PREFIX: &str = "sqlbuild-lint"; +const SQL_RULES_CACHE_FILE: &str = "sql.json"; + +/// The SQL rule result cache file of `project_dir`. +pub(crate) fn sql_rule_cache_path<'py>( + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + let root: PathBuf = project_dir.extract()?; + Ok(rules_bulk_cache_path(&root, SQL_RULES_CACHE_FILE) + .into_pyobject(project_dir.py())? + .into_any()) +} + +/// The native SQL lint configuration that SQL rules run with. +pub(crate) fn sql_lint_config<'py>( + dialect: &Bound<'py, PyAny>, + codes: &[String], + thresholds: &Bound<'py, PyAny>, + (relation_keys, allow_suppressions): (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = dialect.py(); + let threshold = |key: &str, default: &str| -> PyResult> { + thresholds.call_method1( + "get", + ( + imported(py, LINT_CONSTANTS_MODULE, key)?, + imported(py, LINT_CONSTANTS_MODULE, default)?, + ), + ) + }; + call_with( + &imported(py, LINT_MODELS_MODULE, "LintConfig")?, + &[ + ("dialect", dialect.clone()), + ("enabled_native_rules", PyTuple::new(py, codes)?.into_any()), + ("header_rules_enabled", false.into_bound_py_any(py)?), + ( + "max_literal_length", + threshold("MAX_LITERAL_LENGTH_THRESHOLD", "DEFAULT_MAX_LITERAL_LENGTH")?, + ), + ( + "max_ranking_order_by", + threshold( + "MAX_RANKING_ORDER_BY_THRESHOLD", + "DEFAULT_MAX_RANKING_ORDER_BY", + )?, + ), + ("relation_keys", relation_keys), + ("allow_suppressions", allow_suppressions), + ], + ) +} + +/// The project, SQL rule codes and lint settings early SQL lint runs with, if it should run. +#[pyfunction] +#[pyo3(signature = (*, inputs, dialect))] +fn sql_lint_preparation<'py>( + inputs: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = inputs.py(); + let native = |name: &str| imported(py, NATIVE_MODULE, name); + let project_dir: Bound<'py, PyAny> = inputs + .getattr("discovered_inputs")? + .getattr("project_dir")?; + if project_dir.is_none() { + return Ok(None); + } + for diagnostic in inputs.getattr("diagnostics")?.try_iter()? { + if diagnostic?.getattr("is_error")?.is_truthy()? { + return Ok(None); + } + } + let config: Bound<'py, PyAny> = call_with( + &native("valid_rules_config")?, + &[("project_dir", project_dir.clone())], + )?; + if config.is_none() || !config.getattr("select")?.is_truthy()? { + return Ok(None); + } + if config.getattr("cache")?.getattr("enabled")?.is_truthy()? + && call_with( + &native("load_sql_rule_cache")?, + &[("path", sql_rule_cache_path(&project_dir)?)], + )? + .is_truthy()? + { + return Ok(None); + } + let codes: Vec = call_with( + &native("selected_sql_rule_codes")?, + &[ + ( + "select", + PyList::new( + py, + config + .getattr("select")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ( + "ignore", + PyList::new( + py, + config + .getattr("ignore")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ], + )? + .extract()?; + if codes.is_empty() { + return Ok(None); + } + let mut model_rows: Vec> = Vec::new(); + for model in inputs.getattr("model_inputs")?.try_iter()? { + model_rows.push(model?.getattr("relation_key_row")?); + } + let mut seeds: Vec> = Vec::new(); + for seed in inputs.getattr("seed_inputs")?.try_iter()? { + seeds.push(seed?.getattr("schema_entry")?); + } + let mut sources: Vec> = Vec::new(); + for source in inputs.getattr("source_inputs")?.try_iter()? { + sources.push(source?.getattr("source_entry")?); + } + let relation_keys: Bound<'py, PyAny> = call_with( + &native("declared_relation_keys")?, + &[ + ("model_rows", PyList::new(py, model_rows)?.into_any()), + ("seeds", PyList::new(py, seeds)?.into_any()), + ("sources", PyList::new(py, sources)?.into_any()), + ], + )?; + let lint_config: Bound<'py, PyAny> = sql_lint_config( + dialect, + &codes, + &config.getattr("thresholds")?, + (relation_keys, config.getattr("allow_exceptions")?), + )?; + Ok(Some(PyTuple::new( + py, + [ + project_dir, + PyTuple::new(py, codes)?.into_any(), + lint_config, + ], + )?)) +} + +/// Each model's compile expansion for early lint, whole when the compile handed one over. +#[pyfunction] +fn early_lint_expansions<'py>(model_inputs: &Bound<'py, PyAny>) -> PyResult> { + let expansions: Bound<'py, PyDict> = PyDict::new(model_inputs.py()); + for model in model_inputs.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let mut expansion: Bound<'py, PyAny> = model.getattr("lint_expansion")?; + if !expansion.is_truthy()? { + expansion = model.getattr("sql_expansion")?; + } + if !expansion.is_none() { + expansions.set_item( + model.getattr("model_file")?.getattr("file_path")?, + expansion, + )?; + } + } + Ok(expansions) +} + +/// Own speculative SQL checks for one compilation invocation. +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct EarlySqlLint { + #[pyo3(get)] + enabled: bool, + #[pyo3(get)] + preparation: Py, + #[pyo3(get)] + expansion_reuse: Py, + stop_requested: Py, + executor: Py, +} + +impl EarlySqlLint { + /// A manager that starts speculative checks only when `enabled`. + pub(crate) fn new(py: Python<'_>, enabled: bool) -> PyResult { + let executor: Bound<'_, PyAny> = call_with( + &imported(py, FUTURES_MODULE, "ThreadPoolExecutor")?, + &[ + ("max_workers", 1.into_bound_py_any(py)?), + ( + "thread_name_prefix", + LINT_THREAD_PREFIX.into_bound_py_any(py)?, + ), + ], + )?; + Ok(Self { + enabled, + preparation: py.None(), + expansion_reuse: py.None(), + stop_requested: imported(py, THREADING_MODULE, "Event")?.call0()?.unbind(), + executor: executor.unbind(), + }) + } +} + +#[pymethods] +impl EarlySqlLint { + #[new] + #[pyo3(signature = (*, enabled))] + fn py_new(py: Python<'_>, enabled: bool) -> PyResult { + Self::new(py, enabled) + } + + fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { + slf + } + + #[pyo3(signature = (*_exception))] + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { + let _ = self.stop(py)?; + Ok(()) + } + + /// Start early SQL lint for `inputs` when enabled and the project selects SQL rules. + #[pyo3(signature = (*, inputs, dialect))] + fn start( + mut slf: PyRefMut<'_, Self>, + inputs: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, + ) -> PyResult<()> { + if !slf.enabled { + return Ok(()); + } + let py: Python<'_> = inputs.py(); + let preparation: Bound<'_, PyAny> = call_with( + &imported(py, RULE_THREADS_MODULE, "prepare_sql_rules")?, + &[ + ("inputs", inputs.clone()), + ("executor", slf.executor.bind(py).clone()), + ("dialect", dialect.clone()), + ("stop", slf.stop_requested.bind(py).clone()), + ], + )?; + let declaration_scope: Bound<'_, PyAny> = inputs.getattr("declaration_scope")?; + let expansion_reuse: Bound<'_, PyAny> = if declaration_scope.is_none() { + declaration_scope + } else { + call_with( + &imported(py, RULES_MODELS_MODULE, "SqlExpansionReuse")?, + &[ + ("discovered_inputs", inputs.getattr("discovered_inputs")?), + ("declaration_scope", declaration_scope), + ], + )? + }; + slf.preparation = preparation.unbind(); + slf.expansion_reuse = expansion_reuse.unbind(); + Ok(()) + } + + /// Abandon unconsumed speculative work and return the milliseconds spent joining it. + fn stop(&self, py: Python<'_>) -> PyResult { + let started: Instant = Instant::now(); + let _ = self.stop_requested.bind(py).call_method0("set")?; + let _ = self.executor.bind(py).call_method( + "shutdown", + (), + Some(&keywords( + py, + &[ + ("wait", true.into_bound_py_any(py)?), + ("cancel_futures", true.into_bound_py_any(py)?), + ], + )?), + )?; + Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(sql_lint_preparation, module)?)?; + module.add_function(wrap_pyfunction!(early_lint_expansions, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 13cae7439d..b5ef88405a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod catalogue; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; +pub(crate) mod early_sql_lint; pub(crate) mod evaluation; pub(crate) mod findings; pub(crate) mod guidance; @@ -15,9 +16,11 @@ pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; pub(crate) mod rule_cache; +pub(crate) mod rule_plan; pub(crate) mod rule_run; pub(crate) mod rules_config; pub(crate) mod run_lint; pub(crate) mod selection; pub(crate) mod skills; +pub(crate) mod sql_rules; pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 27a85fab45..7d9f87a4b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -3,9 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ - builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, - guidance, lint, lint_expansion, lint_sql_bodies, lint_text, results, rule_cache, rule_run, - rules_config, run_lint, selection, + builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, + early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_text, + results, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -18,6 +18,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { custom_rule_evidence::register(module)?; rule_cache::register(module)?; rule_run::register(module)?; + rule_plan::register(module)?; + sql_rules::register(module)?; + early_sql_lint::register(module)?; custom_rule_cache::register(module)?; builtin_catalogue::register(module)?; catalogue::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs new file mode 100644 index 0000000000..64d111f236 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -0,0 +1,228 @@ +//! Plan one rules run: the catalogue, selected rules, project slice and settings it evaluates. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyFrozenSet, PySet, PySetMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::catalogue::{ + references_custom_rules, resolved_ignore_selectors, rule_catalogue, select_rules, + verify_custom_rules, +}; +use crate::bindings::_helpers::rules::rule_run::freed_memory_released; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; +const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; +const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const PATHLIB_MODULE: &str = "pathlib"; + +/// Build the catalogue and select the rules, project slice and settings one run evaluates. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_rule_run<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> { + let options = Request::new("plan_rule_run", options)?; + let py: Python<'py> = options.py(); + let graph: Bound<'py, PyAny> = options.get("graph")?; + let project: Bound<'py, PyAny> = graph.getattr("project")?; + freed_memory_released(py); + let effective_config: Bound<'py, PyAny> = + resolved_ignore_selectors(&options.get("config")?, &project)?; + let project_dir: Bound<'py, PyAny> = options.get("project_dir")?.call_method0("resolve")?; + let include_custom: bool = references_custom_rules(&effective_config)?; + let catalogue: Vec> = + rule_catalogue(&effective_config, &project_dir, include_custom)?; + let catalogue_tuple: Bound<'py, PyTuple> = PyTuple::new(py, &catalogue)?; + let custom_payloads: Bound<'py, PyAny> = call_with( + &imported(py, CUSTOM_PAYLOADS_MODULE, "custom_rule_payloads")?, + &[ + ("catalogue", catalogue_tuple.clone().into_any()), + ("project_dir", project_dir.clone()), + ], + )?; + let mut selected: Vec> = select_rules( + &catalogue, + &effective_config, + &project_dir, + Some(custom_payloads.clone()), + )?; + let mut skipped_type_proof_rules: Vec> = Vec::new(); + if options.get("no_sql_analysis")?.is_truthy()? + || !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + let type_proof: Bound<'py, PyAny> = + imported(py, RULES_CONSTANTS_MODULE, "TYPE_PROOF_RULE_CODES")?; + let mut kept: Vec> = Vec::with_capacity(selected.len()); + for rule in selected { + let code: Bound<'py, PyAny> = rule.getattr("code")?; + if type_proof.contains(&code)? { + skipped_type_proof_rules.push(code); + } else { + kept.push(rule); + } + } + selected = kept; + } + let mut custom_rules: Vec> = Vec::new(); + for rule in &selected { + if rule.getattr("custom")?.is_truthy()? { + custom_rules.push(rule.clone()); + } + } + if !custom_rules.is_empty() { + verify_custom_rules(&custom_rules, &project_dir)?; + } + let model_paths: Option> = + selected_model_paths(&graph, &options.get("selected_keys")?)?; + let selected_project: Bound<'py, PyAny> = match &model_paths { + None => project.clone(), + Some(paths) => selected_models_project(&project, paths)?, + }; + let mut codes: Vec> = Vec::with_capacity(selected.len()); + for rule in &selected { + codes.push(rule.getattr("code")?); + } + let selection_config: Bound<'py, PyAny> = + selection_rules_config(&effective_config, model_paths.as_ref(), &selected_project)?; + let selected_config: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?.call( + (selection_config,), + Some(&keywords( + py, + &[ + ("select", PyTuple::new(py, codes)?.into_any()), + ("ignore", PyTuple::empty(py).into_any()), + ], + )?), + )?; + call_with( + &imported(py, RULES_MODELS_MODULE, "RuleRunPlan")?, + &[ + ("effective_config", effective_config), + ("project_dir", project_dir), + ("catalogue", catalogue_tuple.into_any()), + ("custom_payloads", custom_payloads), + ("selected", PyTuple::new(py, selected)?.into_any()), + ( + "skipped_type_proof_rules", + PyTuple::new(py, skipped_type_proof_rules)?.into_any(), + ), + ( + "model_paths", + model_paths.map_or_else(|| py.None().into_bound(py), |paths| paths.into_any()), + ), + ("project", selected_project), + ("config", selected_config), + ], + ) +} + +/// The relative paths of the selected models, or `None` when the run evaluates every model. +fn selected_model_paths<'py>( + graph: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = graph.py(); + if selected_keys.is_none() { + return Ok(None); + } + let model_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let names: Bound<'py, PySet> = PySet::empty(py)?; + for key in selected_keys.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if key.getattr("resource_type")?.is(&model_type) { + names.add(key.getattr("name")?)?; + } + } + let paths: Bound<'py, PySet> = PySet::empty(py)?; + for model in graph.getattr("project")?.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + if names.contains(model.getattr("name")?)? { + paths.add(model.getattr("relative_path")?.call_method0("as_posix")?)?; + } + } + Ok(Some(PyFrozenSet::new(py, paths.iter())?)) +} + +/// `project` keeping only the models whose relative path is in `model_paths`. +fn selected_models_project<'py>( + project: &Bound<'py, PyAny>, + model_paths: &Bound<'py, PyFrozenSet>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let mut models: Vec> = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + if model_paths.contains(model.getattr("relative_path")?.call_method0("as_posix")?)? { + models.push(model); + } + } + call_with( + &imported(py, NATIVE_MODULE, "project_with")?, + &[ + ("project", project.clone()), + ("models", PyTuple::new(py, models)?.into_any()), + ], + ) +} + +/// `config` keeping only the exceptions that apply to the selected models. +fn selection_rules_config<'py>( + config: &Bound<'py, PyAny>, + model_paths: Option<&Bound<'py, PyFrozenSet>>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = config.py(); + let Some(model_paths) = model_paths else { + return Ok(config.clone()); + }; + let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let model_names: Bound<'py, PySet> = PySet::empty(py)?; + for model in project.getattr("models")?.try_iter()? { + model_names.add(model?.getattr("name")?)?; + } + let mut rule_exceptions: Vec> = Vec::new(); + for entry in config.getattr("rule_exceptions")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let path: Bound<'py, PyAny> = path_class + .call1((entry.getattr("path")?,))? + .call_method0("as_posix")?; + if model_paths.contains(path)? { + rule_exceptions.push(entry); + } + } + let mut graph_edge_exceptions: Vec> = Vec::new(); + for entry in config.getattr("graph_edge_exceptions")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if model_names.contains(entry.getattr("consumer")?)? { + graph_edge_exceptions.push(entry); + } + } + imported(py, DATACLASSES_MODULE, "replace")?.call( + (config,), + Some(&keywords( + py, + &[ + ( + "rule_exceptions", + PyTuple::new(py, rule_exceptions)?.into_any(), + ), + ( + "graph_edge_exceptions", + PyTuple::new(py, graph_edge_exceptions)?.into_any(), + ), + ], + )?), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_rule_run, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index c46b29a069..d9406540f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -143,10 +143,9 @@ fn rules_bulk_cache_path(project_dir: PathBuf, file_name: &str) -> PathBuf { } /// Hand freed glibc heap pages back to the OS; a no-op elsewhere or without `malloc_trim`. -#[pyfunction] -fn release_freed_memory(py: Python<'_>) -> PyResult<()> { +pub(crate) fn freed_memory_released(py: Python<'_>) { if !cfg!(target_os = "linux") { - return Ok(()); + return; } let trim = || -> PyResult<()> { let process: Bound<'_, PyAny> = imported(py, CTYPES_MODULE, "CDLL")?.call1((py.None(),))?; @@ -154,13 +153,11 @@ fn release_freed_memory(py: Python<'_>) -> PyResult<()> { Ok(()) }; let _ = trim(); - Ok(()) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(finish_rule_run, module)?)?; module.add_function(wrap_pyfunction!(run_rules, module)?)?; module.add_function(wrap_pyfunction!(rules_bulk_cache_path, module)?)?; - module.add_function(wrap_pyfunction!(release_freed_memory, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs new file mode 100644 index 0000000000..5eac95993f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -0,0 +1,662 @@ +//! Run the planned SQL rules through native lint, reusing cached and early-lint results. + +use std::collections::BTreeSet; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, + PyTuple, +}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule_cache_path}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; +const LINT_CONSTANTS_MODULE: &str = "sqlbuild.lint.constants"; +const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const JSON_MODULE: &str = "json"; +const BUILTINS_MODULE: &str = "builtins"; +const SQL_RULE_PREFIX: &str = "SQBRSQL"; +const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSQL000"; +const DEFAULT_REMEDIATION: &str = "Update the authored SQL to satisfy this rule."; + +/// `native.name`, looked up at call time. +fn native<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, NATIVE_MODULE, name) +} + +/// The SQL rule codes among `rules`, in order. +fn sql_rule_codes(rules: &[Bound<'_, PyAny>]) -> PyResult> { + let mut codes: Vec = Vec::new(); + for rule in rules { + let code: String = rule.getattr("code")?.extract()?; + if code.starts_with(SQL_RULE_PREFIX) { + codes.push(code); + } + } + Ok(codes) +} + +/// The settings, project and inputs one SQL rules evaluation reads. +struct SqlRun<'a, 'py> { + codes: &'a [String], + project_dir: Bound<'py, PyAny>, + discovered_inputs: Bound<'py, PyAny>, + project: Bound<'py, PyAny>, + lint_config: Bound<'py, PyAny>, + selected_model_paths: Bound<'py, PyAny>, + cache_enabled: bool, + prepared_sql: Bound<'py, PyAny>, + expansion_reuse: Bound<'py, PyAny>, +} + +/// Run the planned SQL rules on the caller while built-in rules evaluate natively. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult> { + let options = Request::new("run_planned_sql_rules", options)?; + let py: Python<'py> = options.py(); + let plan: Bound<'py, PyAny> = options.get("plan")?; + let graph: Bound<'py, PyAny> = options.get("graph")?; + let effective_config: Bound<'py, PyAny> = plan.getattr("effective_config")?; + let mut native_rules: Vec> = Vec::new(); + for rule in plan.getattr("selected")?.try_iter()? { + let rule: Bound<'py, PyAny> = rule?; + if !rule.getattr("custom")?.is_truthy()? { + native_rules.push(rule); + } + } + let codes: Vec = sql_rule_codes(&native_rules)?; + let lint_config: Bound<'py, PyAny> = sql_lint_config( + &options.get("dialect")?, + &codes, + &effective_config.getattr("thresholds")?, + ( + native(py, "compiled_relation_keys")?.call1((graph.getattr("project")?,))?, + effective_config.getattr("allow_exceptions")?, + ), + )?; + let run = SqlRun { + codes: &codes, + project_dir: plan.getattr("project_dir")?, + discovered_inputs: options.get("discovered_inputs")?, + project: plan.getattr("project")?, + lint_config, + selected_model_paths: plan.getattr("model_paths")?, + cache_enabled: effective_config + .getattr("cache")? + .getattr("enabled")? + .is_truthy()?, + prepared_sql: options.get("prepared_sql")?, + expansion_reuse: options.get("expansion_reuse")?, + }; + let lint_key: String = lint_key(&run.lint_config)?; + let identities: Vec<(String, String)> = model_rule_identities(&run, &lint_key)?; + let project_files: Option> = if !identities.is_empty() + && run.prepared_sql.is_none() + && run.selected_model_paths.is_none() + { + Some(collected_project_files(&run.project_dir)?) + } else { + None + }; + let started: Instant = Instant::now(); + let (findings, hits, misses) = + evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; + let sql_ms: i64 = (started.elapsed().as_secs_f64() * 1000.0).round_ties_even() as i64; + call_with( + &imported(py, RULES_MODELS_MODULE, "SqlRulesEvaluation")?, + &[ + ("findings", findings), + ("cache_hits", hits.into_bound_py_any(py)?), + ("cache_misses", misses.into_bound_py_any(py)?), + ("sql_ms", sql_ms.into_bound_py_any(py)?), + ], + ) +} + +fn lint_key(lint_config: &Bound<'_, PyAny>) -> PyResult { + call_with( + &native(lint_config.py(), "sql_rule_lint_key")?, + &[("config", lint_config.clone())], + )? + .extract() +} + +fn collected_project_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project_dir.py(); + call_with( + &native(py, "collect_lint_project_files")?, + &[ + ("project_dir", project_dir.clone()), + ("selected_paths", py.None().into_bound(py)), + ], + ) +} + +/// The project's models SQL rules analyse, keyed by relative path in project order. +fn sql_rule_models<'py>(project: &Bound<'py, PyAny>) -> PyResult)>> { + let mut models: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(models); + } + let excluded: Bound<'py, PyAny> = false.into_bound_py_any(project.py())?; + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let analysis: Bound<'py, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", ("sql_analysis",))?; + if analysis.is(&excluded) { + continue; + } + let path: String = model + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?; + if let Some(index) = models.iter().position(|(known, _)| *known == path) { + models[index].1 = model; + } else { + models.push((path, model)); + } + } + Ok(models) +} + +/// Each analysed model's SQL rule cache key, in project order. +fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult> { + let py: Python<'_> = run.project.py(); + if run.codes.is_empty() { + return Ok(Vec::new()); + } + let mut models: Vec> = Vec::new(); + let mut paths: Vec = Vec::new(); + for (path, model) in sql_rule_models(&run.project)? { + models.push(PyTuple::new( + py, + [ + PyString::new(py, &path).into_any(), + model.getattr("authored_sql")?, + model.getattr("query_sql")?, + contract_text(&model)?, + ], + )?); + paths.push(path); + } + let keys: Bound<'_, PyAny> = call_with( + &native(py, "sql_model_rule_keys")?, + &[ + ("lint_key", PyString::new(py, lint_key).into_any()), + ("codes", PyList::new(py, run.codes)?.into_any()), + ("models", PyList::new(py, models)?.into_any()), + ], + )?; + paths + .into_iter() + .map(|path| { + let key: String = keys.get_item(&path)?.extract()?; + Ok((path, key)) + }) + .collect() +} + +/// Canonical text of the dynamic column contract that SQL rule findings depend on. +fn contract_text<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let contract: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; + if contract.is_none() { + return Ok(contract); + } + imported(py, JSON_MODULE, "dumps")?.call( + (imported(py, DATACLASSES_MODULE, "asdict")?.call1((contract,))?,), + Some(&keywords( + py, + &[ + ("sort_keys", true.into_bound_py_any(py)?), + ("separators", (",", ":").into_bound_py_any(py)?), + ("default", imported(py, BUILTINS_MODULE, "str")?), + ], + )?), + ) +} + +/// The stored findings of every path whose key still matches; undecodable ones miss. +fn cached_findings<'py>( + bucket: &Bound<'py, PyAny>, + keys: Vec<(String, String)>, +) -> PyResult> { + let py: Python<'py> = bucket.py(); + Ok(call_with( + &native(py, "lookup_sql_rule_cache")?, + &[ + ("entries", bucket.clone()), + ("keys", keys.into_bound_py_any(py)?), + ], + )? + .downcast_into()?) +} + +/// The early SQL lint's full-project file read, when it ran for this project and these codes. +fn prepared_project_files<'py>(run: &SqlRun<'_, 'py>) -> PyResult>> { + let py: Python<'py> = run.project_dir.py(); + let prepared: &Bound<'py, PyAny> = &run.prepared_sql; + if prepared.is_none() + || prepared.getattr("project_dir")?.ne(&run.project_dir)? + || !same_codes(&prepared.getattr("codes")?, run.codes)? + || !prepared + .getattr("future")? + .call_method0("exception")? + .is_none() + { + return Ok(None); + } + let texts: Bound<'py, PyAny> = prepared + .getattr("future")? + .call_method0("result")? + .getattr("result")? + .getattr("source_texts")?; + Ok(Some( + imported(py, BUILTINS_MODULE, "dict")?.call1((texts,))?, + )) +} + +fn same_codes(codes: &Bound<'_, PyAny>, expected: &[String]) -> PyResult { + let mut known: BTreeSet = BTreeSet::new(); + if !codes.is_none() { + for code in codes.try_iter()? { + known.insert(code?.extract()?); + } + } + Ok(known == expected.iter().cloned().collect()) +} + +/// Each analysed model's findings, cached or linted, then cache accounting. +fn evaluated_sql_rules<'py>( + run: &SqlRun<'_, 'py>, + (lint_key, identities): (&str, Vec<(String, String)>), + prepared_files: Option>, +) -> PyResult<(Bound<'py, PyAny>, usize, usize)> { + let py: Python<'py> = run.project_dir.py(); + if run.codes.is_empty() { + return Ok((PyTuple::empty(py).into_any(), 0, 0)); + } + let project_dir: &Bound<'py, PyAny> = &run.project_dir; + let models: Vec<(String, Bound<'py, PyAny>)> = sql_rule_models(&run.project)?; + let cache_path: Bound<'py, PyAny> = sql_rule_cache_path(project_dir)?; + let bucket: Bound<'py, PyAny> = if run.cache_enabled { + call_with( + &native(py, "load_sql_rule_cache")?, + &[("path", cache_path.clone())], + )? + } else { + PyDict::new(py).into_any() + }; + let findings: Bound<'py, PyList> = PyList::empty(py); + let model_hits: Bound<'py, PyDict> = cached_findings(&bucket, identities.clone())?; + let mut misses: Vec<(String, String)> = Vec::new(); + for (path, identity) in &identities { + match model_hits.get_item(path)? { + Some(hit) => { + for finding in hit.try_iter()? { + findings.append(finding?)?; + } + } + None => misses.push((path.clone(), identity.clone())), + } + } + let selected_paths: Bound<'py, PySet> = PySet::empty(py)?; + for (path, _) in &misses { + selected_paths.add(project_dir.div(path)?)?; + } + let mut file_identities: Vec<(String, String)> = Vec::new(); + let mut project_files: Bound<'py, PyAny> = py.None().into_bound(py); + if run.selected_model_paths.is_none() { + let model_paths: Bound<'py, PySet> = PySet::empty(py)?; + for model in run.project.getattr("models")?.try_iter()? { + model_paths.add(project_dir.div(model?.getattr("relative_path")?)?)?; + } + project_files = match prepared_project_files(run)? { + Some(files) => files, + None => match prepared_files { + Some(files) => files, + None => collected_project_files(project_dir)?, + }, + }; + let expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; + let mut file_keys: Vec<(Bound<'py, PyAny>, String, Option)> = Vec::new(); + for item in project_files.downcast::()?.iter() { + let (file_path, contents) = item; + if model_paths.contains(&file_path)? { + continue; + } + let relative_path: String = file_path + .call_method1("relative_to", (project_dir,))? + .call_method0("as_posix")? + .extract()?; + let key: Option = call_with( + &native(py, "sql_file_rule_key")?, + &[ + ("lint_key", PyString::new(py, lint_key).into_any()), + ("codes", PyList::new(py, run.codes)?.into_any()), + ( + "relative_path", + PyString::new(py, &relative_path).into_any(), + ), + ("contents", contents), + ( + "expanded", + expansions.contains(&file_path)?.into_bound_py_any(py)?, + ), + ], + )? + .extract()?; + file_keys.push((file_path, relative_path, key)); + } + let mut keyed: Vec<(String, String)> = Vec::new(); + for (_, path, key) in &file_keys { + if let Some(key) = key { + keyed.push((path.clone(), key.clone())); + } + } + let file_hits: Bound<'py, PyDict> = cached_findings(&bucket, keyed)?; + for (file_path, relative_path, identity) in file_keys { + match file_hits.get_item(&relative_path)? { + Some(hit) => { + for finding in hit.try_iter()? { + findings.append(finding?)?; + } + } + None => { + selected_paths.add(file_path)?; + if let Some(identity) = identity { + file_identities.push((relative_path, identity)); + } + } + } + } + } + let violations: Vec> = if selected_paths.is_empty() { + Vec::new() + } else { + let mut dynamic: Vec> = Vec::new(); + for (_, model) in &models { + let contract: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; + if !contract.is_none() && contract.getattr("output_proven")?.is_truthy()? { + dynamic.push( + project_dir + .div(model.getattr("relative_path")?)? + .call_method0("resolve")?, + ); + } + } + prepared_lint_violations( + run, + &PyFrozenSet::new(py, selected_paths.iter())?, + &PyFrozenSet::new(py, dynamic)?, + &project_files, + )? + }; + let model_paths: BTreeSet = models.into_iter().map(|(path, _)| path).collect(); + let evaluated: Vec> = violation_findings(run, &violations, &model_paths)?; + for finding in &evaluated { + findings.append(finding)?; + } + let mut written: Vec<(String, String)> = misses.clone(); + written.extend(file_identities); + if run.cache_enabled && !written.is_empty() { + write_cache(&bucket, (&cache_path, written), &evaluated)?; + } + let hinted: Bound<'py, PyAny> = call_with( + &native(py, "with_duplicate_literal_hints")?, + &[ + ("findings", PyTuple::new(py, findings.iter())?.into_any()), + ("project_dir", project_dir.clone()), + ], + )?; + Ok((hinted, identities.len() - misses.len(), misses.len())) +} + +/// Store each rewritten path's evaluated findings in `bucket` and replace the cache file. +fn write_cache<'py>( + bucket: &Bound<'py, PyAny>, + (cache_path, written): (&Bound<'py, PyAny>, Vec<(String, String)>), + evaluated: &[Bound<'py, PyAny>], +) -> PyResult<()> { + let py: Python<'py> = bucket.py(); + let encode: Bound<'py, PyAny> = native(py, "encode_sql_rule_findings")?; + for (path, identity) in written { + let mut own: Vec> = Vec::new(); + for finding in evaluated { + let finding_path: String = finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?; + if finding_path == path { + own.push(finding.clone()); + } + } + let encoded: Bound<'py, PyAny> = + call_with(&encode, &[("findings", PyList::new(py, own)?.into_any())])?; + bucket.set_item(path, (identity, encoded))?; + } + let _ = call_with( + &native(py, "write_sql_rule_cache")?, + &[("path", cache_path.clone()), ("entries", bucket.clone())], + )?; + Ok(()) +} + +/// Lint violations of `selected_paths`, reusing a matching early SQL lint. +fn prepared_lint_violations<'py>( + run: &SqlRun<'_, 'py>, + selected_paths: &Bound<'py, PyFrozenSet>, + dynamic_output_paths: &Bound<'py, PyFrozenSet>, + source_files: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = run.project_dir.py(); + let run_lint: Bound<'py, PyAny> = native(py, "run_lint")?; + let compiled_expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; + let enabled_codes: Bound<'py, PyAny> = run.lint_config.getattr("enabled_native_rules")?; + let enabled: Vec = if enabled_codes.is_none() { + Vec::new() + } else { + enabled_codes.extract()? + }; + if run.prepared_sql.is_none() || !same_codes(&run.prepared_sql.getattr("codes")?, &enabled)? { + let expansion_context: Bound<'py, PyAny> = + if run.lint_config.getattr("native_enabled")?.is_truthy()? { + lint_expansion_context(run)? + } else { + py.None().into_bound(py) + }; + return call_with( + &run_lint, + &[ + ("project_dir", run.project_dir.clone()), + ("config", run.lint_config.clone()), + ("selected_paths", selected_paths.clone().into_any()), + ("discovered_inputs", run.discovered_inputs.clone()), + ("compiled_expansions", compiled_expansions), + ( + "dynamic_output_paths", + dynamic_output_paths.clone().into_any(), + ), + ("expansion_context", expansion_context), + ("source_files", source_files.clone()), + ], + )? + .getattr("violations")? + .try_iter()? + .collect(); + } + let completed: Bound<'py, PyAny> = + run.prepared_sql.getattr("future")?.call_method0("result")?; + let prepared: Bound<'py, PyAny> = completed.getattr("result")?; + let proof_paths: Bound<'py, PyAny> = + selected_paths.call_method1("__and__", (dynamic_output_paths,))?; + let mut violations: Vec> = Vec::new(); + for violation in prepared.getattr("violations")?.try_iter()? { + let violation: Bound<'py, PyAny> = violation?; + let path: Bound<'py, PyAny> = violation.getattr("file_path")?; + if selected_paths.contains(&path)? && !proof_paths.contains(&path)? { + violations.push(violation); + } + } + if proof_paths.is_truthy()? { + let proven: Bound<'py, PyAny> = call_with( + &run_lint, + &[ + ("project_dir", run.project_dir.clone()), + ("config", run.lint_config.clone()), + ("selected_paths", proof_paths.clone()), + ("discovered_inputs", run.discovered_inputs.clone()), + ("compiled_expansions", compiled_expansions), + ("dynamic_output_paths", proof_paths), + ("expansion_context", completed.getattr("context")?), + ( + "source_files", + imported(py, BUILTINS_MODULE, "dict")? + .call1((prepared.getattr("source_texts")?,))?, + ), + ], + )?; + for violation in proven.getattr("violations")?.try_iter()? { + violations.push(violation?); + } + } + Ok(violations) +} + +/// Build lint expansion, reusing the compiler's scope only for the same discovery. +fn lint_expansion_context<'py>(run: &SqlRun<'_, 'py>) -> PyResult> { + let py: Python<'py> = run.project_dir.py(); + let reuse: &Bound<'py, PyAny> = &run.expansion_reuse; + let declaration_scope: Bound<'py, PyAny> = if !reuse.is_none() + && reuse + .getattr("discovered_inputs")? + .is(&run.discovered_inputs) + { + reuse.getattr("declaration_scope")? + } else { + py.None().into_bound(py) + }; + call_with( + &native(py, "build_lint_expansion_context")?, + &[ + ("project_dir", run.project_dir.clone()), + ("discovered_inputs", run.discovered_inputs.clone()), + ("declaration_scope", declaration_scope), + ], + ) +} + +/// The findings of the selected SQL rules' violations; an engine failure leaves each unevaluated. +fn violation_findings<'py>( + run: &SqlRun<'_, 'py>, + violations: &[Bound<'py, PyAny>], + model_paths: &BTreeSet, +) -> PyResult>> { + let py: Python<'py> = run.project_dir.py(); + let engine_error: String = imported(py, LINT_EXCEPTIONS_MODULE, "NativeLintError")? + .getattr("code")? + .extract()?; + let safe_fixes: Bound<'py, PyAny> = imported(py, LINT_CONSTANTS_MODULE, "SAFE_RULE_FIX_CODES")?; + let resolved: Bound<'py, PyAny> = run.project_dir.call_method0("resolve")?; + let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let no_paths: BTreeSet = BTreeSet::new(); + let mut evaluated: Vec> = Vec::new(); + for violation in violations { + let code: String = violation.getattr("code")?.extract()?; + if code == SQL_RULE_SUPPRESSION_CODE || run.codes.contains(&code) { + evaluated.push(lint_finding( + violation, + (&resolved, &safe_fixes), + model_paths, + )?); + } else if code == engine_error { + let finding: Bound<'py, PyAny> = + lint_finding(violation, (&resolved, &safe_fixes), &no_paths)?; + let message: String = violation.getattr("message")?.extract()?; + for rule_code in run.codes { + evaluated.push( + replace.call( + (&finding,), + Some(&keywords( + py, + &[ + ("code", PyString::new(py, rule_code).into_any()), + ( + "message", + PyString::new( + py, + &format!( + "Rule {rule_code} could not be evaluated: {message}" + ), + ) + .into_any(), + ), + ("unevaluated", true.into_bound_py_any(py)?), + ], + )?), + )?, + ); + } + } + } + Ok(evaluated) +} + +/// One lint violation as a finding; fixable when `format --fix` would attempt its edit. +fn lint_finding<'py>( + violation: &Bound<'py, PyAny>, + (resolved_dir, safe_fixes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + fixable_paths: &BTreeSet, +) -> PyResult> { + let py: Python<'py> = violation.py(); + let mut path: Bound<'py, PyAny> = violation.getattr("file_path")?; + if path.call_method0("is_absolute")?.is_truthy()? + && path + .call_method1("is_relative_to", (resolved_dir,))? + .is_truthy()? + { + path = path.call_method1("relative_to", (resolved_dir,))?; + } + let code: Bound<'py, PyAny> = violation.getattr("code")?; + let remediation: Bound<'py, PyAny> = violation.getattr("remediation")?; + let fixable: bool = !violation.getattr("fix")?.is_none() + && safe_fixes.contains(&code)? + && fixable_paths.contains(&path.call_method0("as_posix")?.extract::()?); + call_with( + &imported(py, RULES_MODELS_MODULE, "Finding")?, + &[ + ("code", code), + ("path", path), + ("line", violation.getattr("line")?), + ("column", violation.getattr("column")?), + ("message", violation.getattr("message")?), + ( + "remediation", + if remediation.is_truthy()? { + remediation + } else { + PyString::new(py, DEFAULT_REMEDIATION).into_any() + }, + ), + ("fixable", fixable.into_bound_py_any(py)?), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_planned_sql_rules, module)?)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 83547ed762..4e680d87c2 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3937,7 +3937,45 @@ def run_rules( no_sql_analysis: bool = False, ) -> RulesRunResult: ... def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: ... -def release_freed_memory() -> None: ... +def plan_rule_run( + *, + graph: ProjectGraph, + config: RulesConfig, + project_dir: Path, + selected_keys: frozenset[CompiledObjectKey] | None, + no_sql_analysis: bool, +) -> RuleRunPlan: ... +def run_planned_sql_rules( + *, + plan: RuleRunPlan, + graph: ProjectGraph, + discovered_inputs: DiscoveredProjectInputs, + dialect: str, + prepared_sql: PreparedSqlLint | None, + expansion_reuse: SqlExpansionReuse | None, +) -> SqlRulesEvaluation: ... +def sql_lint_preparation( + *, inputs: CompileInputsView, dialect: str +) -> tuple[Path, tuple[str, ...], LintConfig] | None: ... +def early_lint_expansions( + model_inputs: tuple[ModelInput, ...], +) -> dict[Path, CompiledSqlExpansion]: ... + +class EarlySqlLint: + """Own speculative SQL checks for one compilation invocation.""" + + def __init__(self, *, enabled: bool) -> None: ... + @property + def enabled(self) -> bool: ... + @property + def preparation(self) -> PreparedSqlLint | None: ... + @property + def expansion_reuse(self) -> SqlExpansionReuse | None: ... + def __enter__(self) -> EarlySqlLint: ... + def __exit__(self, *exception: object) -> None: ... + def start(self, *, inputs: CompileInputsView, dialect: str) -> None: ... + def stop(self) -> int: ... + def with_duplicate_literal_hints( *, findings: tuple[Finding, ...], project_dir: Path ) -> tuple[Finding, ...]: ... diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 7810636dfb..32a8c500c5 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -7,7 +7,15 @@ from pathlib import Path from threading import Event -from sqlbuild._native import build_lint_expansion_context, finish_rule_run, run_lint +from sqlbuild._native import ( + build_lint_expansion_context, + early_lint_expansions, + finish_rule_run, + plan_rule_run, + run_lint, + run_planned_sql_rules, + sql_lint_preparation, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -20,12 +28,6 @@ from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules -from sqlbuild.rule_engine._helpers.run.rules import ( - early_lint_expansions, - plan_rule_run, - run_planned_sql_rules, - sql_lint_preparation, -) from sqlbuild.rule_engine.models import ( CustomRulesOutcome, PreparedSqlLint, diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py deleted file mode 100644 index aff62ac0c0..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ /dev/null @@ -1,659 +0,0 @@ -"""Unified compiler-rule orchestration implementation.""" - -from __future__ import annotations - -import json -import time -from dataclasses import asdict, dataclass, replace -from pathlib import Path - -from sqlbuild._native import ( - build_lint_expansion_context, - build_rule_catalogue, - collect_lint_project_files, - compiled_relation_keys, - declared_relation_keys, - encode_sql_rule_findings, - load_sql_rule_cache, - lookup_sql_rule_cache, - project_with, - release_freed_memory, - resolve_rule_ignore_selectors, - rules_bulk_cache_path, - run_lint, - select_catalogue_rules, - selected_sql_rule_codes, - sql_file_rule_key, - sql_model_rule_keys, - sql_rule_lint_key, - valid_rules_config, - with_duplicate_literal_hints, - write_sql_rule_cache, -) -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlExpansion, - DeclarationScopeBuild, - SqlExpansionContext, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompileInputsView, - ModelInput, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.lint.constants import ( - DEFAULT_MAX_LITERAL_LENGTH, - DEFAULT_MAX_RANKING_ORDER_BY, - MAX_LITERAL_LENGTH_THRESHOLD, - MAX_RANKING_ORDER_BY_THRESHOLD, - SAFE_RULE_FIX_CODES, -) -from sqlbuild.lint.exceptions import NativeLintError -from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation -from sqlbuild.lint.types import RelationKeys -from sqlbuild.rule_engine._helpers.engine.custom_payloads import custom_rule_payloads -from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules -from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES -from sqlbuild.rule_engine.models import ( - Finding, - PreparedSqlLint, - PreparedSqlLintResult, - Rule, - RuleRunPlan, - RulesConfig, - SqlExpansionReuse, - SqlRulesEvaluation, -) - -_SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" - - -@dataclass(frozen=True) -class _SqlRuleInputs: - identities: dict[str, str] - project_files: dict[Path, str] | None - - -def plan_rule_run( - *, - graph: ProjectGraph, - config: RulesConfig, - project_dir: Path, - selected_keys: frozenset[CompiledObjectKey] | None, - no_sql_analysis: bool, -) -> RuleRunPlan: - """Build the catalogue and select the rules, project slice and settings one run evaluates.""" - - release_freed_memory() - effective_config: RulesConfig = resolve_rule_ignore_selectors( - config=config, project=graph.project - ) - resolved_project_dir: Path = project_dir.resolve() - include_custom: bool = _config_references_custom_rules(effective_config) - catalogue: tuple[Rule, ...] = build_rule_catalogue( - config=effective_config, - project_dir=resolved_project_dir, - include_custom=include_custom, - ) - custom_payloads: list[dict[str, object]] = custom_rule_payloads( - catalogue=catalogue, project_dir=resolved_project_dir - ) - selected: tuple[Rule, ...] = select_catalogue_rules( - catalogue=catalogue, - config=effective_config, - project_dir=resolved_project_dir, - custom_payloads=custom_payloads, - ) - skipped_type_proof_rules: tuple[str, ...] = () - if no_sql_analysis or not graph.project.settings.sql_analysis: - skipped_type_proof_rules = tuple( - rule.code for rule in selected if rule.code in TYPE_PROOF_RULE_CODES - ) - selected = tuple(rule for rule in selected if rule.code not in TYPE_PROOF_RULE_CODES) - custom_rules: tuple[Rule, ...] = tuple(rule for rule in selected if rule.custom) - if custom_rules: - _ = verify_custom_rules(rules=custom_rules, project_dir=resolved_project_dir) - model_paths: frozenset[str] | None = _selected_model_paths( - graph=graph, selected_keys=selected_keys - ) - selected_project: CompiledProjectView = _selected_project(graph=graph, model_paths=model_paths) - selected_config: RulesConfig = replace( - _selection_rules_config( - config=effective_config, - model_paths=model_paths, - project=selected_project, - ), - select=tuple(rule.code for rule in selected), - ignore=(), - ) - return RuleRunPlan( - effective_config=effective_config, - project_dir=resolved_project_dir, - catalogue=catalogue, - custom_payloads=custom_payloads, - selected=selected, - skipped_type_proof_rules=skipped_type_proof_rules, - model_paths=model_paths, - project=selected_project, - config=selected_config, - ) - - -def run_planned_sql_rules( - *, - plan: RuleRunPlan, - graph: ProjectGraph, - discovered_inputs: DiscoveredProjectInputs, - dialect: str, - prepared_sql: PreparedSqlLint | None, - expansion_reuse: SqlExpansionReuse | None, -) -> SqlRulesEvaluation: - """Run the planned SQL rules on the caller while built-in rules evaluate natively.""" - - native_rules: tuple[Rule, ...] = tuple(rule for rule in plan.selected if not rule.custom) - lint_config: LintConfig = sql_lint_config( - dialect=dialect, - codes=_sql_rule_codes(native_rules), - thresholds=plan.effective_config.thresholds, - relation_keys=compiled_relation_keys(graph.project), - allow_suppressions=plan.effective_config.allow_exceptions, - ) - sql_rule_inputs: _SqlRuleInputs = _prepare_sql_rule_inputs( - rules=native_rules, - project=plan.project, - dialect=sql_rule_lint_key(config=lint_config), - project_dir=plan.project_dir, - collect_files=prepared_sql is None and plan.model_paths is None, - discovered_inputs=discovered_inputs, - ) - sql_started: float = time.monotonic() - sql_result: SqlRulesEvaluation = _run_sql_rules( - rules=native_rules, - project_dir=plan.project_dir, - discovered_inputs=discovered_inputs, - project=plan.project, - lint_config=lint_config, - selected_model_paths=plan.model_paths, - cache_enabled=plan.effective_config.cache.enabled, - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - prepared_inputs=sql_rule_inputs, - ) - return replace(sql_result, sql_ms=round((time.monotonic() - sql_started) * 1000)) - - -def _prepare_sql_rule_inputs( - *, - rules: tuple[Rule, ...], - project: CompiledProjectView, - dialect: str, - project_dir: Path, - collect_files: bool, - discovered_inputs: DiscoveredProjectInputs, -) -> _SqlRuleInputs: - """Hash SQL and read the project files on the caller while built-in rules evaluate natively.""" - - identities: dict[str, str] = _sql_model_rule_identities( - rules=rules, project=project, dialect=dialect - ) - return _SqlRuleInputs( - identities=identities, - project_files=( - collect_lint_project_files(project_dir=project_dir, selected_paths=None) - if identities and collect_files - else None - ), - ) - - -def _config_references_custom_rules(config: RulesConfig) -> bool: - ignore_selectors: list[str] = [] - for entry in config.rule_ignores: - ignore_selectors.extend(entry.rules) - selectors: tuple[str, ...] = ( - *config.select, - *config.ignore, - *config.rule_options, - *(entry.rule for entry in config.rule_exceptions), - *ignore_selectors, - ) - return any(selector.startswith("XSQBR") for selector in selectors) - - -def sql_lint_preparation( - *, inputs: CompileInputsView, dialect: str -) -> tuple[Path, tuple[str, ...], LintConfig] | None: - """The project, SQL rule codes and lint settings early SQL lint runs with, if it should run.""" - - project_dir: Path | None = inputs.discovered_inputs.project_dir - if project_dir is None or any(diagnostic.is_error for diagnostic in inputs.diagnostics): - return None - config: RulesConfig | None = valid_rules_config(project_dir=project_dir) - if ( - config is None - or not config.select - or (config.cache.enabled and _read_sql_rule_cache(project_dir)) - ): - return None - codes: tuple[str, ...] = _selected_sql_codes(config) - if not codes: - return None - return ( - project_dir, - codes, - sql_lint_config( - dialect=dialect, - codes=codes, - thresholds=config.thresholds, - allow_suppressions=config.allow_exceptions, - relation_keys=declared_relation_keys( - model_rows=(model.relation_key_row for model in inputs.model_inputs), - seeds=(seed.schema_entry for seed in inputs.seed_inputs), - sources=(source.source_entry for source in inputs.source_inputs), - ), - ), - ) - - -def early_lint_expansions( - model_inputs: tuple[ModelInput, ...], -) -> dict[Path, CompiledSqlExpansion]: - """Each model's compile expansion for early lint, whole when the compile handed one over.""" - - expansions: dict[Path, CompiledSqlExpansion] = {} - for model in model_inputs: - expansion: CompiledSqlExpansion | None = model.lint_expansion or model.sql_expansion - if expansion is not None: - expansions[model.model_file.file_path] = expansion - return expansions - - -def _selection_rules_config( - *, - config: RulesConfig, - model_paths: frozenset[str] | None, - project: CompiledProjectView, -) -> RulesConfig: - if model_paths is None: - return config - selected_model_names: frozenset[str] = frozenset(model.name for model in project.models) - return replace( - config, - rule_exceptions=tuple( - entry for entry in config.rule_exceptions if Path(entry.path).as_posix() in model_paths - ), - graph_edge_exceptions=tuple( - entry - for entry in config.graph_edge_exceptions - if entry.consumer in selected_model_names - ), - ) - - -def _selected_sql_codes(config: RulesConfig) -> tuple[str, ...]: - return tuple(selected_sql_rule_codes(select=list(config.select), ignore=list(config.ignore))) - - -def _selected_project( - *, graph: ProjectGraph, model_paths: frozenset[str] | None -) -> CompiledProjectView: - if model_paths is None: - return graph.project - return project_with( - project=graph.project, - models=tuple( - model for model in graph.project.models if model.relative_path.as_posix() in model_paths - ), - ) - - -def _prepared_project_files( - *, prepared_sql: PreparedSqlLint | None, project_dir: Path, codes: tuple[str, ...] -) -> dict[Path, str] | None: - """Reuse the full-project read of a matching early SQL lint for this invocation.""" - - if ( - prepared_sql is None - or prepared_sql.project_dir != project_dir - or set(prepared_sql.codes) != set(codes) - or prepared_sql.future.exception() is not None - ): - return None - return dict(prepared_sql.future.result().result.source_texts) - - -def _run_prepared_lint( - *, - project_dir: Path, - config: LintConfig, - selected_paths: frozenset[Path], - discovered_inputs: DiscoveredProjectInputs, - compiled_expansions: dict[Path, CompiledSqlExpansion], - dynamic_output_paths: frozenset[Path], - prepared_sql: PreparedSqlLint | None, - expansion_reuse: SqlExpansionReuse | None, - source_files: dict[Path, str] | None, -) -> LintRunResult: - if prepared_sql is None or set(prepared_sql.codes) != set(config.enabled_native_rules or ()): - return run_lint( - project_dir=project_dir, - config=config, - selected_paths=selected_paths, - discovered_inputs=discovered_inputs, - compiled_expansions=compiled_expansions, - dynamic_output_paths=dynamic_output_paths, - expansion_context=( - _lint_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - expansion_reuse=expansion_reuse, - ) - if config.native_enabled - else None - ), - source_files=source_files, - ) - completed: PreparedSqlLintResult = prepared_sql.future.result() - prepared: LintRunResult = completed.result - proof_paths: frozenset[Path] = selected_paths & dynamic_output_paths - violations: tuple[LintViolation, ...] = tuple( - violation - for violation in prepared.violations - if violation.file_path in selected_paths and violation.file_path not in proof_paths - ) - if proof_paths: - proven: LintRunResult = run_lint( - project_dir=project_dir, - config=config, - selected_paths=proof_paths, - discovered_inputs=discovered_inputs, - compiled_expansions=compiled_expansions, - dynamic_output_paths=proof_paths, - expansion_context=completed.context, - source_files=dict(prepared.source_texts), - ) - violations = (*violations, *proven.violations) - return replace(prepared, violations=violations, files_checked=len(selected_paths)) - - -def _run_sql_rules( - *, - rules: tuple[Rule, ...], - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - project: CompiledProjectView, - lint_config: LintConfig, - selected_model_paths: frozenset[str] | None, - cache_enabled: bool, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, - prepared_inputs: _SqlRuleInputs | None = None, -) -> SqlRulesEvaluation: - codes: tuple[str, ...] = _sql_rule_codes(rules) - if not codes: - return SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) - models_by_path: dict[str, CompiledModelView] = _sql_rule_models(project) - bucket: dict[str, tuple[str, str]] = _read_sql_rule_cache(project_dir) if cache_enabled else {} - identities: dict[str, str] = ( - prepared_inputs.identities - if prepared_inputs is not None - else _sql_model_rule_identities( - rules=rules, project=project, dialect=sql_rule_lint_key(config=lint_config) - ) - ) - findings: list[Finding] = [] - model_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( - bucket=bucket, keys=identities - ) - misses: set[str] = set(identities) - set(model_hits) - for path in identities: - findings.extend(model_hits.get(path, ())) - selected_paths: set[Path] = {project_dir / path for path in misses} - file_identities: dict[str, str] = {} - project_files: dict[Path, str] | None = None - if selected_model_paths is None: - model_paths: frozenset[Path] = frozenset( - project_dir / model.relative_path for model in project.models - ) - project_files = _prepared_project_files( - prepared_sql=prepared_sql, project_dir=project_dir, codes=codes - ) - if project_files is None: - project_files = ( - prepared_inputs.project_files - if prepared_inputs is not None and prepared_inputs.project_files is not None - else collect_lint_project_files(project_dir=project_dir, selected_paths=None) - ) - lint_key: str = sql_rule_lint_key(config=lint_config) - file_keys: dict[str, str | None] = { - file_path.relative_to(project_dir).as_posix(): sql_file_rule_key( - lint_key=lint_key, - codes=list(codes), - relative_path=file_path.relative_to(project_dir).as_posix(), - contents=contents, - expanded=file_path in project.sql_expansions, - ) - for file_path, contents in project_files.items() - if file_path not in model_paths - } - file_hits: dict[str, tuple[Finding, ...]] = _cached_sql_findings( - bucket=bucket, - keys={path: key for path, key in file_keys.items() if key is not None}, - ) - file_path: Path - for file_path in project_files: - if file_path in model_paths: - continue - relative_path: str = file_path.relative_to(project_dir).as_posix() - file_identity: str | None = file_keys[relative_path] - cached_file: tuple[Finding, ...] | None = file_hits.get(relative_path) - if cached_file is None: - selected_paths.add(file_path) - if file_identity is not None: - file_identities[relative_path] = file_identity - else: - findings.extend(cached_file) - result: LintRunResult | None = ( - _run_prepared_lint( - project_dir=project_dir, - config=lint_config, - selected_paths=frozenset(selected_paths), - discovered_inputs=discovered_inputs, - compiled_expansions=project.sql_expansions, - dynamic_output_paths=frozenset( - (project_dir / model.relative_path).resolve() - for model in models_by_path.values() - if model.dynamic_column_contract is not None - and model.dynamic_column_contract.output_proven - ), - prepared_sql=prepared_sql, - expansion_reuse=expansion_reuse, - source_files=project_files, - ) - if selected_paths - else None - ) - selected_codes: frozenset[str] = frozenset((*codes, _SQL_RULE_SUPPRESSION_CODE)) - evaluated: list[Finding] = [] - for violation in () if result is None else result.violations: - if violation.code in selected_codes: - evaluated.append( - _lint_finding( - violation=violation, - project_dir=project_dir, - fixable_paths=frozenset(models_by_path), - ) - ) - elif violation.code == NativeLintError.code: - finding: Finding = _lint_finding( - violation=violation, project_dir=project_dir, fixable_paths=frozenset() - ) - evaluated.extend( - replace( - finding, - code=code, - message=f"Rule {code} could not be evaluated: {violation.message}", - unevaluated=True, - ) - for code in codes - ) - findings.extend(evaluated) - written_identities: dict[str, str] = { - **{path: identities[path] for path in misses}, - **file_identities, - } - if cache_enabled and written_identities: - by_path: dict[str, list[Finding]] = {path: [] for path in written_identities} - for finding in evaluated: - path: str = finding.path.as_posix() - if path in by_path: - by_path[path].append(finding) - for path, identity in written_identities.items(): - bucket[path] = (identity, encode_sql_rule_findings(findings=by_path[path])) - write_sql_rule_cache(path=_sql_rule_cache_path(project_dir), entries=bucket) - return SqlRulesEvaluation( - findings=with_duplicate_literal_hints(findings=tuple(findings), project_dir=project_dir), - cache_hits=len(identities) - len(misses), - cache_misses=len(misses), - ) - - -def _lint_expansion_context( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - expansion_reuse: SqlExpansionReuse | None, -) -> SqlExpansionContext: - """Build lint expansion, reusing the compiler's scope only for the same discovery.""" - - declaration_scope: DeclarationScopeBuild | None = ( - expansion_reuse.declaration_scope - if expansion_reuse is not None and expansion_reuse.discovered_inputs is discovered_inputs - else None - ) - return build_lint_expansion_context( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - declaration_scope=declaration_scope, - ) - - -def _lint_finding( - *, violation: LintViolation, project_dir: Path, fixable_paths: frozenset[str] -) -> Finding: - """Convert one lint violation; fixable when `format --fix` would attempt its edit.""" - - path: Path = violation.file_path - if path.is_absolute() and path.is_relative_to(project_dir.resolve()): - path = path.relative_to(project_dir.resolve()) - return Finding( - code=violation.code, - path=path, - line=violation.line, - column=violation.column, - message=violation.message, - remediation=violation.remediation or "Update the authored SQL to satisfy this rule.", - fixable=violation.fix is not None - and violation.code in SAFE_RULE_FIX_CODES - and path.as_posix() in fixable_paths, - ) - - -def sql_lint_config( - *, - dialect: str, - codes: tuple[str, ...], - thresholds: dict[str, int], - relation_keys: RelationKeys, - allow_suppressions: bool, -) -> LintConfig: - """Build the native SQL lint configuration that SQL Rules run with.""" - - return LintConfig( - dialect=dialect, - enabled_native_rules=codes, - header_rules_enabled=False, - max_literal_length=thresholds.get(MAX_LITERAL_LENGTH_THRESHOLD, DEFAULT_MAX_LITERAL_LENGTH), - max_ranking_order_by=thresholds.get( - MAX_RANKING_ORDER_BY_THRESHOLD, DEFAULT_MAX_RANKING_ORDER_BY - ), - relation_keys=relation_keys, - allow_suppressions=allow_suppressions, - ) - - -def _sql_rule_codes(rules: tuple[Rule, ...]) -> tuple[str, ...]: - return tuple(rule.code for rule in rules if rule.code.startswith("SQBRSQL")) - - -def _sql_rule_models(project: CompiledProjectView) -> dict[str, CompiledModelView]: - return { - model.relative_path.as_posix(): model - for model in project.models - if project.settings.sql_analysis and model.config.values.get("sql_analysis") is not False - } - - -def _sql_model_rule_identities( - *, rules: tuple[Rule, ...], project: CompiledProjectView, dialect: str -) -> dict[str, str]: - codes: tuple[str, ...] = _sql_rule_codes(rules) - if not codes: - return {} - models: list[tuple[str, str, str, str | None]] = [ - (path, model.authored_sql, model.query_sql, _contract_text(model)) - for path, model in _sql_rule_models(project).items() - ] - keys: dict[str, str] = sql_model_rule_keys(lint_key=dialect, codes=list(codes), models=models) - return {path: keys[path] for path, *_ in models} - - -def _contract_text(model: CompiledModelView) -> str | None: - """Canonical text of the dynamic column contract that SQL rule findings depend on.""" - - if model.dynamic_column_contract is None: - return None - return json.dumps( - asdict(model.dynamic_column_contract), - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - -def _sql_rule_cache_path(project_dir: Path) -> Path: - return rules_bulk_cache_path(project_dir=project_dir, file_name=SQL_RULES_CACHE_FILE) - - -def _read_sql_rule_cache(project_dir: Path) -> dict[str, tuple[str, str]]: - """Return usable entries so incompatible builds cannot suppress early lint.""" - - return load_sql_rule_cache(path=_sql_rule_cache_path(project_dir)) - - -def _cached_sql_findings( - *, bucket: dict[str, tuple[str, str]], keys: dict[str, str] -) -> dict[str, tuple[Finding, ...]]: - """Decoded stored findings of each path whose key still matches; undecodable ones miss.""" - - hits: dict[str, list[Finding]] = lookup_sql_rule_cache(entries=bucket, keys=list(keys.items())) - return {path: tuple(findings) for path, findings in hits.items()} - - -def _selected_model_paths( - *, graph: ProjectGraph, selected_keys: frozenset[CompiledObjectKey] | None -) -> frozenset[str] | None: - if selected_keys is None: - return None - selected_names: frozenset[str] = frozenset( - key.name for key in selected_keys if key.resource_type is CompiledResourceType.MODEL - ) - return frozenset( - model.relative_path.as_posix() - for model in graph.project.models - if model.name in selected_names - ) diff --git a/src/sqlbuild/rule_engine/classes/early_sql_lint.py b/src/sqlbuild/rule_engine/classes/early_sql_lint.py deleted file mode 100644 index 2b319c38aa..0000000000 --- a/src/sqlbuild/rule_engine/classes/early_sql_lint.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Own speculative SQL checks for one compilation invocation.""" - -from __future__ import annotations - -import time -from concurrent.futures import ThreadPoolExecutor -from threading import Event -from types import TracebackType - -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.rule_engine._helpers.run.rule_threads import prepare_sql_rules -from sqlbuild.rule_engine.models import PreparedSqlLint, SqlExpansionReuse - - -class EarlySqlLint: - def __init__(self, *, enabled: bool) -> None: - self.enabled: bool = enabled - self.preparation: PreparedSqlLint | None = None - self.expansion_reuse: SqlExpansionReuse | None = None - self.stop_requested: Event = Event() - self.executor: ThreadPoolExecutor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="sqlbuild-lint" - ) - - def __enter__(self) -> EarlySqlLint: - return self - - def __exit__( - self, - exception_type: type[BaseException] | None, - exception: BaseException | None, - traceback: TracebackType | None, - ) -> None: - _ = self.stop() - - def start(self, *, inputs: CompileInputsView, dialect: str) -> None: - if self.enabled: - self.preparation = prepare_sql_rules( - inputs=inputs, - executor=self.executor, - dialect=dialect, - stop=self.stop_requested, - ) - self.expansion_reuse = ( - None - if inputs.declaration_scope is None - else SqlExpansionReuse( - discovered_inputs=inputs.discovered_inputs, - declaration_scope=inputs.declaration_scope, - ) - ) - - def stop(self) -> int: - """Abandon unconsumed speculative work and return the milliseconds spent joining it.""" - - stop_start: float = time.monotonic() - self.stop_requested.set() - self.executor.shutdown(wait=True, cancel_futures=True) - return int((time.monotonic() - stop_start) * 1000) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 98648b433a..289af32bd6 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -15,7 +15,6 @@ import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module import sqlbuild.rule_engine._helpers.run.rule_threads as rule_threads_module -import sqlbuild.rule_engine._helpers.run.rules as rules_module from sqlbuild._native import rules_bulk_cache_path from sqlbuild._native import run_cli as main from sqlbuild.compiler.compile.models import DeclarationScopeBuild, LoadedMacro @@ -713,7 +712,7 @@ def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rule cold_exit: int = main(command) cold: dict[str, object] = json.loads(capsys.readouterr().out) - cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = f"{stored['build_identity']}-rebuilt" cache_path.write_text(json.dumps(stored), encoding="utf-8") @@ -931,7 +930,7 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_ b"SELECT 1\r\n" ) collections: list[frozenset[Path] | None] = [] - original_collect: Callable[..., dict[Path, str]] = rules_module.collect_lint_project_files + original_collect: Callable[..., dict[Path, str]] = native_module.collect_lint_project_files def counting_collect( *, project_dir: Path, selected_paths: frozenset[Path] | None @@ -939,7 +938,7 @@ def counting_collect( collections.append(selected_paths) return original_collect(project_dir=project_dir, selected_paths=selected_paths) - monkeypatch.setattr(rules_module, "collect_lint_project_files", counting_collect) + monkeypatch.setattr(native_module, "collect_lint_project_files", counting_collect) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) cold_collections: int = len(collections) @@ -994,7 +993,7 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( project_dir=tmp_path, capsys=capsys ) - cache_path: Path = rules_module._sql_rule_cache_path(tmp_path) + cache_path: Path = rules_bulk_cache_path(project_dir=tmp_path, file_name="sql.json") stored: dict[str, object] = json.loads(cache_path.read_text(encoding="utf-8")) stored["build_identity"] = "previous-build" cache_path.write_text(json.dumps(stored), encoding="utf-8") From 84060c043eb97549575d8fd3bb04bebd227cb6a7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:37:54 +0100 Subject: [PATCH 29/32] chore: update the Python boundary after rule orchestration port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 8872cc876d..7da18f1a41 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 44051 +remaining_lines = 43190 [[stays]] path = "executor" From 414738f02c39f1d4f85364fe92bd2bc52ea4e665 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:56:43 +0100 Subject: [PATCH 30/32] refactor: host planner scopes and loader graphs in Rust --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/planner_scope/constants.rs | 5 + .../src/planner_scope/main/expand_loaders.rs | 67 +++++ .../main/intermediate_sources.rs | 38 +++ .../src/planner_scope/main/managed_sources.rs | 21 ++ .../src/planner_scope/main/mod.rs | 6 + .../planner_scope/main/source_selectors.rs | 42 +++ .../src/planner_scope/mod.rs | 8 + .../src/planner_scope/models.rs | 61 ++++ .../src/planner_scope/tests.rs | 6 + .../src/planner_scope/tests/test_loaders.rs | 122 ++++++++ .../src/planner_scope/tests/test_types.rs | 15 + .../_helpers/build_command/lifecycle.rs | 2 +- .../src/bindings/_helpers/graph/views.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_scope/loaders.rs | 139 +++++++++ .../bindings/_helpers/planner_scope/mod.rs | 7 + .../_helpers/planner_scope/registration.rs | 35 +++ .../bindings/_helpers/planner_scope/scope.rs | 160 ++++++++++ .../_helpers/planner_scope/sources.rs | 115 +++++++ .../bindings/_helpers/planner_scope/values.rs | 78 +++++ .../src/bindings/_helpers/rules/catalogue.rs | 2 +- .../src/bindings/main/register.rs | 6 +- src/sqlbuild/_native.pyi | 60 ++++ .../_helpers/build_planning/defer_clone.py | 2 +- .../planner/_helpers/graph/auto_load.py | 29 -- .../planner/_helpers/graph/loader_dag.py | 282 ------------------ .../compiler/planner/_helpers/graph/scope.py | 114 ------- .../_helpers/graph/source_load_nodes.py | 87 ------ .../planner/_helpers/output/plan_entry.py | 2 +- .../planner/_helpers/output/plan_output.py | 8 +- .../planner/_helpers/planning/scopes.py | 3 +- .../planner/_helpers/warehouse/snapshot.py | 4 +- .../planner/main/selection/_loader_dag.py | 18 -- .../compiler/planner/main/selection/scope.py | 27 -- .../compiler/planner/_helpers/_test_types.py | 8 + .../_helpers/test_source_load_nodes.py | 43 ++- 37 files changed, 1046 insertions(+), 580 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/planner_scope/constants.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/mod.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/models.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/tests.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs create mode 100644 crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/scope.py delete mode 100644 src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/_loader_dag.py delete mode 100644 src/sqlbuild/compiler/planner/main/selection/scope.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 2f4aba477e..7046ea142a 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -21,6 +21,7 @@ pub mod lineage_graph; pub mod manifest; pub mod planner_changes; pub mod planner_identity; +pub mod planner_scope; pub mod planner_selection; pub mod planner_strategy; pub mod python_plan; diff --git a/crates/sqlbuild-analysis/src/planner_scope/constants.rs b/crates/sqlbuild-analysis/src/planner_scope/constants.rs new file mode 100644 index 0000000000..cd3b1d2877 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/constants.rs @@ -0,0 +1,5 @@ +//! Planner resource and selector kinds. + +pub const SOURCE: &str = "source"; +pub const MODEL: &str = "model"; +pub const NAME: &str = "name"; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs new file mode 100644 index 0000000000..9c33d9f004 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/expand_loaders.rs @@ -0,0 +1,67 @@ +//! Expand validated loader DAGs without recursive traversal. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::SOURCE; +use crate::planner_scope::models::LoaderGraph; +use std::collections::{HashMap, HashSet}; + +/// Add loader dependency edges and executable upstream source keys in input iteration order. +pub fn expand_loaders( + graph: &LoaderGraph, + mut selected: Vec, + mut upstream: Vec<(ObjectKey, Vec)>, + executable: &HashSet, +) -> (Vec, Vec<(ObjectKey, Vec)>) { + let mut positions: HashMap = upstream + .iter() + .enumerate() + .map(|(i, (k, _))| (k.clone(), i)) + .collect(); + let mut selected_seen: HashSet = selected.iter().cloned().collect(); + for key in selected.clone() { + if key.0 != SOURCE { + continue; + } + let Some(loader) = graph.source_loader(&key) else { + if !graph.sources.iter().any(|(name, _)| name == &key.1) + && graph.loaders.contains_key(&key.1) + && !positions.contains_key(&key) + { + positions.insert(key.clone(), upstream.len()); + upstream.push((key, Vec::new())); + } + continue; + }; + let Some(deps) = graph.loaders.get(loader) else { + continue; + }; + let mut pending = vec![(key.clone(), deps)]; + let mut visited: HashSet = HashSet::new(); + while let Some((node, dependencies)) = pending.pop() { + if !visited.insert(node.clone()) { + continue; + } + let edges: Vec = dependencies.iter().map(|name| graph.key(name)).collect(); + if let Some(index) = positions.get(&node) { + upstream[*index].1 = edges; + } else { + positions.insert(node.clone(), upstream.len()); + upstream.push((node, edges)); + } + for name in dependencies.iter().rev() { + if let Some(next) = graph.loaders.get(name) { + pending.push((graph.key(name), next)); + } + } + } + if executable.contains(&key) { + for name in graph.upstream(deps) { + let node = graph.key(&name); + if selected_seen.insert(node.clone()) { + selected.push(node); + } + } + } + } + (selected, upstream) +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs b/crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs new file mode 100644 index 0000000000..bfe0a7146e --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/intermediate_sources.rs @@ -0,0 +1,38 @@ +//! Intermediate source names required by selected loader nodes. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::SOURCE; +use crate::planner_scope::models::LoaderGraph; +use std::collections::HashSet; + +/// Intermediate loader names in selected or upstream first-seen order. +pub fn intermediate_sources( + graph: &LoaderGraph, + selected: &[ObjectKey], + upstream: bool, +) -> Vec { + let mut names: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + for key in selected { + if key.0 != SOURCE { + continue; + } + let candidates = if upstream { + graph + .source_loader(key) + .and_then(|loader| graph.loaders.get(loader)) + .map(|deps| graph.upstream(deps)) + .unwrap_or_default() + } else if !graph.sources.iter().any(|(name, _)| name == &key.1) { + vec![key.1.clone()] + } else { + Vec::new() + }; + for name in candidates { + if graph.intermediate(&name) && seen.insert(name.clone()) { + names.push(name); + } + } + } + names +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs b/crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs new file mode 100644 index 0000000000..93222c064c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/managed_sources.rs @@ -0,0 +1,21 @@ +//! Direct managed sources required by selected models. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::{MODEL, SOURCE}; +use std::collections::HashSet; + +/// Select only direct managed source dependencies of selected models. +pub fn managed_sources( + selected: &[ObjectKey], + upstream: &[(ObjectKey, Vec)], + managed: &HashSet, +) -> HashSet { + let selected: HashSet<&ObjectKey> = selected.iter().filter(|key| key.0 == MODEL).collect(); + upstream + .iter() + .filter(|(key, _)| selected.contains(key)) + .flat_map(|(_, deps)| deps) + .filter(|key| key.0 == SOURCE && managed.contains(&key.1)) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs new file mode 100644 index 0000000000..3fbafe8e4c --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs @@ -0,0 +1,6 @@ +//! Planner scope entry points. + +pub mod expand_loaders; +pub mod intermediate_sources; +pub mod managed_sources; +pub mod source_selectors; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs b/crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs new file mode 100644 index 0000000000..3b66fcebc4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/source_selectors.rs @@ -0,0 +1,42 @@ +//! Explicit upstream source selector eligibility. + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::ParsedSelector; +use crate::planner_scope::constants::{NAME, SOURCE}; +use std::collections::HashSet; + +/// Find selected sources explicitly named by upstream source or bare-name selectors. +pub fn source_selectors( + select: &[String], + names: &[(String, ObjectKey)], + selected: &[ObjectKey], +) -> Result, SelectorError> { + let mut keys: HashSet = HashSet::new(); + for part in select + .iter() + .flat_map(|raw| raw.split_whitespace()) + .flat_map(|token| token.split(',')) + { + let ParsedSelector::Kind { + kind, + value, + upstream: true, + .. + } = parse_selector(part)? + else { + continue; + }; + if kind != NAME && kind != SOURCE { + continue; + } + if let Some((_, key)) = names.iter().rev().find(|(name, _)| name == &value) + && key.0 == SOURCE + && selected.contains(key) + { + keys.insert(key.clone()); + } + } + Ok(keys) +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/mod.rs b/crates/sqlbuild-analysis/src/planner_scope/mod.rs new file mode 100644 index 0000000000..361e6c6bec --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/mod.rs @@ -0,0 +1,8 @@ +//! Planner scope and loader dependency selection. + +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_scope/models.rs b/crates/sqlbuild-analysis/src/planner_scope/models.rs new file mode 100644 index 0000000000..38a1602733 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/models.rs @@ -0,0 +1,61 @@ +//! Ordered loader facts used by planner scope resolution. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::SOURCE; + +/// Loader dependencies resolved from Python function objects at the boundary. +#[derive(Debug, Default)] +pub struct LoaderGraph { + pub loaders: HashMap>, + pub sources: Vec<(String, Option)>, +} + +impl LoaderGraph { + /// Upstream loader names in breadth-first, first-seen order. + pub fn upstream(&self, dependencies: &[String]) -> Vec { + let mut pending: VecDeque = dependencies.iter().cloned().collect(); + let mut seen: HashSet = HashSet::new(); + let mut names: Vec = Vec::new(); + while let Some(name) = pending.pop_front() { + if !seen.insert(name.clone()) { + continue; + } + if let Some(dependencies) = self.loaders.get(&name) { + pending.extend(dependencies.iter().cloned()); + } + names.push(name); + } + names + } + + /// Source key for a loader, using the last source declaration for that loader. + pub fn key(&self, loader: &str) -> ObjectKey { + let name = self.sources.iter().rev().find_map(|(name, source_loader)| { + (source_loader.as_deref() == Some(loader)).then_some(name.as_str()) + }); + ("source".to_owned(), name.unwrap_or(loader).to_owned()) + } + + /// Managed loader name for a source key. + pub fn source_loader(&self, key: &ObjectKey) -> Option<&str> { + if key.0 != SOURCE { + return None; + } + self.sources + .iter() + .rev() + .find_map(|(name, loader)| (name == &key.1).then_some(loader.as_deref())) + .flatten() + } + + /// Whether a loader is intermediate rather than attached to a declared source. + pub fn intermediate(&self, name: &str) -> bool { + self.loaders.contains_key(name) + && !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests.rs b/crates/sqlbuild-analysis/src/planner_scope/tests.rs new file mode 100644 index 0000000000..1f484c175a --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/tests.rs @@ -0,0 +1,6 @@ +//! Planner scope regression tests. + +#[path = "tests/test_loaders.rs"] +mod test_loaders; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs new file mode 100644 index 0000000000..838ceeee51 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -0,0 +1,122 @@ +//! Iterative loader DAG ordering and execution eligibility regressions. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::main::expand_loaders::expand_loaders; +use crate::planner_scope::models::LoaderGraph; +use crate::planner_scope::tests::test_types::{DeepLoaderTestCase, LoaderTestCase}; +use std::collections::HashSet; + +#[test] +fn given_shared_loader_dependencies_when_expanding_then_preserves_depth_first_edges_and_breadth_first_names() + { + let test_cases = [ + LoaderTestCase { + description: "read only upstream", + executable: vec![], + expected_selected: vec!["raw_orders"], + }, + LoaderTestCase { + description: "execute upstream", + executable: vec!["raw_orders"], + expected_selected: vec!["raw_orders", "first", "shared", "last", "leaf"], + }, + ]; + for test_case in test_cases { + let graph = LoaderGraph { + loaders: [ + ( + "terminal".to_owned(), + vec!["first".to_owned(), "shared".to_owned()], + ), + ( + "first".to_owned(), + vec!["shared".to_owned(), "last".to_owned()], + ), + ("shared".to_owned(), vec!["leaf".to_owned()]), + ("last".to_owned(), vec!["leaf".to_owned()]), + ("leaf".to_owned(), Vec::new()), + ] + .into_iter() + .collect(), + sources: vec![("raw_orders".to_owned(), Some("terminal".to_owned()))], + }; + let root: ObjectKey = ("source".to_owned(), "raw_orders".to_owned()); + let executable: HashSet = test_case + .executable + .iter() + .map(|name| ("source".to_owned(), (*name).to_owned())) + .collect(); + let (selected, edges) = expand_loaders(&graph, vec![root], Vec::new(), &executable); + assert_eq!( + selected + .iter() + .map(|key| key.1.as_str()) + .collect::>(), + test_case.expected_selected, + "{}", + test_case.description + ); + assert_eq!( + edges + .iter() + .map(|(key, _)| key.1.as_str()) + .collect::>(), + vec!["raw_orders", "first", "shared", "leaf", "last"], + "{}", + test_case.description + ); + assert_eq!( + graph.upstream(&graph.loaders["terminal"]), + vec!["first", "shared", "last", "leaf"], + "{}", + test_case.description + ); + } +} + +#[test] +fn given_deep_loader_chain_when_expanding_then_uses_iterative_traversal() { + let test_cases = [DeepLoaderTestCase { + description: "deep validated chain", + depth: 2048, + expected_count: 2048, + }]; + for test_case in test_cases { + let mut graph = LoaderGraph { + loaders: (0..test_case.depth - 1) + .map(|index| { + ( + format!("loader_{index}"), + vec![format!("loader_{}", index + 1)], + ) + }) + .collect(), + ..LoaderGraph::default() + }; + graph + .loaders + .insert(format!("loader_{}", test_case.depth - 1), Vec::new()); + graph + .sources + .push(("raw_orders".to_owned(), Some("loader_0".to_owned()))); + let root: ObjectKey = ("source".to_owned(), "raw_orders".to_owned()); + let (selected, edges) = expand_loaders( + &graph, + vec![root.clone()], + Vec::new(), + &HashSet::from([root]), + ); + assert_eq!( + selected.len(), + test_case.expected_count, + "{}", + test_case.description + ); + assert_eq!( + edges.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs new file mode 100644 index 0000000000..f3f3d6c266 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs @@ -0,0 +1,15 @@ +//! Loader graph test cases. + +#[derive(Debug)] +pub(crate) struct LoaderTestCase { + pub description: &'static str, + pub executable: Vec<&'static str>, + pub expected_selected: Vec<&'static str>, +} + +#[derive(Debug)] +pub(crate) struct DeepLoaderTestCase { + pub description: &'static str, + pub depth: usize, + pub expected_count: usize, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index ad9c071ce9..122d228eeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; -const LOADER_DAG_MODULE: &str = "sqlbuild.compiler.planner.main.selection._loader_dag"; +const LOADER_DAG_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index 8d2fe707be..01218702a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -51,7 +51,7 @@ fn lineage_graph_names<'py>( Ok(names) } -fn edge_dict<'py>( +pub(crate) fn edge_dict<'py>( cache: &KeyCache<'py>, edges: &[(ObjectKey, Vec)], ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 3d6ee4b804..08ef0a6c01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -47,6 +47,7 @@ pub(crate) mod plan_command; pub(crate) mod plan_output; pub(crate) mod planner_changes; pub(crate) mod planner_identity; +pub(crate) mod planner_scope; pub(crate) mod planner_selection; pub(crate) mod planner_strategy; pub(crate) mod presentation_text; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs new file mode 100644 index 0000000000..40cc222cce --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -0,0 +1,139 @@ +//! Loader dependency mapping and synthetic source boundary construction. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::views::edge_dict; +use crate::bindings::_helpers::planner_scope::values::{ + NATIVE, SOURCE_MODELS, dependency_names, edges, keys, loader_graph, loader_names, project_graph, +}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::planner_scope::main::{ + expand_loaders::expand_loaders, intermediate_sources::intermediate_sources, +}; +use std::collections::HashSet; + +#[pyfunction] +#[pyo3(signature = (*, loader_function, loader_functions))] +pub(crate) fn upstream_loader_dependency_names<'py>( + loader_function: &Bound<'py, PyAny>, + loader_functions: &Bound<'py, PyAny>, +) -> PyResult> { + let graph = loader_graph(loader_functions)?; + let names = loader_names(loader_functions)?; + let deps = dependency_names(loader_function, &names)?; + PyTuple::new(loader_function.py(), graph.upstream(&deps)) +} + +#[pyfunction] +#[pyo3(signature = (*, project, loader))] +pub(crate) fn loader_to_source_entry<'py>( + project: &Bound<'py, PyAny>, + loader: &Bound<'py, PyAny>, +) -> PyResult> { + let py = project.py(); + let name = loader.getattr("name")?; + let mut destination = loader.getattr("destination")?; + if destination.is_none() { + destination = format!("__loader__{}", name.extract::()?) + .into_pyobject(py)? + .into_any(); + } + let parts = call_with( + &imported(py, NATIVE, "loader_destination_parts")?, + &[ + ("destination", destination), + ( + "default_database", + project.getattr("effective_target_database")?, + ), + ( + "default_schema", + project.getattr("effective_target_schema")?, + ), + ], + )?; + let kwargs = PyDict::new(py); + kwargs.set_item("name", &name)?; + kwargs.set_item("loader", &name)?; + for field in ["database", "schema", "table"] { + kwargs.set_item(field, parts.getattr(field)?)?; + } + for field in [ + "write_strategy", + "cursor_column", + "unique_key", + "contract", + "columns", + ] { + kwargs.set_item(field, loader.getattr(field)?)?; + } + let meta = PyDict::new(py); + meta.set_item("sqlbuild_loader_node", true)?; + kwargs.set_item("meta", meta)?; + imported(py, SOURCE_MODELS, "SourceEntry")?.call((), Some(&kwargs)) +} + +pub(crate) fn intermediate_map<'py>( + project: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, + upstream: bool, +) -> PyResult> { + let graph = project_graph(project)?; + let names = intermediate_sources(&graph, &keys(selected)?, upstream); + let loaders = PyDict::new(project.py()); + for loader in project.getattr("loader_functions")?.try_iter()? { + let loader = loader?; + loaders.set_item(loader.getattr("name")?, &loader)?; + } + let result = PyDict::new(project.py()); + for name in names { + if let Some(loader) = loaders.get_item(&name)? { + result.set_item(&name, loader_to_source_entry(project, &loader)?)?; + } + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys))] +pub(crate) fn build_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + intermediate_map(project, selected_keys, false) +} + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys))] +pub(crate) fn build_upstream_intermediate_source_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, +) -> PyResult> { + intermediate_map(project, selected_keys, true) +} + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys, upstream_deps, executable_dependency_source_keys = None))] +pub(crate) fn expand_selected_loader_dependencies<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, + executable_dependency_source_keys: Option<&Bound<'py, PyAny>>, +) -> PyResult<(Bound<'py, PyFrozenSet>, Bound<'py, PyDict>)> { + let graph = project_graph(project)?; + let selected = keys(selected_keys)?; + let upstream = edges(upstream_deps)?; + let executable = executable_dependency_source_keys + .map(keys) + .transpose()? + .unwrap_or_default() + .into_iter() + .collect::>(); + let (selected, upstream) = expand_loaders(&graph, selected, upstream, &executable); + let cache = KeyCache::new(project.py())?; + Ok(( + PyFrozenSet::new(project.py(), cache.keys(&selected)?)?, + edge_dict(&cache, &upstream)?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs new file mode 100644 index 0000000000..e7194181c9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/mod.rs @@ -0,0 +1,7 @@ +//! Planner scope Python boundary models. + +mod loaders; +pub(crate) mod registration; +mod scope; +mod sources; +mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs new file mode 100644 index 0000000000..517f9c5e66 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/registration.rs @@ -0,0 +1,35 @@ +//! Register planner scope and loader entry points. + +use crate::bindings::_helpers::planner_scope::{loaders, scope, sources}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope::build_planner_scope, module)?)?; + module.add_function(wrap_pyfunction!( + scope::managed_source_upstream_keys, + module + )?)?; + module.add_function(wrap_pyfunction!( + loaders::upstream_loader_dependency_names, + module + )?)?; + module.add_function(wrap_pyfunction!(loaders::loader_to_source_entry, module)?)?; + module.add_function(wrap_pyfunction!( + loaders::build_intermediate_source_map, + module + )?)?; + module.add_function(wrap_pyfunction!( + loaders::build_upstream_intermediate_source_map, + module + )?)?; + module.add_function(wrap_pyfunction!( + loaders::expand_selected_loader_dependencies, + module + )?)?; + module.add_function(wrap_pyfunction!(sources::build_source_load_map, module)?)?; + module.add_function(wrap_pyfunction!( + sources::build_source_load_entries, + module + )?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs new file mode 100644 index 0000000000..52d74f9dc4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -0,0 +1,160 @@ +//! Planner scope assembly using native graph and selector operations. + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_dependencies; +use crate::bindings::_helpers::planner_scope::values::{MODELS, NATIVE, edges, keys}; +use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; +use pyo3::types::PyDictMethods; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; +use sqlbuild_analysis::planner_scope::constants::SOURCE; +use sqlbuild_analysis::planner_scope::main::{ + managed_sources::managed_sources, source_selectors::source_selectors, +}; +use std::collections::HashSet; + +#[pyfunction] +#[pyo3(signature = (*, selected_keys, upstream_deps, project))] +pub(crate) fn managed_source_upstream_keys<'py>( + selected_keys: &Bound<'py, PyAny>, + upstream_deps: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, +) -> PyResult> { + let mut managed: HashSet = HashSet::new(); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if !source.getattr("source_entry")?.getattr("loader")?.is_none() { + managed.insert(source.getattr("name")?.extract()?); + } + } + let selected = managed_sources(&keys(selected_keys)?, &edges(upstream_deps)?, &managed); + PyFrozenSet::new( + project.py(), + KeyCache::new(project.py())?.keys(&selected.into_iter().collect::>())?, + ) +} + +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_planner_scope<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let args = KeywordArguments::read( + py, + "build_planner_scope", + options, + ( + &["project"], + &[ + "select", + "exclude", + "auto_load_sources", + "selected_keys", + "python_read_source_names", + "sql_test_selection", + ], + ), + )?; + let project = args.value("project")?; + let select = default_tuple(&args.value("select")?); + let exclude = default_tuple(&args.value("exclude")?); + let auto_load = args.value("auto_load_sources")?.is_truthy()?; + let mut upstream = imported(py, NATIVE, "build_execution_upstream_deps")?.call1((&project,))?; + let mut downstream = imported(py, NATIVE, "build_downstream_deps")?.call1((&upstream,))?; + let lineage = project.getattr("lineage_graph")?; + let all_keys = imported(py, NATIVE, "lineage_graph_names")?.call1((&lineage,))?; + let mut selected = args.value("selected_keys")?; + if selected.is_none() { + selected = call_with( + &imported(py, NATIVE, "resolve_graph_selection")?, + &[ + ("graph", lineage), + ("select", select.clone()), + ("exclude", exclude), + ], + )?; + } + let names: Vec<(String, (String, String))> = all_keys + .call_method0("items")? + .try_iter()? + .map(|item| { + let (name, key): (String, Bound<'py, PyAny>) = item?.extract()?; + Ok(( + name, + crate::bindings::_helpers::graph::keys::native_key(&key)?, + )) + }) + .collect::>()?; + let texts = py_strings(&select)?; + let carrier = Carrier::new(py, &texts, || { + Ok(names.iter().map(|(name, _)| name.clone()).collect()) + })?; + let source_keys = source_selectors(&carrier.send_all(&texts)?, &names, &keys(&selected)?) + .map_err(|error| carrier.error(error))?; + let cache = KeyCache::new(py)?; + let mut executable = PyFrozenSet::new( + py, + cache.keys(&source_keys.into_iter().collect::>())?, + )? + .into_any(); + if auto_load { + let loaded = managed_source_upstream_keys(&selected, &upstream, &project)?; + selected = selected.call_method1("union", (&loaded,))?; + executable = executable.call_method1("union", (&loaded,))?; + } + executable = executable.call_method1("intersection", (&selected,))?; + if auto_load || keys(&selected)?.iter().any(|key| key.0 == SOURCE) { + let (expanded, deps) = + expand_selected_loader_dependencies(&project, &selected, &upstream, Some(&executable))?; + selected = expanded.into_any(); + upstream = deps.into_any(); + downstream = imported(py, NATIVE, "build_downstream_deps")?.call1((&upstream,))?; + } + let origins = imported(py, NATIVE, "build_execution_edge_origins")?.call1((&project,))?; + let order = call_with( + &imported(py, NATIVE, "topologically_order_keys")?, + &[ + ("upstream", upstream.clone()), + ("injected_edge_origins", origins), + ], + )?; + let models = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model = model?; + models.set_item(model.getattr("name")?, model)?; + } + let mut reads = args.value("python_read_source_names")?; + if reads.is_none() { + reads = PyFrozenSet::empty(py)?.into_any(); + } + let mut tests = args.value("sql_test_selection")?; + if !tests.is_truthy()? { + tests = imported(py, MODELS, "SqlTestSelection")?.call0()?; + } + call_with( + &imported(py, MODELS, "PlannerScope")?, + &[ + ("upstream_deps", upstream), + ("downstream_deps", downstream), + ("all_keys", all_keys), + ("models_by_name", models.into_any()), + ("selected_keys", selected.clone()), + ("execution_order", order), + ("user_selected_keys", selected), + ("python_read_source_names", reads), + ("sql_test_selection", tests), + ], + ) +} + +fn default_tuple<'py>(value: &Bound<'py, PyAny>) -> Bound<'py, PyAny> { + if value.is_none() { + PyTuple::empty(value.py()).into_any() + } else { + value.clone() + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs new file mode 100644 index 0000000000..ba79f23caa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/sources.rs @@ -0,0 +1,115 @@ +//! Existing source and load plan models materialised at the Python boundary. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::graph::algorithms::closure; +use crate::bindings::_helpers::planner_scope::loaders::intermediate_map; +use crate::bindings::_helpers::planner_scope::values::MODELS; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::types::{PyDictMethods, PySetMethods}; +use sqlbuild_analysis::planner_scope::constants::SOURCE; + +const EXECUTION_TYPES: &str = "sqlbuild.runtime.contracts.types"; + +#[pyfunction] +#[pyo3(signature = (*, project, selected_keys, upstream_deps = None))] +pub(crate) fn build_source_load_map<'py>( + project: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + upstream_deps: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let relevant = upstream_deps + .map(|edges| { + closure( + selected_keys.try_iter()?.collect::>>()?, + edges, + (true, None), + ) + }) + .transpose()?; + let result = PyDict::new(project.py()); + for source in project.getattr("sources")?.try_iter()? { + let source = source?; + if let Some(keys) = &relevant + && !keys.contains(source.getattr("key")?)? + { + continue; + } + let entry = source.getattr("source_entry")?; + result.set_item(entry.getattr("name")?, entry)?; + } + for upstream in [false, true] { + for (key, entry) in intermediate_map(project, selected_keys, upstream)?.iter() { + result.set_item(key, entry)?; + } + } + Ok(result) +} + +#[pyfunction] +#[pyo3(signature = (*, execution_order, selected_keys, source_map, is_reload))] +pub(crate) fn build_source_load_entries<'py>( + execution_order: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + source_map: &Bound<'py, PyDict>, + is_reload: bool, +) -> PyResult> { + let py = execution_order.py(); + let class = imported(py, MODELS, "SourceLoadPlanEntry")?; + let kinds = imported(py, EXECUTION_TYPES, "ExecutionResourceKind")?; + let mut entries: Vec> = Vec::new(); + for key in execution_order.try_iter()? { + let key = key?; + if !selected_keys.contains(&key)? + || key.getattr("resource_type")?.str()?.extract::()? != SOURCE + { + continue; + } + let Some(entry) = source_map.get_item(key.getattr("name")?)? else { + continue; + }; + if entry.getattr("loader")?.is_none() { + continue; + } + let kwargs = PyDict::new(py); + kwargs.set_item("key", key)?; + for field in [ + "name", + "loader", + "write_strategy", + "cursor_column", + "unique_key", + ] { + kwargs.set_item(field, entry.getattr(field)?)?; + } + let table = entry.getattr("table")?; + kwargs.set_item( + "destination", + if table.is_truthy()? { + table + } else { + entry.getattr("name")? + }, + )?; + let intermediate = entry + .getattr("meta")? + .call_method1("get", ("sqlbuild_loader_node",))? + .is(PyBool::new(py, true)); + kwargs.set_item( + "resource_kind", + kinds.getattr(if intermediate { "LOADER" } else { "SOURCE" })?, + )?; + kwargs.set_item("is_reload", is_reload)?; + let integration = entry.getattr("integration_loader")?; + kwargs.set_item( + "integration_kind", + if integration.is_none() { + py.None().into_bound(py) + } else { + integration.getattr("kind")? + }, + )?; + entries.push(class.call((), Some(&kwargs))?); + } + PyTuple::new(py, entries) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs new file mode 100644 index 0000000000..a9fd5203d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/values.rs @@ -0,0 +1,78 @@ +//! Python function identity and compiled graph conversion. + +use crate::bindings::_helpers::graph::keys::native_key; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyDict; +use pyo3::types::PyDictMethods; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::planner_scope::models::LoaderGraph; + +pub(crate) const NATIVE: &str = "sqlbuild._native"; +pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const SOURCE_MODELS: &str = "sqlbuild.spec.contracts.models"; + +pub(crate) fn keys(values: &Bound<'_, PyAny>) -> PyResult> { + values + .try_iter()? + .map(|value| native_key(&value?)) + .collect() +} + +pub(crate) fn edges(values: &Bound<'_, PyAny>) -> PyResult)>> { + values + .call_method0("items")? + .try_iter()? + .map(|item| { + let (key, deps): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + Ok((native_key(&key)?, keys(&deps)?)) + }) + .collect() +} + +/// Resolve dependencies using the same Python dictionary lookup as the authored function boundary. +pub(crate) fn loader_names<'py>(loaders: &Bound<'py, PyAny>) -> PyResult> { + let names = PyDict::new(loaders.py()); + for loader in loaders.try_iter()? { + let loader = loader?; + names.set_item(loader.getattr("function")?, loader.getattr("name")?)?; + } + Ok(names) +} + +pub(crate) fn dependency_names( + loader: &Bound<'_, PyAny>, + names: &Bound<'_, PyDict>, +) -> PyResult> { + let mut dependencies: Vec = Vec::new(); + for dependency in loader.getattr("depends_on")?.try_iter()? { + if let Some(name) = names.get_item(dependency?)? { + dependencies.push(name.extract()?); + } + } + Ok(dependencies) +} + +pub(crate) fn loader_graph(loaders: &Bound<'_, PyAny>) -> PyResult { + let names = loader_names(loaders)?; + let mut graph = LoaderGraph::default(); + for loader in loaders.try_iter()? { + let loader = loader?; + graph.loaders.insert( + loader.getattr("name")?.extract()?, + dependency_names(&loader, &names)?, + ); + } + Ok(graph) +} + +pub(crate) fn project_graph(project: &Bound<'_, PyAny>) -> PyResult { + let mut graph = loader_graph(&project.getattr("loader_functions")?)?; + for source in project.getattr("sources")?.try_iter()? { + let entry = source?.getattr("source_entry")?; + graph.sources.push(( + entry.getattr("name")?.extract()?, + entry.getattr("loader")?.extract()?, + )); + } + Ok(graph) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs index 8d9fc09f6d..7b74700c6e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/catalogue.rs @@ -20,7 +20,7 @@ use crate::bindings::_helpers::rules::selection::{rules_error, selected_rule_cod const CUSTOM_CATALOGUE_MODULE: &str = "sqlbuild.rule_engine._helpers.engine._custom_catalogue"; const CUSTOM_PAYLOADS_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.custom_payloads"; const HERMETICITY_MODULE: &str = "sqlbuild.rule_engine._helpers.engine.hermeticity"; -const PLANNER_SCOPE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.scope"; +const PLANNER_SCOPE_MODULE: &str = "sqlbuild._native"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 578f78ab95..316a1427d3 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -8,8 +8,9 @@ use crate::bindings::_helpers::{ compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, plan_output, planner_changes, planner_identity, - planner_selection, planner_strategy, project_assembly, refactor, references, render, rules, - scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, + planner_scope, planner_selection, planner_strategy, project_assembly, refactor, references, + render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, + type_system, }; use crate::bindings::models; @@ -48,6 +49,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { planner_changes::registration::register(module)?; planner_identity::registration::register(module)?; planner_selection::registration::register(module)?; + planner_scope::registration::register(module)?; planner_strategy::registration::register(module)?; references::registration::register(module)?; authored_values::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 4e680d87c2..9ef86c388e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -222,7 +222,9 @@ from sqlbuild.compiler.planner.models import ( SelectionStalenessGraph, SelectionStalenessWarning, SelectorExpansion, + SourceLoadPlanEntry, SqlTestPlanEntry, + SqlTestSelection, UnitTestSelectorSplit, WarehouseSnapshot, ) @@ -4272,3 +4274,61 @@ def resolve_effective_microbatch_batch_size(*, batch_size: str, effective_grain: def aligned_timestamp_bounds( *, start: DateValue | TimestampValue, end: DateValue | TimestampValue ) -> tuple[datetime, datetime]: ... +def build_planner_scope( + *, + project: CompiledProjectView, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + auto_load_sources: bool = False, + selected_keys: frozenset[CompiledObjectKey] | None = None, + python_read_source_names: frozenset[str] = frozenset(), + sql_test_selection: SqlTestSelection | None = None, +) -> PlannerScope: ... +def managed_source_upstream_keys( + *, + selected_keys: frozenset[CompiledObjectKey], + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + project: CompiledProjectView, +) -> frozenset[CompiledObjectKey]: ... +def expand_selected_loader_dependencies( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + executable_dependency_source_keys: frozenset[CompiledObjectKey] = frozenset(), +) -> tuple[ + frozenset[CompiledObjectKey], dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] +]: ... +def upstream_loader_dependency_names( + *, + loader_function: DiscoveredLoaderFunction, + loader_functions: tuple[DiscoveredLoaderFunction, ...], +) -> tuple[str, ...]: ... +def loader_to_source_entry( + *, + project: CompiledProjectView, + loader: DiscoveredLoaderFunction, +) -> SourceEntry: ... +def build_intermediate_source_map( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], +) -> dict[str, SourceEntry]: ... +def build_upstream_intermediate_source_map( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], +) -> dict[str, SourceEntry]: ... +def build_source_load_map( + *, + project: CompiledProjectView, + selected_keys: frozenset[CompiledObjectKey], + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, +) -> dict[str, SourceEntry]: ... +def build_source_load_entries( + *, + execution_order: tuple[CompiledObjectKey, ...], + selected_keys: frozenset[CompiledObjectKey], + source_map: dict[str, SourceEntry], + is_reload: bool, +) -> tuple[SourceLoadPlanEntry, ...]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index d26fbb6e19..2600991963 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( build_compiled_project, + build_planner_scope, enforce_model_execution_limit, resolve_target_config, run_clone_pipeline_with_options, @@ -47,7 +48,6 @@ from sqlbuild.compiler.planner.main.clone.resolve_skipped_view_chain import ( resolve_skipped_view_chain, ) -from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope from sqlbuild.compiler.planner.models import PlannerScope, UnitTestSelectorSplit from sqlbuild.compiler.planner.types import MaterializationType from sqlbuild.executor.clone.main.build_destination_transient_models import ( diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py b/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py deleted file mode 100644 index 8b347bc534..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Planner helpers for source auto-load selection.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType - - -def managed_source_upstream_keys( - *, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - project: CompiledProjectView, -) -> frozenset[CompiledObjectKey]: - """Return selected managed source keys reachable from the current selection.""" - - managed_source_names: frozenset[str] = frozenset( - source.name for source in project.sources if source.source_entry.loader is not None - ) - direct_upstream_sources: set[CompiledObjectKey] = set() - selected_key: CompiledObjectKey - for selected_key in selected_keys: - if selected_key.resource_type != CompiledResourceType.MODEL: - continue - upstream_key: CompiledObjectKey - for upstream_key in upstream_deps.get(selected_key, ()): - if upstream_key.resource_type == CompiledResourceType.SOURCE: - direct_upstream_sources.add(upstream_key) - return frozenset(key for key in direct_upstream_sources if key.name in managed_source_names) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py deleted file mode 100644 index f921cf634e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Loader DAG planning helpers.""" - -from __future__ import annotations - -from sqlbuild._native import loader_destination_parts -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction -from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry - - -def expand_selected_loader_dependencies( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - executable_dependency_source_keys: frozenset[CompiledObjectKey] = frozenset(), -) -> tuple[frozenset[CompiledObjectKey], dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]]: - """Add intermediate loader nodes needed by selected managed source loads.""" - - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in project.loader_functions - } - source_by_loader: dict[str, SourceEntry] = { - source.source_entry.loader: source.source_entry - for source in project.sources - if source.source_entry.loader is not None - } - expanded_upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = dict(upstream_deps) - expanded_selected: set[CompiledObjectKey] = set(selected_keys) - - source_by_name: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry for source in project.sources - } - selected_key: CompiledObjectKey - for selected_key in selected_keys: - if selected_key.resource_type != CompiledResourceType.SOURCE: - continue - source_entry: SourceEntry | None = source_by_name.get(selected_key.name) - if source_entry is None: - if selected_key.name in loader_by_name: - expanded_upstream.setdefault(selected_key, ()) - continue - if source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(source_entry.loader) - if loader_function is None: - continue - expanded_upstream[selected_key] = _dependency_keys( - loader_function=loader_function, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - expanded_upstream.update( - _dependency_closure_edges( - loader_function=loader_function, - loader_by_name=loader_by_name, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - ) - if selected_key not in executable_dependency_source_keys: - continue - dependency_loader_name: str - for dependency_loader_name in upstream_loader_dependency_names( - loader_function=loader_function, - loader_functions=project.loader_functions, - ): - expanded_selected.add( - _loader_node_key( - loader_name=dependency_loader_name, - source_by_loader=source_by_loader, - ) - ) - - return frozenset(expanded_selected), expanded_upstream - - -def build_intermediate_source_map( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] -) -> dict[str, SourceEntry]: - """Return synthetic source entries for selected intermediate loader nodes.""" - - source_by_loader: dict[str, SourceEntry] = { - source.source_entry.loader: source.source_entry - for source in project.sources - if source.source_entry.loader is not None - } - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - source_names: frozenset[str] = frozenset(source.source_entry.name for source in project.sources) - entries: dict[str, SourceEntry] = {} - key: CompiledObjectKey - for key in selected_keys: - if key.resource_type != CompiledResourceType.SOURCE: - continue - if key.name in entries or key.name in source_names: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(key.name) - if loader_function is None or loader_function.name in source_by_loader: - continue - entries[key.name] = loader_to_source_entry(project=project, loader=loader_function) - return entries - - -def build_upstream_intermediate_source_map( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] -) -> dict[str, SourceEntry]: - """Return synthetic source entries for upstream intermediate loaders of selected sources.""" - - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in project.loader_functions - } - source_by_name: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry for source in project.sources - } - source_by_loader: dict[str, SourceEntry] = { - source.source_entry.loader: source.source_entry - for source in project.sources - if source.source_entry.loader is not None - } - entries: dict[str, SourceEntry] = {} - key: CompiledObjectKey - for key in selected_keys: - if key.resource_type != CompiledResourceType.SOURCE: - continue - source_entry: SourceEntry | None = source_by_name.get(key.name) - if source_entry is None or source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get(source_entry.loader) - if loader_function is None: - continue - dependency_name: str - for dependency_name in upstream_loader_dependency_names( - loader_function=loader_function, - loader_functions=project.loader_functions, - ): - if dependency_name in source_by_loader: - continue - dependency_loader: DiscoveredLoaderFunction | None = loader_by_name.get(dependency_name) - if dependency_loader is None: - continue - entries.setdefault( - dependency_name, - loader_to_source_entry(project=project, loader=dependency_loader), - ) - return entries - - -def upstream_loader_dependency_names( - *, - loader_function: DiscoveredLoaderFunction, - loader_functions: tuple[DiscoveredLoaderFunction, ...], -) -> tuple[str, ...]: - """Return upstream loader dependency names for one loader function.""" - - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in loader_functions - } - loader_name_by_function: dict[object, str] = { - loader.function: loader.name for loader in loader_functions - } - names: list[str] = [] - pending: list[str] = list( - _dependency_loader_names( - loader_function=loader_function, - loader_name_by_function=loader_name_by_function, - ) - ) - while pending: - current: str = pending.pop(0) - if current in names: - continue - names.append(current) - dependency_loader: DiscoveredLoaderFunction | None = loader_by_name.get(current) - if dependency_loader is None: - continue - pending.extend( - _dependency_loader_names( - loader_function=dependency_loader, - loader_name_by_function=loader_name_by_function, - ) - ) - return tuple(names) - - -def loader_to_source_entry( - *, project: CompiledProjectView, loader: DiscoveredLoaderFunction -) -> SourceEntry: - destination: LoaderDestinationParts = loader_destination_parts( - destination=( - loader.destination if loader.destination is not None else f"__loader__{loader.name}" - ), - default_database=project.effective_target_database, - default_schema=project.effective_target_schema, - ) - return SourceEntry( - name=loader.name, - database=destination.database, - schema=destination.schema, - table=destination.table, - loader=loader.name, - write_strategy=loader.write_strategy, - cursor_column=loader.cursor_column, - unique_key=loader.unique_key, - contract=loader.contract, - meta={"sqlbuild_loader_node": True}, - columns=loader.columns, - ) - - -def _dependency_closure_edges( - *, - loader_function: DiscoveredLoaderFunction, - loader_by_name: dict[str, DiscoveredLoaderFunction], - loader_name_by_function: dict[object, str], - source_by_loader: dict[str, SourceEntry], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - edges: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = {} - dependency_name: str - for dependency_name in _dependency_loader_names( - loader_function=loader_function, loader_name_by_function=loader_name_by_function - ): - dependency_key: CompiledObjectKey = _loader_node_key( - loader_name=dependency_name, source_by_loader=source_by_loader - ) - dependency_function: DiscoveredLoaderFunction = loader_by_name[dependency_name] - edges[dependency_key] = _dependency_keys( - loader_function=dependency_function, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - edges.update( - _dependency_closure_edges( - loader_function=dependency_function, - loader_by_name=loader_by_name, - loader_name_by_function=loader_name_by_function, - source_by_loader=source_by_loader, - ) - ) - return edges - - -def _dependency_keys( - *, - loader_function: DiscoveredLoaderFunction, - loader_name_by_function: dict[object, str], - source_by_loader: dict[str, SourceEntry], -) -> tuple[CompiledObjectKey, ...]: - return tuple( - _loader_node_key(loader_name=loader_name, source_by_loader=source_by_loader) - for loader_name in _dependency_loader_names( - loader_function=loader_function, - loader_name_by_function=loader_name_by_function, - ) - ) - - -def _dependency_loader_names( - *, - loader_function: DiscoveredLoaderFunction, - loader_name_by_function: dict[object, str], -) -> tuple[str, ...]: - return tuple( - loader_name_by_function[dependency] - for dependency in loader_function.depends_on - if dependency in loader_name_by_function - ) - - -def _loader_node_key( - *, loader_name: str, source_by_loader: dict[str, SourceEntry] -) -> CompiledObjectKey: - source_entry: SourceEntry | None = source_by_loader.get(loader_name) - return CompiledObjectKey( - resource_type=CompiledResourceType.SOURCE, - name=source_entry.name if source_entry is not None else loader_name, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py deleted file mode 100644 index 8fee6aa857..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Planner scope resolution helpers.""" - -from __future__ import annotations - -from sqlbuild._native import ( - build_downstream_deps, - build_execution_edge_origins, - build_execution_upstream_deps, - lineage_graph_names, - parse_project_selector, - resolve_graph_selection, - topologically_order_keys, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys -from sqlbuild.compiler.planner._helpers.graph.loader_dag import expand_selected_loader_dependencies -from sqlbuild.compiler.planner.models import ( - ParsedSelector, - PathSelector, - PlannerScope, - SqlTestSelection, -) -from sqlbuild.compiler.planner.types import SelectorKind - - -def build_planner_scope( - *, - project: CompiledProjectView, - select: tuple[str, ...], - exclude: tuple[str, ...], - auto_load_sources: bool, - selected_keys: frozenset[CompiledObjectKey] | None = None, - python_read_source_names: frozenset[str] = frozenset(), - sql_test_selection: SqlTestSelection | None = None, -) -> PlannerScope: - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream_deps - ) - all_keys: dict[str, CompiledObjectKey] = lineage_graph_names(project.lineage_graph) - resolved_selected_keys: frozenset[CompiledObjectKey] = ( - selected_keys - if selected_keys is not None - else resolve_graph_selection(graph=project.lineage_graph, select=select, exclude=exclude) - ) - executable_dependency_source_keys: frozenset[CompiledObjectKey] = ( - _upstream_source_selector_keys( - select=select, - all_keys=all_keys, - selected_keys=resolved_selected_keys, - ) - ) - if auto_load_sources: - auto_loaded_source_keys: frozenset[CompiledObjectKey] = managed_source_upstream_keys( - selected_keys=resolved_selected_keys, - upstream_deps=upstream_deps, - project=project, - ) - resolved_selected_keys = resolved_selected_keys | auto_loaded_source_keys - executable_dependency_source_keys = ( - executable_dependency_source_keys | auto_loaded_source_keys - ) - executable_dependency_source_keys = executable_dependency_source_keys & resolved_selected_keys - if auto_load_sources or any( - key.resource_type == CompiledResourceType.SOURCE for key in resolved_selected_keys - ): - resolved_selected_keys, upstream_deps = expand_selected_loader_dependencies( - project=project, - selected_keys=resolved_selected_keys, - upstream_deps=upstream_deps, - executable_dependency_source_keys=executable_dependency_source_keys, - ) - downstream_deps = build_downstream_deps(upstream_deps) - return PlannerScope( - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - all_keys=all_keys, - models_by_name={model.name: model for model in project.models}, - selected_keys=resolved_selected_keys, - execution_order=topologically_order_keys( - upstream=upstream_deps, - injected_edge_origins=build_execution_edge_origins(project), - ), - user_selected_keys=resolved_selected_keys, - python_read_source_names=python_read_source_names, - sql_test_selection=sql_test_selection or SqlTestSelection(), - ) - - -def _upstream_source_selector_keys( - *, - select: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - selected_keys: frozenset[CompiledObjectKey], -) -> frozenset[CompiledObjectKey]: - keys: set[CompiledObjectKey] = set() - raw_select: str - for raw_select in select: - token: str - for token in raw_select.split(): - part: str - for part in token.split(","): - parsed: ParsedSelector | PathSelector = parse_project_selector(part) - if isinstance(parsed, PathSelector) or not parsed.upstream: - continue - if parsed.kind not in {SelectorKind.NAME, SelectorKind.SOURCE}: - continue - key: CompiledObjectKey | None = all_keys.get(parsed.value) - if key is not None and key.resource_type == CompiledResourceType.SOURCE: - keys.add(key) - return frozenset(key for key in keys if key in selected_keys) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py b/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py deleted file mode 100644 index 1544e1189e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Planner helpers for executable source-load nodes.""" - -from __future__ import annotations - -from sqlbuild._native import transitive_closure_many -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_intermediate_source_map, - build_upstream_intermediate_source_map, -) -from sqlbuild.compiler.planner.models import SourceLoadPlanEntry -from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_source_load_map( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, -) -> dict[str, SourceEntry]: - """Return source entries available to source-load and source-read planning.""" - - relevant_keys: frozenset[CompiledObjectKey] | None = ( - transitive_closure_many(starts=selected_keys, edges=upstream_deps, include_starts=True) - if upstream_deps is not None - else None - ) - source_map: dict[str, SourceEntry] = { - source.source_entry.name: source.source_entry - for source in project.sources - if relevant_keys is None or source.key in relevant_keys - } - source_map.update(build_intermediate_source_map(project=project, selected_keys=selected_keys)) - source_map.update( - build_upstream_intermediate_source_map(project=project, selected_keys=selected_keys) - ) - return source_map - - -def build_source_load_entries( - *, - execution_order: tuple[CompiledObjectKey, ...], - selected_keys: frozenset[CompiledObjectKey], - source_map: dict[str, SourceEntry], - is_reload: bool, -) -> tuple[SourceLoadPlanEntry, ...]: - """Return executable source-load plan entries in DAG execution order.""" - - entries: list[SourceLoadPlanEntry] = [] - key: CompiledObjectKey - for key in execution_order: - if key not in selected_keys: - continue - if key.resource_type != CompiledResourceType.SOURCE: - continue - source_entry: SourceEntry | None = source_map.get(key.name) - if source_entry is None or source_entry.loader is None: - continue - entries.append( - SourceLoadPlanEntry( - key=key, - name=source_entry.name, - loader=source_entry.loader, - destination=source_entry.table or source_entry.name, - resource_kind=_source_load_resource_kind(source_entry), - write_strategy=source_entry.write_strategy, - cursor_column=source_entry.cursor_column, - unique_key=source_entry.unique_key, - is_reload=is_reload, - integration_kind=( - source_entry.integration_loader.kind - if source_entry.integration_loader is not None - else None - ), - ) - ) - return tuple(entries) - - -def _source_load_resource_kind(source_entry: SourceEntry) -> ExecutionResourceKind: - return ( - ExecutionResourceKind.LOADER - if source_entry.meta.get("sqlbuild_loader_node") is True - else ExecutionResourceKind.SOURCE - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 7a9d8f542e..1d7271da0a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( build_model_warnings, + build_source_load_map, cap_microbatch_bounds, check_cursor_type_consistency, count_microbatches, @@ -57,7 +58,6 @@ CompiledSeedView, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( check_full_rebuild_allowed, describe_full_rebuild_cause, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index a8bdb7ae5f..61c803dd83 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -2,7 +2,11 @@ from __future__ import annotations -from sqlbuild._native import project_lineage_views +from sqlbuild._native import ( + build_source_load_entries, + project_lineage_views, + upstream_loader_dependency_names, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -25,8 +29,6 @@ from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_context, ) -from sqlbuild.compiler.planner._helpers.graph.loader_dag import upstream_loader_dependency_names -from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_entries from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner._helpers.output.plan_entry import ( audit_is_selected, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py index 707f22d76c..22bb97e84b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py @@ -4,10 +4,9 @@ from dataclasses import replace -from sqlbuild._native import transitive_closure_many +from sqlbuild._native import build_planner_scope, transitive_closure_many from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope from sqlbuild.compiler.planner.models import ( PlannerPolicies, PlannerScope, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index ddeaf62240..b18690f71c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( build_execution_upstream_deps, + build_upstream_intermediate_source_map, classify_cursor_sql_type, get_config_cursor_bound, get_config_str, @@ -63,9 +64,6 @@ ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.migrations.constants import MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_upstream_intermediate_source_map, -) from sqlbuild.compiler.planner._helpers.planning.full_refresh import ( effectively_full_refreshed_model_names, ) diff --git a/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py b/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py deleted file mode 100644 index 5455e59d6b..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public planner entrypoints for loader DAG helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_intermediate_source_map as _build_intermediate_source_map, -) -from sqlbuild.spec.contracts.models import SourceEntry - - -def build_intermediate_source_map( - *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] -) -> dict[str, SourceEntry]: - """Return synthetic source entries for selected intermediate loader nodes.""" - - return _build_intermediate_source_map(project=project, selected_keys=selected_keys) diff --git a/src/sqlbuild/compiler/planner/main/selection/scope.py b/src/sqlbuild/compiler/planner/main/selection/scope.py deleted file mode 100644 index 035d762d25..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/scope.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public planner scope entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope as _build -from sqlbuild.compiler.planner.models import PlannerScope - - -def build_planner_scope( - *, - project: CompiledProjectView, - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - auto_load_sources: bool = False, - selected_keys: frozenset[CompiledObjectKey] | None = None, -) -> PlannerScope: - """Build the resolved planner graph scope for a project.""" - - return _build( - project=project, - select=select, - exclude=exclude, - auto_load_sources=auto_load_sources, - selected_keys=selected_keys, - ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 93a797f7ac..345612fbe9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -872,3 +872,11 @@ class EmptyCursorInputsSnapshotTestCase: expected_empty_input_names: tuple[str, ...] expected_unreadable_input_names: tuple[str, ...] expected_decision: EmptyCursorInputDecision + + +@dataclass(frozen=True) +class LoaderDependencyNamesTestCase: + """Expected first-seen loader names after resolving Python function dependencies.""" + + description: str + expected_names: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index b8f8830034..06ad20a51d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -8,7 +8,12 @@ import pytest from sqlbuild._native import ( + build_intermediate_source_map, + build_source_load_entries, + build_source_load_map, + expand_selected_loader_dependencies, project_with, + upstream_loader_dependency_names, ) from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -22,14 +27,7 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( - build_intermediate_source_map, - expand_selected_loader_dependencies, -) -from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import ( - build_source_load_entries, - build_source_load_map, -) +from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.compiler.planner._helpers.warehouse.snapshot import _build_metadata_name_filter from sqlbuild.compiler.planner.models import SourceLoadPlanEntry from sqlbuild.runtime.contracts.types import ExecutionResourceKind @@ -37,6 +35,7 @@ from sqlbuild.spec.contracts.types import SourceWriteStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( LoaderDagExpansionTestCase, + LoaderDependencyNamesTestCase, LoaderDestinationPlanningTestCase, MetadataNameFilterTestCase, SourceLoadNodesTestCase, @@ -326,3 +325,31 @@ def test_given_qualified_intermediate_destination_when_building_map_then_parses_ entry: SourceEntry = entries["fetch_orders"] assert (entry.database, entry.schema, entry.table) == test_case.expected_parts + + +@pytest.mark.parametrize( + "test_case", + [ + LoaderDependencyNamesTestCase( + description="drops non-loader function dependencies and deduplicates repeated loaders", + expected_names=("fetch_orders",), + ) + ], + ids=lambda case: case.description, +) +def test_given_mixed_function_dependencies_when_resolving_then_returns_only_known_loaders( + test_case: LoaderDependencyNamesTestCase, +) -> None: + project: CompiledProjectView = build_source_load_nodes_project() + dependency: DiscoveredLoaderFunction = project.loader_functions[0] + root: DiscoveredLoaderFunction = replace( + project.loader_functions[1], + depends_on=(dependency.function, build_source_load_nodes_project, dependency.function), + ) + assert ( + upstream_loader_dependency_names( + loader_function=root, + loader_functions=project.loader_functions, + ) + == test_case.expected_names + ) From 02b77052d36b18b1c0eefa3a54532ccb80d5286a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:00:54 +0100 Subject: [PATCH 31/32] chore: update Python boundary after planner scope port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 7da18f1a41..852366b9f6 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 43190 +remaining_lines = 42632 [[stays]] path = "executor" From eee1021a4a8197144148f9d4ec0527be479044e3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:15:47 +0100 Subject: [PATCH 32/32] fix: group SQL rule cache findings once per publication --- .../src/bindings/_helpers/rules/sql_rules.rs | 21 ++-- .../rule_engine/native_rows/_test_types.py | 9 ++ .../rule_engine/native_rows/helpers.py | 12 +- .../test_sql_rule_cache_publication.py | 104 ++++++++++++++++++ 4 files changed, 134 insertions(+), 12 deletions(-) create mode 100644 tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 5eac95993f..c8172a16bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -1,6 +1,6 @@ //! Run the planned SQL rules through native lint, reusing cached and early-lint results. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -433,17 +433,16 @@ fn write_cache<'py>( ) -> PyResult<()> { let py: Python<'py> = bucket.py(); let encode: Bound<'py, PyAny> = native(py, "encode_sql_rule_findings")?; + let mut by_path: HashMap>> = HashMap::new(); + for finding in evaluated { + let path: String = finding + .getattr("path")? + .call_method0("as_posix")? + .extract()?; + by_path.entry(path).or_default().push(finding.clone()); + } for (path, identity) in written { - let mut own: Vec> = Vec::new(); - for finding in evaluated { - let finding_path: String = finding - .getattr("path")? - .call_method0("as_posix")? - .extract()?; - if finding_path == path { - own.push(finding.clone()); - } - } + let own: &[Bound<'py, PyAny>] = by_path.get(&path).map(Vec::as_slice).unwrap_or_default(); let encoded: Bound<'py, PyAny> = call_with(&encode, &[("findings", PyList::new(py, own)?.into_any())])?; bucket.set_item(path, (identity, encoded))?; diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py index 3388c46221..9be0333b4d 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py @@ -1,6 +1,15 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class SqlRulePublicationTestCase: + """Increasing path counts and the linear conversion budget.""" + + description: str + sizes: tuple[int, ...] + expected_counts: list[tuple[int, int]] + + @dataclass(frozen=True) class NativeRulesRequestTestCase: """A compile of the rules fixture: findings and cache use.""" diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py index 00059e1f38..6fb27cb42a 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py @@ -1,7 +1,7 @@ import json import shutil from pathlib import Path -from typing import Any, cast +from typing import Any, ClassVar, cast import pytest @@ -14,6 +14,16 @@ type FindingKey = tuple[str, str, int | None, int | None, str] +class CountingRulePath(Path): + """Count finding path conversions across the real native cache publication.""" + + conversions: ClassVar[int] = 0 + + def as_posix(self) -> str: + CountingRulePath.conversions += 1 + return super().as_posix() + + def write_rules_fixture(*, project_dir: Path) -> None: """Copy the neutral fixture and select every built-in rule.""" diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py new file mode 100644 index 0000000000..78fda746e9 --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -0,0 +1,104 @@ +"""Bound native SQL rule cache publication work while preserving persisted findings.""" + +import json +from concurrent.futures import Future +from pathlib import Path +from time import monotonic +from types import SimpleNamespace +from typing import Any, cast + +import pytest + +import sqlbuild._native as native +from sqlbuild.rule_engine.models import Finding, SqlRulesEvaluation +from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( + SqlRulePublicationTestCase, +) +from tests.integration.src.sqlbuild.rule_engine.native_rows.helpers import CountingRulePath + + +@pytest.mark.parametrize( + "test_case", + [ + SqlRulePublicationTestCase( + "doubling paths and findings", (4, 8, 16), [(4, 12), (8, 24), (16, 48)] + ) + ], + ids=lambda case: case.description, +) +def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and_order_preserved( + test_case: SqlRulePublicationTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exercise planning, finding conversion, encoding and disk publication through the binding.""" + monkeypatch.setattr(native, "compiled_relation_keys", lambda project: {}) + counts: list[tuple[int, int]] = [] + started: float = monotonic() + for size in test_case.sizes: + project_dir: Path = tmp_path / str(size) + paths: list[Path] = [Path(f"models/orders_{index}.sql") for index in range(size)] + models: list[SimpleNamespace] = [ + SimpleNamespace( + relative_path=path, + authored_sql="SELECT 1 AS order_id", + query_sql="SELECT 1 AS order_id", + config=SimpleNamespace(values={}), + dynamic_column_contract=None, + ) + for path in paths + ] + project: SimpleNamespace = SimpleNamespace( + settings=SimpleNamespace(sql_analysis=True), models=models, sql_expansions={} + ) + plan: SimpleNamespace = SimpleNamespace( + project_dir=project_dir, + project=project, + model_paths=frozenset(paths), + selected=(SimpleNamespace(code="SQBRSQL004", custom=False),), + effective_config=SimpleNamespace( + thresholds={}, allow_exceptions=True, cache=SimpleNamespace(enabled=True) + ), + ) + violations: list[SimpleNamespace] = [ + SimpleNamespace( + file_path=CountingRulePath(project_dir / paths[index % (size - 1)]), + code="SQBRSQL004", + line=index + 1, + column=1, + message=f"Finding {index}", + remediation="Use an explicit projection.", + fix=None, + ) + for index in range(size) + ] + future: Future[Any] = Future() + future.set_result(SimpleNamespace(result=SimpleNamespace(violations=violations))) + CountingRulePath.conversions = 0 + result: SqlRulesEvaluation = native.run_planned_sql_rules( + plan=cast(Any, plan), + graph=cast(Any, SimpleNamespace(project=project)), + dialect="duckdb", + discovered_inputs=cast(Any, None), + prepared_sql=cast(Any, SimpleNamespace(codes=("SQBRSQL004",), future=future)), + expansion_reuse=None, + ) + conversions: int = CountingRulePath.conversions + assert result.cache_misses == size + assert len(result.findings) == size + stored: dict[str, tuple[str, str]] = native.load_sql_rule_cache( + path=native.rules_bulk_cache_path(project_dir=project_dir, file_name="sql.json") + ) + assert set(stored) == {path.as_posix() for path in paths} + expected: list[list[Finding]] = [[] for _ in paths] + for position, finding in enumerate(result.findings): + expected[position % (size - 1)].append(finding) + for path, own in zip(paths, expected, strict=True): + assert stored[path.as_posix()][1] == native.encode_sql_rule_findings(findings=own) + assert json.loads(stored[paths[-1].as_posix()][1]) == [] + # One conversion each for grouping, encoding, and duplicate-literal hints. + counts.append((size, conversions)) + assert counts == test_case.expected_counts + assert monotonic() - started < 30 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])