From aa091cf3ac9643ff8743f1dac5d14a3f478716e6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:54:21 +0100 Subject: [PATCH 01/23] refactor: delete unused compiler data-model constants and types --- python-boundary.toml | 2 +- .../compiler/analysis_session/types.py | 17 ----- src/sqlbuild/compiler/compile/constants.py | 32 -------- src/sqlbuild/compiler/compile/exceptions.py | 18 ----- src/sqlbuild/compiler/compile/models.py | 66 ----------------- src/sqlbuild/compiler/compile/types.py | 1 - .../compiler/contract_adoption/types.py | 7 -- src/sqlbuild/compiler/discovery/constants.py | 21 ------ .../compiler/fingerprints/constants.py | 3 - src/sqlbuild/compiler/lineage/models.py | 9 --- .../compiler/macro_bridge/constants.py | 1 - src/sqlbuild/compiler/python_nodes/models.py | 8 -- src/sqlbuild/compiler/references/constants.py | 22 ------ src/sqlbuild/compiler/scopes/constants.py | 73 ------------------- .../compiler/sql_analysis/constants.py | 1 - .../compiler/sql_test_glue/constants.py | 7 -- src/sqlbuild/lint/constants.py | 4 - src/sqlbuild/rule_engine/constants.py | 46 ------------ src/sqlbuild/rule_engine/models.py | 8 -- src/sqlbuild/spec/contracts/constants.py | 10 --- src/sqlbuild/spec/contracts/types.py | 13 ---- 21 files changed, 1 insertion(+), 368 deletions(-) delete mode 100644 src/sqlbuild/compiler/references/constants.py delete mode 100644 src/sqlbuild/compiler/sql_test_glue/constants.py diff --git a/python-boundary.toml b/python-boundary.toml index ddce5dfd34..d1055ee399 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 13066 +remaining_lines = 12699 [[stays]] path = "executor" diff --git a/src/sqlbuild/compiler/analysis_session/types.py b/src/sqlbuild/compiler/analysis_session/types.py index 41e9822db7..1c2694b5f1 100644 --- a/src/sqlbuild/compiler/analysis_session/types.py +++ b/src/sqlbuild/compiler/analysis_session/types.py @@ -4,27 +4,10 @@ from typing import Literal -type ShapeRows = list[tuple[str, list[tuple[str, str]]]] type ColumnRow = tuple[str, str | None, str] type DiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] -type LineageItem = tuple[str, int, int, list[tuple[str, str, str]]] -type StepRow = tuple[ShapeRows, list[str]] -type OutcomeRow = tuple[ - bool, - list[ColumnRow] | None, - str, - list[LineageItem], - bool, - bool, - list[DiagnosticRow], - bool, - str, -] -type FamilyRow = tuple[str, str, str, str, str, str | None] type ProofRow = tuple[ bool, list[ColumnRow], list[tuple[str, str | None]], list[str], str | None, bool ] -type ContractRow = tuple[str, ProofRow | None] -type FinishRow = tuple[ShapeRows, list[str], list[bool]] type QueryColumnsMode = Literal["batch", "reanalysis", "parse", "legacy"] diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 50c48ffdaf..5f7b168b20 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -6,13 +6,7 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.python_nodes.types import SqlResourceRefKind -PRESERVE_TARGET_VALUE: str = "preserve" - -ATTACHED_AUDIT_READS_OWN_DEPENDANT_CODE: str = "P005" -HOOK_READS_OWN_DEPENDANT_CODE: str = "P007" -HARD_CODED_PROJECT_RELATION_CODE: str = "P008" UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" -REFERENCE_CALL_SYNTAX_CODE: str = "P012" SQL_TEST_HELPER_REFERENCE_CODE: str = "P013" DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { "model_files": "model", @@ -47,30 +41,11 @@ "command_output_sinks": "runtime command outputs, not graph resources", "native_session": "native parse state kept for later stages, not a project input", } -HOOK_DIRECTORY_NAME: str = "hooks" -MODEL_DIRECTORY_NAME: str = "models" -NOT_NULL_AUDIT_NAME: str = "not_null" -MODEL_AUDIT_OVERRIDE_KEYS: frozenset[str] = frozenset({"by_type", "by_column"}) -CURSOR_INPUTS_CONFIG_KEY: str = "cursor_inputs" MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" -MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" -COLUMN_MIGRATE_FROM_KEY: str = "migrate_from" -MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" - -SQL_OPEN_PAREN_TOKEN: str = "(" -SQL_ARGUMENT_SEPARATOR_TOKEN: str = "," -SQL_STATEMENT_TERMINATOR_TOKEN: str = ";" -SQL_WITH_KEYWORD: str = "WITH" -SQL_CEREMONIAL_SELECT_VALUE: str = "1" -OMITTED_CEREMONIAL_SELECT_SQL: str = f"\nSELECT {SQL_CEREMONIAL_SELECT_VALUE}" -MACRO_TOKEN: str = "@" -DECLARATION_REFERENCE_NAMES: frozenset[str] = frozenset({"enum", "const"}) MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" -SQL_INTERPOLATION_TOKEN: str = "@@" -TEMPLATE_OPEN_TOKEN: str = "${" EXPECTED_TEST_CTE_PREFIX: str = "__expected__" ASSERT_TEST_CTE_PREFIX: str = "__assert__" @@ -79,13 +54,6 @@ SEED_TEST_CTE_PREFIX: str = SqlReferenceKind.SEED.fixture_cte_prefix DBT_REF_TEST_CTE_PREFIX: str = SqlReferenceKind.DBT_REF.fixture_cte_prefix TABLE_FN_TEST_CTE_PREFIX: str = SqlReferenceKind.TABLE_FUNCTION.fixture_cte_prefix -MACRO_TEST_CTE_PREFIX: str = "__macro__" -MACRO_ACTUAL_TEST_CTE_NAME: str = "__macro_actual__" -MACRO_EXPECTED_TEST_CTE_NAME: str = "__macro_expected__" -UDF_ACTUAL_TEST_CTE_NAME: str = "__udf_actual__" -UDF_EXPECTED_TEST_CTE_NAME: str = "__udf_expected__" -TABLE_FN_ACTUAL_TEST_CTE_NAME: str = "__table_fn_actual__" -TABLE_FN_EXPECTED_TEST_CTE_NAME: str = "__table_fn_expected__" DEFAULT_SQL_TEST_MODE: SqlTestMode = SqlTestMode.MODEL COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index 5a1d9961e1..d038ba2808 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -97,21 +97,3 @@ def __init__(self, message: str, *, location: SourceLocation, help: str) -> None class MacroDeclarationLookupError(CompileInputError, KeyError): """Retain authored diagnostics while honoring the Mapping lookup contract.""" - - -class SqlTestExtractionError(CompileInputError): - """Raised when a SQL test cannot be split into CTEs; names the test and offending text.""" - - def __init__( - self, - message: str, - *, - help: str | None = None, - test_index: int = 0, - token: str | None = None, - token_offset: int | None = None, - ) -> None: - super().__init__(message, help=help) - self.test_index = test_index - self.token = token - self.token_offset = token_offset diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index b0a97c1a9d..2e7f47cc58 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -192,14 +192,6 @@ class DynamicColumnContractProof: bare_dynamic_pivot: bool = False -@dataclass(frozen=True) -class NonNullFilterContext: - """Resolved relations and columns constrained by non-null filters.""" - - relations: tuple[tuple[str, dict[str, InferredNullability]], ...] - columns: frozenset[tuple[str, str]] - - @dataclass(frozen=True) class CompileModelConfig: """Pre-semantic effective config layers attached to a model input.""" @@ -371,15 +363,6 @@ class MacroExpansionResult: argument_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DeclarationExpansionResult: - """Expanded SQL and resolved declaration usage facts from one authored string.""" - - sql: str - spans: tuple[ExpansionSpan, ...] = field(default_factory=tuple) - usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class VisibleModelDeclarations: """One model's visible declarations and macros, as the native model loop resolves them.""" @@ -468,16 +451,6 @@ def declaration_expansion(self) -> DeclarationExpansionContext: ) -@dataclass(frozen=True) -class ModelInputScopeBuild: - """Model inputs paired with public declaration artifacts and updated context.""" - - inputs: tuple[ModelInput, ...] - declarations: DeclarationResolutionContext - context: ModelInputBuildContext - native_project: _native.NativeCompiledProject = field(repr=False, compare=False) - - @dataclass(frozen=True) class CompileSqlReference: """One logical SQL reference discovered from compiled SQL text.""" @@ -816,17 +789,6 @@ class ModelSqlAnalysis: dynamic_column_contract: DynamicColumnContractProof | None = None -@dataclass(frozen=True) -class ModelSqlAnalysisRequest: - """Inputs to one model's SQL analysis.""" - - model_input: ModelInput - query_sql: str - placeholders: dict[str, str] | None - cache_key: str | None - binding_schema: dict[str, dict[str, str]] | None - - @dataclass(frozen=True) class CompileSeedInput: """One discovered seed file with its attached seed metadata.""" @@ -1385,16 +1347,6 @@ class ExpansionSpan: output_end: int -@dataclass(frozen=True) -class SqlInterpolation: - """One SQL text after `@@` interpolation, or the error that stopped it, with its reads.""" - - sql: str - spans: tuple[ExpansionSpan, ...] - reads: tuple[tuple[str, str], ...] - error: str | None - - @dataclass(frozen=True) class CompiledSqlExpansion: """Process-local authored-to-expanded SQL evidence shared with compiler checks.""" @@ -1480,15 +1432,6 @@ def __post_init__(self) -> None: ) -@dataclass(frozen=True) -class SharedBindingQuery: - """Relation-stubbed binding query shared by models with identical input shapes.""" - - sql: str - stubs: dict[str, str] - key: tuple[object, ...] - - @dataclass(frozen=True) class NativeModelConfigSession: """The native config builder, validator, and inherited storage policies for one compile.""" @@ -1520,12 +1463,3 @@ class ModelValidationRequest: references: tuple[CompileSqlReference, ...] declared_columns: tuple[SchemaColumn, ...] | None query_sql: str - - -@dataclass(frozen=True) -class ModelValidatorContext: - """The project facts the model validators read, and the native config session.""" - - settings: SettingsConfig - external_sql_reference_resolver: ExternalSqlReferenceResolver | None - native_config: NativeModelConfigSession diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 38ba355622..2ef7d95c3a 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -659,5 +659,4 @@ class SqlTestMode(StrEnum): TABLE_FN = "table_fn" -type SqlReferenceScanFailure = tuple[str, int] type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] diff --git a/src/sqlbuild/compiler/contract_adoption/types.py b/src/sqlbuild/compiler/contract_adoption/types.py index f8f882a7ec..55abc7c162 100644 --- a/src/sqlbuild/compiler/contract_adoption/types.py +++ b/src/sqlbuild/compiler/contract_adoption/types.py @@ -5,13 +5,6 @@ from enum import StrEnum -class ContractAction(StrEnum): - """Supported contract command actions.""" - - DIFF = "diff" - GENERATE = "generate" - - class ContractFindingKind(StrEnum): """Stable physical-to-authored contract difference categories.""" diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 38ef4fb6c4..2f1f7cef23 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -4,8 +4,6 @@ PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.toml" LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.toml" -LEGACY_PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.yml" -LEGACY_LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.yml" YAML_FILE_SUFFIXES: frozenset[str] = frozenset({".yml", ".yaml"}) PYTHON_INIT_MODULE_STEM: str = "__init__" @@ -102,9 +100,6 @@ ) REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) -DLT_SOURCE_TYPE_SQL_DATABASE: str = "sql_database" -DLT_SOURCE_TYPE_REST_API: str = "rest_api" -DLT_SOURCE_TYPE_FILESYSTEM: str = "filesystem" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" SQL_TEST_HEADER_KEYS: frozenset[str] = frozenset( @@ -113,27 +108,12 @@ SQL_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset({"description", "tags"}) SQL_ANALYSIS_SETTING_KEY: str = "sql_analysis" SETTINGS_SECTION: str = "settings" -QUERY_CHANGE_TRACKING_SETTING_KEY: str = "query_change_tracking" -REFERENCES_SECTION: str = "references" TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" STAGED_PROMOTION_MODE: str = "staged" -DBT_SECTION: str = "dbt" -DEFAULTS_SECTION: str = "defaults" -DBT_PROJECT_DIR_KEY: str = "project_dir" -MODELS_DIRECTORY_NAME: str = "models" SQL_TESTS_OWNERSHIP_ROOT: str = "tests/unit" SQL_SCENARIOS_OWNERSHIP_ROOT: str = "tests/scenarios" -CANONICAL_AUTHORED_ROOTS: tuple[tuple[str, ...], ...] = ( - ("models",), - ("tests", "unit"), - ("tests", "scenarios"), - ("functions", "sql"), - ("sources",), -) - -NOT_NULL_AUDIT_NAME: str = "not_null" SQL_HOOK_OUTPUT_FIELDS: tuple[str, ...] = ( "statement", @@ -143,4 +123,3 @@ "kwargs", "description", ) -SQL_HOOK_IDENTITY_FIELDS: tuple[str, ...] = ("statement", "name", "definition_sql", "kwargs") diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index e85edd7507..586c4174a5 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -18,9 +18,6 @@ COLUMN_TIMESTAMP: str = "ts" NODE_TYPE_MODEL: str = "model" -NODE_TYPE_UDF: str = "udf" -NODE_TYPE_TABLE_FN: str = "table_fn" -FUNCTION_NODE_TYPES: tuple[str, str] = (NODE_TYPE_UDF, NODE_TYPE_TABLE_FN) NODE_TYPE_SEED: str = "seed" NODE_TYPE_HOOK: str = "hook" NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" diff --git a/src/sqlbuild/compiler/lineage/models.py b/src/sqlbuild/compiler/lineage/models.py index a56889257c..34d5e52baf 100644 --- a/src/sqlbuild/compiler/lineage/models.py +++ b/src/sqlbuild/compiler/lineage/models.py @@ -12,15 +12,6 @@ ) -@dataclass(frozen=True) -class PhysicalResource: - """A physical SQL identifier mapped to its SQLBuild resource.""" - - resource_type: CompiledResourceType - resource_name: str - physical_name: str - - @dataclass(frozen=True) class QualifiedLineageColumn: """A resource-qualified column in the collapsed SQLBuild lineage graph.""" diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 3dcfd15726..6016efad5f 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -21,7 +21,6 @@ STORE_MODULE_PATHS: set[str] = set() MACRO_CALL_STORE_FILE_NAME: str = "macro-calls.bin" MACRO_CALL_STORE_ENVIRONMENT_VERSION: str = "macro-call-store-v2" -VALUE_RENDERER_FIELD: str = "_value_renderer" PROJECT_ROOT_SEARCH_PATH_STAMP: int = -2 MISSING_SEARCH_PATH_STAMP: int = -1 INSTALLED_PACKAGE_DIRECTORY_NAMES: frozenset[str] = frozenset({"site-packages", "dist-packages"}) diff --git a/src/sqlbuild/compiler/python_nodes/models.py b/src/sqlbuild/compiler/python_nodes/models.py index 532d74095f..6ec5d49a17 100644 --- a/src/sqlbuild/compiler/python_nodes/models.py +++ b/src/sqlbuild/compiler/python_nodes/models.py @@ -114,14 +114,6 @@ def selected_sql_refs(self, *, selected_names: frozenset[str]) -> frozenset[SqlR return frozenset(refs) -@dataclass(frozen=True) -class PythonSqlSelection: - """Unified selector result across compiled SQL resources and Python nodes.""" - - sql_keys: frozenset[CompiledObjectKey] - python_node_names: frozenset[str] - - @dataclass(frozen=True) class PythonSqlRunSelection: """Unified run-command selection excluding Python checks.""" diff --git a/src/sqlbuild/compiler/references/constants.py b/src/sqlbuild/compiler/references/constants.py deleted file mode 100644 index 68c894a12a..0000000000 --- a/src/sqlbuild/compiler/references/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Stable SQL reference patterns.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.references.types import SqlReferenceKind - -REF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.REF.function_name)}\(" -SEED_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.SEED.function_name)}\(" -SOURCE_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.SOURCE.function_name)}\(" -DBT_REF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.DBT_REF.function_name)}\(" -UDF_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.UDF.function_name)}\(" -TABLE_FUNCTION_CALL_PREFIX: str = rf"{re.escape(SqlReferenceKind.TABLE_FUNCTION.function_name)}\(" -_QUOTED_NAME_ARGUMENT: str = r'"([^"]+)"\)' -REF_PATTERN: re.Pattern[str] = re.compile(REF_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) -SEED_PATTERN: re.Pattern[str] = re.compile(SEED_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) -SOURCE_PATTERN: re.Pattern[str] = re.compile(SOURCE_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) -UDF_PATTERN: re.Pattern[str] = re.compile(UDF_CALL_PREFIX + _QUOTED_NAME_ARGUMENT) -TABLE_FUNCTION_PATTERN: re.Pattern[str] = re.compile( - TABLE_FUNCTION_CALL_PREFIX + _QUOTED_NAME_ARGUMENT -) diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index 1e90b63f1d..922ac5749e 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -2,80 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind - -SCOPE_METADATA_SCHEMA_VERSION: int = 3 SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" SCOPE_CACHE_FILENAME: str = "scope-index.json" -DEFAULT_ENUM_MEMBER_PREVIEW: int = 20 -QUALIFIED_IDENTITY_SEPARATOR: str = ":" -PATH_SEPARATOR: str = "/" -WINDOWS_PATH_SEPARATOR: str = "\\" CURRENT_PATH_COMPONENT: str = "." DECLARATION_LEXICAL_STEM: str = "__declaration__.sql" -PARENT_PATH_COMPONENT: str = ".." -WINDOWS_DRIVE_SEPARATOR: str = ":" -WINDOWS_DRIVE_PREFIX_LENGTH: int = 2 -PUBLIC_IDENTITY_PART_COUNT: int = 2 -PRIVATE_IDENTITY_PART_COUNT: int = 3 -EMPTY_TEXT: str = "" -AVAILABLE_SECTION: str = "available" -USED_SECTION: str = "used" -RELATIONSHIP_SECTION: str = "relationship_scope" -GLOBAL_SUMMARY_POLICY: str = "summary" -GLOBAL_USED_POLICY: str = "used" -GLOBAL_ALL_POLICY: str = "all" -LIST_SECTION: str = "list" -METADATA_FIELD: str = "metadata" -KIND_COUNTS_FIELD: str = "kind_counts" -DECLARATION_KIND_VALUES: frozenset[str] = frozenset(item.value for item in DeclarationKind) - -DECLARATION_GROUP_DIRECTORY: str = "_sqlbuild" -GLOBAL_DECLARATION_DIRECTORIES: frozenset[str] = frozenset({"macros", "enums", "constants"}) -INHERITED_DECLARATION_DIRECTORIES: frozenset[str] = GLOBAL_DECLARATION_DIRECTORIES -LOCAL_DECLARATION_DIRECTORIES: frozenset[str] = frozenset({"_macros", "_enums", "_constants"}) -SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( - INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES -) -DECLARATION_DIRECTORY_FACTS: dict[str, tuple[DeclarationKind, ScopeKind]] = { - "macros": (DeclarationKind.MACRO, ScopeKind.INHERITED), - "enums": (DeclarationKind.ENUM, ScopeKind.INHERITED), - "constants": (DeclarationKind.CONSTANT, ScopeKind.INHERITED), - "_macros": (DeclarationKind.MACRO, ScopeKind.LOCAL), - "_enums": (DeclarationKind.ENUM, ScopeKind.LOCAL), - "_constants": (DeclarationKind.CONSTANT, ScopeKind.LOCAL), -} - -GLOBAL_NAMED_DECLARATION_ROOTS: dict[tuple[str, ...], DeclarationKind] = { - ("audits", "generic"): DeclarationKind.AUDIT, - ("audits", "singular"): DeclarationKind.SINGULAR_AUDIT, - ("schemas",): DeclarationKind.SCHEMA, - ("hooks", "sql"): DeclarationKind.SQL_HOOK, - ("hooks", "python"): DeclarationKind.PYTHON_HOOK, -} -GROUPED_NAMED_DECLARATION_ROLES: dict[tuple[str, ...], tuple[DeclarationKind, ScopeKind]] = { - ("audits", "generic"): (DeclarationKind.AUDIT, ScopeKind.INHERITED), - ("_audits", "generic"): (DeclarationKind.AUDIT, ScopeKind.LOCAL), - ("audits", "singular"): (DeclarationKind.SINGULAR_AUDIT, ScopeKind.INHERITED), - ("schemas",): (DeclarationKind.SCHEMA, ScopeKind.INHERITED), - ("_schemas",): (DeclarationKind.SCHEMA, ScopeKind.LOCAL), - ("hooks", "sql"): (DeclarationKind.SQL_HOOK, ScopeKind.INHERITED), - ("_hooks", "sql"): (DeclarationKind.SQL_HOOK, ScopeKind.LOCAL), - ("hooks", "python"): (DeclarationKind.PYTHON_HOOK, ScopeKind.INHERITED), - ("_hooks", "python"): (DeclarationKind.PYTHON_HOOK, ScopeKind.LOCAL), -} -GROUPED_NAMED_DECLARATION_DIRECTORIES: frozenset[str] = frozenset( - role[0] for role in GROUPED_NAMED_DECLARATION_ROLES -) -GLOBAL_NAMED_DECLARATION_DIRECTORIES: frozenset[str] = frozenset( - role[0] for role in GLOBAL_NAMED_DECLARATION_ROOTS -) -NAMED_DECLARATION_KINDS: frozenset[DeclarationKind] = frozenset( - GLOBAL_NAMED_DECLARATION_ROOTS.values() -) -DECLARATION_ROLE_PARTS: dict[DeclarationKind, tuple[str, ...]] = { - DeclarationKind.MACRO: ("macros",), - DeclarationKind.ENUM: ("enums",), - DeclarationKind.CONSTANT: ("constants",), - **{kind: role for role, kind in GLOBAL_NAMED_DECLARATION_ROOTS.items()}, -} diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 5c1744c6bf..a0dcf02871 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -17,4 +17,3 @@ POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 TYPE_CHECKED_DIALECTS: frozenset[str] = frozenset({"duckdb", "snowflake", "postgresql", "bigquery"}) -NATIVE_FAILURE_PREFIX: str = "NativeCompilerError: " diff --git a/src/sqlbuild/compiler/sql_test_glue/constants.py b/src/sqlbuild/compiler/sql_test_glue/constants.py deleted file mode 100644 index 25e03e597a..0000000000 --- a/src/sqlbuild/compiler/sql_test_glue/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Constants for the native SQL-test planning glue.""" - -from __future__ import annotations - -CURSOR_INTRINSIC_NAMES: tuple[str, ...] = ("__cursor_start", "__cursor_end") -SQL_TEST_INPUT_FAILURE: str = "input" -SQL_TEST_DECIMAL_OVERFLOW: str = "decimal_overflow" diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 951a4b5ae4..60e87c3258 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -5,7 +5,6 @@ from sqlbuild.lint.types import LintSeverity LINT_ENGINE_SQLBUILD: str = "sqlbuild" -LINT_ENGINE_NATIVE: str = "native" VIOLATION_SEVERITY_FAULT: LintSeverity = LintSeverity.FAULT VIOLATION_SEVERITY_WARNING: LintSeverity = LintSeverity.WARNING @@ -26,6 +25,3 @@ "SQBRSQL042", } ) - -RELATION_KIND_SOURCE: str = "source" -RELATION_KIND_SEED: str = "seed" diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index b388d81a9b..f599d97893 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -11,12 +11,7 @@ CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULE_PROJECT_SUBJECT: str = "" -RULES_CACHE_DIRECTORY_NAME: str = "rules-cache" -RULES_BULK_CACHE_DIRECTORY_NAME: str = "bulk" CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" -NATIVE_RULES_MEMO_FILE: str = "native-response.json" -SQL_RULES_CACHE_FILE: str = "sql.json" -NATIVE_RULES_CACHE_FILE: str = "native.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" CUSTOM_HOST_ABANDONED_SUFFIX: str = ".abandoned" @@ -71,59 +66,18 @@ "unicodedata", } ) -SKILL_FRESH: str = "fresh" -SKILL_MISSING: str = "missing" -SKILL_STALE: str = "stale" -REPLACEABLE_SKILL_STATES: frozenset[str] = frozenset({SKILL_FRESH, SKILL_MISSING, SKILL_STALE}) AST_CTE_KIND: str = "cte" CONTRACT_ENFORCED: str = "enforced" TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) -BOOLEAN_TYPE: str = "BOOLEAN" -DATE_TYPE: str = "DATE" -TIMESTAMP_TYPE: str = "TIMESTAMP" - TARGET_DIRECTORY_NAME: str = "target" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" PARENT_DIRECTORY_TOKEN: str = ".." -MIN_AUDITS_PER_MODEL: str = "min_audits_per_model" -MIN_TESTS_PER_MODEL: str = "min_tests_per_model" MIN_CUSTOM_RULE_TEST_CASES: str = "min_custom_rule_test_cases" -CUSTOM_RULE_COVERAGE_CODE: str = "SQBRTEST301" -RULES_THRESHOLD_RULE_CODES: frozenset[str] = frozenset({"SQBRTEST201", "SQBRTEST202"}) -RULES_LAYOUT_RULE_PREFIXES: tuple[str, str] = ("SQBRPROJECT2", "SQBRDECLARATION3") -RULES_LAYOUT_THRESHOLD_RULE_CODES: frozenset[str] = frozenset( - { - "SQBRPROJECT203", - "SQBRPROJECT204", - "SQBRDECLARATION303", - "SQBRDECLARATION304", - "SQBRDECLARATION306", - } -) -MAX_SUBDOMAIN_DEPTH: str = "max_subdomain_depth" -MIN_SHARED_OWNER_PREFIX_DIRECTORIES: str = "min_shared_owner_prefix_directories" -MAX_ROLE_CONTAINER_DEPTH: str = "max_role_container_depth" -MAX_MACRO_CONTAINER_FILES: str = "max_macro_container_files" -MAX_CONSTANT_CONTAINER_FILES: str = "max_constant_container_files" -MAX_ENUM_CONTAINER_FILES: str = "max_enum_container_files" -MIN_SHARED_CONTAINER_PREFIX_FILES: str = "min_shared_container_prefix_files" -RULES_THRESHOLD_DEFAULTS: dict[str, int] = { - MIN_AUDITS_PER_MODEL: 1, - MIN_TESTS_PER_MODEL: 1, - MIN_CUSTOM_RULE_TEST_CASES: 1, - MAX_SUBDOMAIN_DEPTH: 1, - MIN_SHARED_OWNER_PREFIX_DIRECTORIES: 2, - MAX_ROLE_CONTAINER_DEPTH: 1, - MAX_MACRO_CONTAINER_FILES: 10, - MAX_CONSTANT_CONTAINER_FILES: 10, - MAX_ENUM_CONTAINER_FILES: 10, - MIN_SHARED_CONTAINER_PREFIX_FILES: 2, -} RULE_FIX_AVAILABLE_NOTE: str = ( "Fix available: `sqb format --fix` applies it after compiler verification" ) diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 610da26686..e13f8ce197 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -380,14 +380,6 @@ class RulesRunResult: skipped_type_proof_rules: tuple[str, ...] = () -@dataclass(frozen=True) -class ResolvedRuleset: - catalogue: tuple[Rule, ...] - rules: tuple[Rule, ...] - fingerprint: str - cacheable: bool - - @dataclass(frozen=True) class RuleFile: path: str diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index 658c3c4a6a..fa9fd13c6e 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -5,15 +5,5 @@ from sqlbuild.spec.contracts.models import SeedCsvSettings DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() -CURSOR_POLICY_DISABLED: str = "disabled" ZERO_DAY_CURSOR_DURATION: str = "0d" -EFFECTIVE_BATCH_SIZE_TOKEN: str = "effective" -SCENARIO_RUN_NAMESPACE_CONFIG_KEY: str = "run_namespace" -TIME_TRAVEL_RETENTION_MATERIALIZATIONS: tuple[str, ...] = ( - "table", - "incremental", - "snapshot", -) -CONNECTION_WAREHOUSE_KEY: str = "warehouse" -UNLIMITED_DIFF_FULL_ROWS: str = "unlimited" DEFAULT_DIFF_MAX_FULL_ROWS: int = 10_000_000 diff --git a/src/sqlbuild/spec/contracts/types.py b/src/sqlbuild/spec/contracts/types.py index b505530fe8..38e32f06c2 100644 --- a/src/sqlbuild/spec/contracts/types.py +++ b/src/sqlbuild/spec/contracts/types.py @@ -71,13 +71,6 @@ class TimeTravelRetentionSource(StrEnum): MODEL = "model" -class TimeTravelRetentionValue(StrEnum): - """Named non-duration retention values.""" - - INHERIT = "inherit" - DISABLED = "disabled" - - class TableType(StrEnum): """Snowflake managed table type.""" @@ -94,12 +87,6 @@ class TableTypeSource(StrEnum): MODEL = "model" -class TableTypeValue(StrEnum): - """Named non-type table-type value.""" - - INHERIT = "inherit" - - class TableTypeDowngradePolicy(StrEnum): """Permanent-to-transient conversion policy.""" From 7dcdb28aa2bed1c943fe74d66a641be609643c57 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:38:31 +0100 Subject: [PATCH 02/23] refactor: move Rust-read compiler constants into the native extension --- .../compile_attachment/declaration_scope.rs | 9 +- .../compile_attachment/model_inputs.rs | 20 +++- .../_helpers/compile_attachment/target.rs | 46 ++----- .../_helpers/compile_command/constants.rs | 5 + .../_helpers/compile_command/report.rs | 9 +- .../_helpers/compile_command/reuse_attempt.rs | 12 +- .../_helpers/compile_driver/analysis.rs | 5 +- .../_helpers/compile_driver/analysis_cache.rs | 9 +- .../_helpers/compile_driver/discovery.rs | 8 +- .../_helpers/compile_driver/project.rs | 21 ++-- .../_helpers/compiled_project/attachment.rs | 19 ++- .../bindings/_helpers/diff_report/command.rs | 5 +- .../bindings/_helpers/discovery/constants.rs | 113 ++++++++++++++++++ .../_helpers/discovery/declaration_records.rs | 7 +- .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/model_records.rs | 5 +- .../_helpers/discovery/sorted_keys.rs | 19 +-- .../_helpers/discovery/sql_test_records.rs | 13 +- .../_helpers/frontier/cache_directory.rs | 23 ++-- .../bindings/_helpers/frontier/constants.rs | 9 ++ .../src/bindings/_helpers/frontier/mod.rs | 1 + .../_helpers/render/macro_expansion_result.rs | 18 +-- .../src/bindings/_helpers/rules/constants.rs | 15 +++ .../bindings/_helpers/rules/early_sql_lint.rs | 15 +-- .../bindings/_helpers/rules/fix_session.rs | 5 +- .../src/bindings/_helpers/rules/selection.rs | 8 +- .../src/bindings/_helpers/rules/sql_rules.rs | 4 +- .../bindings/_helpers/scopes/scope_loading.rs | 24 ++-- .../_helpers/semantic_checks/stage_objects.rs | 14 +-- .../_helpers/sql_test_glue/compiled_tests.rs | 5 +- .../_helpers/sql_test_glue/constants.rs | 6 + .../bindings/_helpers/sql_test_glue/mod.rs | 2 +- .../_helpers/sql_test_glue/plan_errors.rs | 9 +- .../src/scope_index/constants.rs | 2 +- python-boundary.toml | 2 +- .../_helpers/comparing/compare.py | 7 +- .../_helpers/corpus/emitted_codes.py | 2 +- .../_helpers/coverage/render.py | 2 +- .../classes/capture_file.py | 2 +- scripts/compiler_differential/constants.py | 7 ++ .../compiler/analysis_session/constants.py | 4 - src/sqlbuild/compiler/auditing/constants.py | 6 - src/sqlbuild/compiler/compile/constants.py | 55 --------- src/sqlbuild/compiler/discovery/constants.py | 109 ----------------- src/sqlbuild/compiler/frontier/constants.py | 13 -- .../compiler/macro_bridge/constants.py | 4 - src/sqlbuild/compiler/scopes/constants.py | 8 -- .../compiler/sql_analysis/constants.py | 2 - src/sqlbuild/lint/constants.py | 16 --- src/sqlbuild/rule_engine/constants.py | 11 -- src/sqlbuild/spec/contracts/constants.py | 1 - .../cli/commands/main/compile/helpers.py | 15 +-- .../compile/test_required_descriptions.py | 17 ++- .../cli/commands/main/lineage/test_lineage.py | 3 +- .../cli/commands/main/scope/helpers.py | 4 +- .../src/sqlbuild/cli/commands/main/helpers.py | 3 +- .../compiler/analysis_session/helpers.py | 3 +- .../sqlbuild/compiler/discovery/helpers.py | 25 +++- .../_helpers/sql_test_assembly/helpers.py | 9 +- .../src/sqlbuild/compiler/scopes/helpers.py | 4 +- .../compile/_helpers/test_descriptions.py | 38 +++++- .../sqlbuild/compiler/discovery/helpers.py | 77 +++++++++++- .../main/test_serialize_hook_entries.py | 10 +- .../compiler/frontier/test_stage_capture.py | 7 +- .../_helpers/sql_test_assembly/helpers.py | 17 ++- .../compiler/scopes/test_offline_cache.py | 4 +- tests/unit/src/sqlbuild/lint/test_run_lint.py | 4 +- .../rule_engine/_helpers/engine/helpers.py | 8 +- .../rule_engine/main/skills/test_skills.py | 7 +- 69 files changed, 497 insertions(+), 495 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs delete mode 100644 src/sqlbuild/compiler/analysis_session/constants.py delete mode 100644 src/sqlbuild/compiler/auditing/constants.py delete mode 100644 src/sqlbuild/compiler/scopes/constants.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index 82f8bd7b0a..d3af02c52d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -9,10 +9,10 @@ use crate::bindings::_helpers::render::declaration_values::declaration_scope_res use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_validation::validate_scope_index; +use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCAN_CACHE_MODULE: &str = "sqlbuild._native"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; @@ -100,11 +100,8 @@ pub(crate) fn build_declaration_scope<'py>( py, [ scan_cache.call_method0("opened")?, - imported( - py, - COMPILE_CONSTANTS_MODULE, - "SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM", - )?, + pyo3::types::PyString::new(py, SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM) + .into_any(), ], )? .into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index ca5b0aebec..b312e9695b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -19,7 +19,6 @@ use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_c const NATIVE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; @@ -29,6 +28,20 @@ const PROJECT_VARS_MODULE: &str = "sqlbuild._native"; const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; +const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; +/// The explicit reference call kinds and the `SqlResourceRefKind` member each resolves to. +const EXPLICIT_REFERENCE_KINDS: [(&str, &str); 3] = + [("ref", "MODEL"), ("source", "SOURCE"), ("seed", "SEED")]; + +/// The explicit reference call kinds mapped to their `SqlResourceRefKind` members. +fn explicit_reference_kinds(py: Python<'_>) -> PyResult> { + let kinds: Bound<'_, PyAny> = imported(py, PYTHON_NODE_TYPES_MODULE, "SqlResourceRefKind")?; + let built: Bound<'_, PyDict> = PyDict::new(py); + for (call_kind, member) in EXPLICIT_REFERENCE_KINDS { + built.set_item(call_kind, kinds.getattr(member)?)?; + } + Ok(built) +} fn dict<'py>( py: Python<'py>, @@ -136,10 +149,7 @@ fn sql_hook_inputs<'py>( "sql_resource_ref", imported(py, PYTHON_NODE_MODELS_MODULE, "SqlResourceRef")?, ), - ( - "explicit_kinds", - imported(py, COMPILE_CONSTANTS_MODULE, "EXPLICIT_REFERENCE_KINDS")?, - ), + ("explicit_kinds", explicit_reference_kinds(py)?.into_any()), ( "declaration_identity", imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index df0f86093d..54674bfb26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -5,16 +5,20 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::compile_attachment::effective_config::project_vars; +use crate::bindings::_helpers::compile_command::constants::{ + COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; +use crate::bindings::_helpers::frontier::constants::{ + COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, +}; use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; -const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -73,15 +77,12 @@ fn compile_cache_root<'py>( no_cache: bool, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let constant = |name: &str| imported(py, COMPILE_CONSTANTS_MODULE, name); - let disabled_value: String = constant("COMPILE_CACHE_DISABLE_VALUE")?.extract()?; - let disable_env: String = constant("COMPILE_CACHE_DISABLE_ENV_VAR")?.extract()?; let disabled: bool = py .import("os")? .getattr("environ")? - .call_method1("get", (disable_env,))? + .call_method1("get", (COMPILE_CACHE_DISABLE_ENV_VAR,))? .extract::>()? - .is_some_and(|value| value == disabled_value); + .is_some_and(|value| value == COMPILE_CACHE_DISABLE_VALUE); if no_cache || project_dir.is_none() || (!target_config.is_none() @@ -100,39 +101,14 @@ fn compile_cache_root<'py>( /// Delete the fact and reference caches older releases kept under each retired cache root. fn remove_retired_caches(root: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = root.py(); - let directory_name: String = imported( - py, - FRONTIER_CONSTANTS_MODULE, - "COMPILER_CACHE_DIRECTORY_NAME", - )? - .extract()?; - let names: [Bound<'_, PyAny>; 2] = [ - imported( - py, - COMPILE_CONSTANTS_MODULE, - "RETIRED_FACT_CACHE_DIRECTORY_NAME", - )?, - imported( - py, - COMPILE_CONSTANTS_MODULE, - "RETIRED_REFERENCE_CACHE_DIRECTORY_NAME", - )?, - ]; let rmtree: Bound<'_, PyAny> = imported(py, "shutil", "rmtree")?; let ignore: Bound<'_, PyDict> = PyDict::new(py); ignore.set_item("ignore_errors", true)?; - for suffix in imported( - py, - FRONTIER_CONSTANTS_MODULE, - "RETIRED_CACHE_NAMESPACE_SUFFIXES", - )? - .try_iter()? - { - let suffix: String = suffix?.extract()?; - for name in &names { + for suffix in RETIRED_CACHE_NAMESPACE_SUFFIXES { + for name in RETIRED_CACHE_DIRECTORY_NAMES { let retired: Bound<'_, PyAny> = root .getattr("parent")? - .div(format!("{directory_name}{suffix}"))? + .div(format!("{COMPILER_CACHE_DIRECTORY_NAME}{suffix}"))? .div(name)?; if retired.call_method0("is_dir")?.is_truthy()? { rmtree.call((retired,), Some(&ignore))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs index b278291eed..5d1a7f85fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/constants.rs @@ -79,3 +79,8 @@ pub(crate) const REUSE_ATTEMPT_FIELDS: (&[&str], &[&str]) = ( "json_output", ], ); +/// The variable that disables the compile cache when it holds `COMPILE_CACHE_DISABLE_VALUE`. +pub(crate) const COMPILE_CACHE_DISABLE_ENV_VAR: &str = "SQLBUILD_DISABLE_COMPILE_CACHE"; +pub(crate) const COMPILE_CACHE_DISABLE_VALUE: &str = "1"; +/// The fact and reference cache directories older releases kept under each retired cache root. +pub(crate) const RETIRED_CACHE_DIRECTORY_NAMES: [&str; 2] = ["facts-v1", "references-v3"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 18eeddf21a..2a39b14a90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -18,10 +18,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::compile_outputs::json_report::{ emit_json_report, emit_orjson_report, }; +use crate::bindings::_helpers::discovery::constants::SQL_HOOK_OUTPUT_FIELDS; use crate::bindings::_helpers::discovery::hook_entries::hook_entries; use crate::bindings::_helpers::semantic_shapes::coverage::semantic_coverage; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const METADATA_MODULE: &str = "importlib.metadata"; @@ -631,12 +631,7 @@ fn model_resource<'py>( if !hooks.is_none() { item.set_item( hook_key, - hook_entries( - &hooks, - &imported(py, DISCOVERY_CONSTANTS_MODULE, "SQL_HOOK_OUTPUT_FIELDS")? - .extract::>()?, - None, - )?, + hook_entries(&hooks, &SQL_HOOK_OUTPUT_FIELDS.map(str::to_owned), None)?, )?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs index 0305e5f21d..c8271b4702 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs @@ -14,9 +14,10 @@ use sqlbuild_cli::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_FOR use crate::bindings::_helpers::cli::launch_state::launch_record_path; use crate::bindings::_helpers::compile_command::constants::{ - COMPILE_COMMAND_NAME, COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, - EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_ATTRIBUTE, EXCLUDED_ROOT_DIRECTORIES, - MILLISECONDS_PER_SECOND, MISSING_ENVIRONMENT_VALUE, MISSING_FILE_DIGEST, MISSING_PATH_MTIME_NS, + COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, COMPILE_COMMAND_NAME, + COMPILED_DIRECTORY_NAME, COMPUTE_LOG_DIRECTORY_NAME, EXCLUDED_DIRECTORIES, + EXCLUDED_ROOT_ATTRIBUTE, EXCLUDED_ROOT_DIRECTORIES, MILLISECONDS_PER_SECOND, + MISSING_ENVIRONMENT_VALUE, MISSING_FILE_DIGEST, MISSING_PATH_MTIME_NS, NATIVE_REUSE_DIRECTORY_NAME, PRESENCE_ONLY_FILE_SUFFIXES, PROJECT_CONFIG_FILENAMES, PROJECT_ROOT_PATH_MTIME_NS, RACY_WINDOW_ATTRIBUTE, RACY_WINDOW_NS, RETIRED_REUSE_DIRECTORY_NAME, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_DISABLE_ENV_VAR, @@ -33,7 +34,6 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const REUSE_REQUEST: (&str, &str) = ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"); -const COMPILE_CONSTANTS: &str = "sqlbuild.compiler.compile.constants"; const RUNTIME_IDENTITY_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.runtime_identity"; const STARTUP_INPUTS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.startup_inputs"; const COMPUTE_LOG_TEE_MODULE: &str = "sqlbuild.runtime.compute_logs.classes.text_tee"; @@ -87,8 +87,8 @@ pub(crate) fn reuse_rules(py: Python<'_>, with_output_files: bool) -> PyResult( imported(py, COMPILE_TYPES_MODULE, "DiagnosticPhase")?.getattr("RULE")?; let severity: Bound<'py, PyAny> = imported(py, COMPILE_TYPES_MODULE, "DiagnosticSeverity")?.getattr("ERROR")?; - let fix_note: Bound<'py, PyAny> = - imported(py, RULE_CONSTANTS_MODULE, "RULE_FIX_AVAILABLE_NOTE")?; + let fix_note: Bound<'py, PyAny> = PyString::new(py, RULE_FIX_AVAILABLE_NOTE).into_any(); let mut diagnostics: Vec> = core.iter().collect(); for fault in rules_result.getattr("findings")?.try_iter()? { let fault: Bound<'py, PyAny> = fault?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs index 28137f8425..a597342342 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -19,7 +19,8 @@ const LOCAL_QUALNAME_MARKER: &str = ""; const PACKAGE_NAME: &str = "sqlbuild"; const UNKNOWN_VERSION: &str = "unknown"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +/// Sorted; the dialects whose expression types compile checks. +const TYPE_CHECKED_DIALECTS: [&str; 4] = ["bigquery", "duckdb", "postgresql", "snowflake"]; const METRIC_MODULE: &str = "sqlbuild.compiler.profiling.main._metric"; const METADATA_MODULE: &str = "importlib.metadata"; const CONTEXT_REQUIRED: [&str; 3] = ["root", "inference_profile", "allow_compact_analysis"]; @@ -57,11 +58,7 @@ pub(crate) fn analysis_cache_context<'py>( shared.set_item("rich_type_inference", rich_type_inference)?; shared.set_item( "type_checked_dialects", - imported(py, "builtins", "sorted")?.call1((imported( - py, - SQL_ANALYSIS_CONSTANTS_MODULE, - "TYPE_CHECKED_DIALECTS", - )?,))?, + PyList::new(py, TYPE_CHECKED_DIALECTS)?, )?; shared.set_item("inference_profile", profile)?; let digests = (|| -> PyResult<(String, String)> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 3db998718a..6048f80a10 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -33,7 +33,7 @@ const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.discovery"; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const EVENT_EXPORTERS_MODULE: &str = "sqlbuild.runtime.event_exporting.main._configure_discovered_event_exporters"; const FILESYSTEM_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.core"; @@ -226,11 +226,7 @@ fn assembled_inputs<'py>( sql_analysis_override.clone() } else if local_config .getattr("setting_overrides")? - .contains(imported( - py, - DISCOVERY_CONSTANTS_MODULE, - "SQL_ANALYSIS_SETTING_KEY", - )?)? + .contains(SQL_ANALYSIS_SETTING_KEY)? { local_config.getattr("settings")?.getattr("sql_analysis")? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 103aac491a..a180c5e8ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -25,6 +25,7 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild._native"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild._native"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; @@ -236,18 +237,14 @@ fn targeted_project<'py>( DISCOVERY_CONSTANTS_MODULE, "TABLE_PROMOTION_MODE_SETTING_KEY", )?; - let settings_file: Bound<'py, PyAny> = imported( - py, - DISCOVERY_CONSTANTS_MODULE, - if local_config - .getattr("setting_overrides")? - .contains(promotion_key)? - { - "LOCAL_CONFIG_FILENAME" - } else { - "PROJECT_CONFIG_FILENAME" - }, - )?; + let settings_file: Bound<'py, PyAny> = if local_config + .getattr("setting_overrides")? + .contains(promotion_key)? + { + pyo3::types::PyString::new(py, LOCAL_CONFIG_FILENAME).into_any() + } else { + imported(py, DISCOVERY_CONSTANTS_MODULE, "PROJECT_CONFIG_FILENAME")? + }; let conflicts: Bound<'py, PyAny> = call_with( &imported(py, PROMOTION_MODULE, "promotion_conflict_diagnostics")?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 2bb97a8e51..edbc5f2c56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -19,11 +19,11 @@ use crate::bindings::_helpers::compiled_project::source_loop::SourceRow; use crate::bindings::_helpers::compiled_project::sql_test_loop::{ TestBlockRow, TestCaseRow, TestFileRow, }; +use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; @@ -31,7 +31,7 @@ const BUILTIN_AUDITS_MODULE: &str = "sqlbuild.compiler.auditing.main._builtins"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PYTHON_RESOURCES_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.python_resources"; const PATHLIB_MODULE: &str = "pathlib"; @@ -362,10 +362,7 @@ fn sql_test_settings<'py>( settings.set_item("scan_store", scan_cache.call_method0("opened")?)?; settings.set_item( "scan_key", - ( - imported(py, COMPILE_CONSTANTS_MODULE, "SQL_TEST_CTE_SCAN_ALGORITHM")?, - syntax.getattr("cache_key")?, - ), + (SQL_TEST_CTE_SCAN_ALGORITHM, syntax.getattr("cache_key")?), )?; Ok(settings) } @@ -457,9 +454,6 @@ fn audit_settings<'py>( /// Each loader's description, relative path and whether that path is a YAML file. fn loader_rows<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = discovered.py(); - let suffixes: Bound<'py, PyAny> = - imported(py, DISCOVERY_CONSTANTS_MODULE, "YAML_FILE_SUFFIXES")?; let mut rows: HashMap = HashMap::new(); for loader in items(discovered, "loader_functions")? { let relative_path: Bound<'py, PyAny> = loader.getattr("relative_path")?; @@ -468,7 +462,12 @@ fn loader_rows<'py>(discovered: &Bound<'py, PyAny>) -> PyResult()? + .as_str(), + ), ), ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index 23776640f3..a9a614ba8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -1,5 +1,6 @@ //! `sqb diff`: validate, discover, run a model or raw-query comparison and publish its output. +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyTuple}; use sqlbuild_cli::diff_report::errors::DiffUserError; @@ -33,7 +34,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; -const CONTRACT_CONSTANTS_MODULE: &str = "sqlbuild.spec.contracts.constants"; +const DEFAULT_DIFF_MAX_FULL_ROWS: i64 = 10_000_000; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const EXECUTE_MODULE: &str = "sqlbuild.executor.diff.main.execute"; const DIFF_MODELS_MODULE: &str = "sqlbuild.executor.diff.models"; @@ -239,7 +240,7 @@ fn target_max_full_rows<'py>( .getattr("diff")? .getattr("max_full_rows")?; if configured.is_none() { - return imported(py, CONTRACT_CONSTANTS_MODULE, "DEFAULT_DIFF_MAX_FULL_ROWS"); + return DEFAULT_DIFF_MAX_FULL_ROWS.into_bound_py_any(py); } if configured.is_instance_of::() && !configured.is_instance_of::() { return Ok(configured); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs new file mode 100644 index 0000000000..ecba49d408 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs @@ -0,0 +1,113 @@ +//! Header keys native discovery accepts, sorted by code point as Python sorts them. + +pub(crate) const SQL_MODEL_HEADER_KEYS: [&str; 67] = [ + "alias", + "append_cursor_inclusive", + "audit_factories", + "audits", + "batch_concurrency", + "batch_size", + "check_columns", + "columns", + "config", + "constants", + "contract", + "cursor", + "cursor_end", + "cursor_future_action", + "cursor_future_max_distance", + "cursor_grain", + "cursor_inputs", + "cursor_start", + "cursor_start_max_action", + "cursor_start_max_ahead", + "cursor_type", + "cursor_watermark_mode", + "database", + "description", + "dynamic_columns", + "enabled", + "enums", + "full_refresh", + "historical_input", + "incremental_mode", + "incremental_strategy", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "max_microbatches", + "merge_exclude_columns", + "microbatch_limit", + "microbatch_strategy", + "migrate_force", + "migrate_from", + "model_schema", + "observed_at", + "old_name_view", + "on_schema_change", + "placeholders", + "post_hooks", + "pre_hooks", + "replay_on_change", + "row_diff_exclude_columns", + "row_diff_sample_rows", + "row_diff_sample_seed", + "row_diff_tolerances", + "schema", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "sql_analysis", + "sql_validation", + "table_type", + "tags", + "time_travel_retention", + "unaccounted_partition_policy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", +]; +pub(crate) const REMOVED_SQL_MODEL_HEADER_KEYS: [&str; 1] = ["run_despite_unchanged"]; +pub(crate) const SQL_FUNCTION_HEADER_KEYS: [&str; 6] = [ + "arguments", + "database", + "description", + "returns", + "schema", + "tags", +]; +pub(crate) const SQL_AUDIT_HEADER_KEYS: [&str; 11] = [ + "always_run", + "evaluation", + "minimum_samples", + "name", + "run_scope", + "sample_count", + "sample_unit", + "severity", + "sql_analysis", + "thresholds", + "value", +]; +pub(crate) const SQL_HOOK_HEADER_KEYS: [&str; 1] = ["description"]; +pub(crate) const SQL_TEST_HEADER_KEYS: [&str; 7] = [ + "cases", + "cursor_end", + "cursor_start", + "mode", + "name", + "parameters", + "sql_analysis", +]; +pub(crate) const SQL_SCENARIO_HEADER_KEYS: [&str; 2] = ["description", "tags"]; +/// The fields a SQL hook entry reports, in output order. +pub(crate) const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ + "statement", + "name", + "relative_path", + "definition_sql", + "kwargs", + "description", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 125dda07a5..5c0e6178eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -17,6 +17,7 @@ use sqlbuild_sqltext::compiler::models::AuthoredValue; use std::sync::Arc; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::header_declarations::typed_constant_record; use crate::bindings::_helpers::discovery::project_files::path_object; @@ -379,9 +380,9 @@ pub(crate) fn discovery_session<'py>( inner: DiscoverySession::new( project_root(project_dir)?, DeclarationFileOptions { - function_keys: sorted_keys(py, "SQL_FUNCTION_HEADER_KEYS")?, - audit_keys: sorted_keys(py, "SQL_AUDIT_HEADER_KEYS")?, - hook_keys: sorted_keys(py, "SQL_HOOK_HEADER_KEYS")?, + function_keys: sorted_keys(&constants::SQL_FUNCTION_HEADER_KEYS), + audit_keys: sorted_keys(&constants::SQL_AUDIT_HEADER_KEYS), + hook_keys: sorted_keys(&constants::SQL_HOOK_HEADER_KEYS), }, ), }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 991c53a503..5b7d67aa84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod configuration; +pub(crate) mod constants; pub(crate) mod declaration_files; pub(crate) mod declaration_records; pub(crate) mod description_inputs; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index d3d808391f..e7bba18f30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -8,6 +8,7 @@ use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOpti use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; @@ -74,8 +75,8 @@ pub(crate) fn discovered_model_files<'py>( let py: Python<'py> = project_dir.py(); let (extract_implicit_alias_columns, extract_output_column_locations) = extraction; let options = ModelFileOptions { - supported_keys: sorted_keys(py, "SQL_MODEL_HEADER_KEYS")?, - removed_keys: sorted_keys(py, "REMOVED_SQL_MODEL_HEADER_KEYS")?, + supported_keys: sorted_keys(&constants::SQL_MODEL_HEADER_KEYS), + removed_keys: sorted_keys(&constants::REMOVED_SQL_MODEL_HEADER_KEYS), extract_implicit_alias_columns, extract_output_column_locations, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs index a32dc24efb..f377167577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs @@ -1,19 +1,8 @@ //! Header key sets as the sorted lists native discovery options take. -use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyFrozenSet, PyFrozenSetMethods}; - -use crate::bindings::_helpers::compile_driver::python_calls::imported; - -const CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; - -/// `sorted(constants.)`: a header key set, sorted by code point as Python sorts it. -pub(crate) fn sorted_keys(py: Python<'_>, name: &str) -> PyResult> { - let keys: Bound<'_, PyFrozenSet> = imported(py, CONSTANTS_MODULE, name)?.downcast_into()?; - let mut keys: Vec = keys - .iter() - .map(|key| key.extract()) - .collect::>()?; +/// A header key set as owned keys, sorted by code point as Python sorts them. +pub(crate) fn sorted_keys(keys: &[&str]) -> Vec { + let mut keys: Vec = keys.iter().map(|key| (*key).to_owned()).collect(); keys.sort(); - Ok(keys) + keys } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 1ea062a0cc..827da0c072 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -16,6 +16,7 @@ use sqlbuild_discovery::sql_tests::models::{ use sqlbuild_discovery::tree::models::ProjectTree; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; use crate::bindings::_helpers::discovery::project_files::path_object; use crate::bindings::_helpers::discovery::raised::{ @@ -47,10 +48,10 @@ type Discoverer = fn( type RecordBuilder<'py, T> = fn((&Bound<'py, PyAny>, &Bound<'py, PyAny>), T) -> PyResult>; -fn options(py: Python<'_>) -> PyResult { +fn options() -> PyResult { Ok(SqlTestFileOptions { - test_keys: sorted_keys(py, "SQL_TEST_HEADER_KEYS")?, - scenario_keys: sorted_keys(py, "SQL_SCENARIO_HEADER_KEYS")?, + test_keys: sorted_keys(&constants::SQL_TEST_HEADER_KEYS), + scenario_keys: sorted_keys(&constants::SQL_SCENARIO_HEADER_KEYS), }) } @@ -72,7 +73,7 @@ fn discovered_files<'py, T: Send>( in_snapshot(project_dir, || { let tree: Bound<'py, NativeProjectTree> = project_tree(project_dir)?; let root = project_root(project_dir)?; - let options: SqlTestFileOptions = options(py)?; + let options: SqlTestFileOptions = options()?; let discovered: Result>, StageFailure> = { let inner = &tree.get().inner; py.detach(|| discover(&root, inner, &options)) @@ -284,7 +285,7 @@ pub(crate) fn parse_sql_test_file<'py>( ) -> PyResult> { let py: Python<'py> = file_path.py(); let path: String = file_path.str()?.to_str()?.to_owned(); - let options: SqlTestFileOptions = options(py)?; + let options: SqlTestFileOptions = options()?; let parsed = py.detach(|| parse_sql_test_text(&path, contents, &options)); let file: DiscoveredSqlTestFile = parsed.map_err(|failure| failure_error(py, failure))?; test_blocks(file_path, file.blocks, file.failure) @@ -300,7 +301,7 @@ pub(crate) fn parse_sql_scenario_file<'py>( ) -> PyResult> { let py: Python<'py> = file_path.py(); let path: String = file_path.str()?.to_str()?.to_owned(); - let options: SqlTestFileOptions = options(py)?; + let options: SqlTestFileOptions = options()?; let parsed = py.detach(|| parse_scenario_text(&path, contents, &options)); let file: DiscoveredScenarioFile = parsed.map_err(|failure| failure_error(py, failure))?; scenario_file((file_path, relative_path), file) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs index a24bff111a..8f958be39b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/cache_directory.rs @@ -1,26 +1,19 @@ //! The project-local compiler store root. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; - -const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; +use crate::bindings::_helpers::compile_command::constants::TARGET_DIRECTORY_NAME; +use crate::bindings::_helpers::frontier::constants::{ + CACHE_DIRECTORY_NAME, COMPILER_CACHE_DIRECTORY_NAME, NATIVE_CACHE_NAMESPACE_SUFFIX, +}; /// `project_dir / target / cache / compiler`, keeping the path type of `project_dir`. pub(crate) fn cache_directory<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let constant = |name: &str| -> PyResult { - imported(py, FRONTIER_CONSTANTS_MODULE, name)?.extract() - }; - let store: String = format!( - "{}{}", - constant("COMPILER_CACHE_DIRECTORY_NAME")?, - constant("NATIVE_CACHE_NAMESPACE_SUFFIX")? - ); + let store: String = format!("{COMPILER_CACHE_DIRECTORY_NAME}{NATIVE_CACHE_NAMESPACE_SUFFIX}"); project_dir - .div(constant("TARGET_DIRECTORY_NAME")?)? - .div(constant("CACHE_DIRECTORY_NAME")?)? + .div(TARGET_DIRECTORY_NAME)? + .div(CACHE_DIRECTORY_NAME)? .div(store) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs new file mode 100644 index 0000000000..2aea69d91f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/constants.rs @@ -0,0 +1,9 @@ +//! Compiler store directory names and stage capture numbering. + +pub(crate) const CACHE_DIRECTORY_NAME: &str = "cache"; +pub(crate) const COMPILER_CACHE_DIRECTORY_NAME: &str = "compiler"; +/// The namespace suffix of the current native compiler store. +pub(crate) const NATIVE_CACHE_NAMESPACE_SUFFIX: &str = "-native-v1"; +/// The namespace suffixes of compiler store roots older releases wrote. +pub(crate) const RETIRED_CACHE_NAMESPACE_SUFFIXES: [&str; 3] = + ["", "-native-v1", "-native-preview-v1"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs index d752ee27cf..45ef619c78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod cache_directory; pub(crate) mod compile_frontier; +pub(crate) mod constants; pub(crate) mod encoder; pub(crate) mod python_types; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 1ee2b2e061..a3882b0bcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -3,7 +3,7 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{ @@ -31,7 +31,15 @@ const TYPED_REFERENCE_FUNCTIONS: [(&str, &str); 3] = [ ]; const USER_MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; -const BRIDGE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.macro_bridge.constants"; +/// The call prefix of every reference function, in `SqlReferenceKind` order. +const GENERATED_REFERENCE_MARKERS: [&str; 6] = [ + "__ref(", + "__seed(", + "__source(", + "__dbt_ref(", + "__udf(", + "__table_fn(", +]; const DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; @@ -62,11 +70,7 @@ fn generated_classes(py: Python<'_>) -> PyResult> { &[("allowed", PyString::new(py, GENERATED_ALLOWED).into_any())], )?), )?; - let markers: Bound<'_, PyAny> = imported(py, BUILTINS_MODULE, "list")?.call1((imported( - py, - BRIDGE_CONSTANTS_MODULE, - "GENERATED_REFERENCE_MARKERS", - )?,))?; + let markers: Bound<'_, PyAny> = PyList::new(py, GENERATED_REFERENCE_MARKERS)?.into_any(); keywords( py, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index 3c352fc657..d535039dd1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -1,3 +1,18 @@ //! Authored SQL-test mode identifiers. pub(crate) const MODEL_TEST_MODE: &str = "model"; +/// The rule codes whose fixes `sqb format --fix` may apply after compiler verification. +pub(crate) const SAFE_RULE_FIX_CODES: [&str; 8] = [ + "SQBRSQL002", + "SQBRSQL003", + "SQBRSQL005", + "SQBRSQL006", + "SQBRSQL008", + "SQBRSQL017", + "SQBRSQL025", + "SQBRSQL042", +]; +/// The note attached to a rule finding that has an available fix. +pub(crate) const RULE_FIX_AVAILABLE_NOTE: &str = + "Fix available: `sqb format --fix` applies it after compiler verification"; +pub(crate) const RULES_NATIVE_API_VERSION: i64 = 1; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 975e1a0262..089cbdbf8f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -41,13 +41,7 @@ pub(crate) fn sql_lint_config<'py>( ) -> PyResult> { let py: Python<'py> = dialect.py(); let threshold = |key: &str, default: &str| -> PyResult> { - thresholds.call_method1( - "get", - ( - imported(py, LINT_CONSTANTS_MODULE, key)?, - imported(py, LINT_CONSTANTS_MODULE, default)?, - ), - ) + thresholds.call_method1("get", (key, imported(py, LINT_CONSTANTS_MODULE, default)?)) }; call_with( &imported(py, LINT_MODELS_MODULE, "LintConfig")?, @@ -57,14 +51,11 @@ pub(crate) fn sql_lint_config<'py>( ("header_rules_enabled", false.into_bound_py_any(py)?), ( "max_literal_length", - threshold("MAX_LITERAL_LENGTH_THRESHOLD", "DEFAULT_MAX_LITERAL_LENGTH")?, + threshold("max_literal_length", "DEFAULT_MAX_LITERAL_LENGTH")?, ), ( "max_ranking_order_by", - threshold( - "MAX_RANKING_ORDER_BY_THRESHOLD", - "DEFAULT_MAX_RANKING_ORDER_BY", - )?, + threshold("max_ranking_order_by", "DEFAULT_MAX_RANKING_ORDER_BY")?, ), ("relation_keys", relation_keys), ("allow_suppressions", allow_suppressions), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index d92218d242..4ccc3d3135 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -14,12 +14,12 @@ use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; const NATIVE: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild.lint.models"; -const CONSTANTS: &str = "sqlbuild.lint.constants"; const ADAPTER: &str = "sqlbuild.adapter.contract.classes.base_adapter"; const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; const LINT_ERRORS: &str = "sqlbuild.lint.exceptions"; @@ -235,14 +235,13 @@ impl<'py> FixSession<'py> { violation: &Bound<'py, PyAny>, model_paths: &Bound<'py, PyAny>, ) -> PyResult> { - let py = violation.py(); let code = violation.getattr("code")?; let path = violation.getattr("file_path")?; let has_fix = !violation.getattr("fix")?.is_none(); let facts = FixEligibility { model_body: model_paths.contains(&path)?, has_fix, - safe_code: imported(py, CONSTANTS, "SAFE_RULE_FIX_CODES")?.contains(&code)?, + safe_code: SAFE_RULE_FIX_CODES.contains(&code.extract::()?.as_str()), conditionless_join: code.eq("SQBRSQL003")?, snowflake: self.config.getattr("dialect")?.eq("snowflake")?, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs index fc819aaec2..289a085c0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/selection.rs @@ -9,8 +9,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::rules::main::selected_codes::selected_codes_json; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::rules::constants::RULES_NATIVE_API_VERSION; -const RULES_CONSTANTS_MODULE: &str = "sqlbuild.rule_engine.constants"; const RULES_EXCEPTIONS_MODULE: &str = "sqlbuild.rule_engine.exceptions"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; @@ -25,11 +25,7 @@ pub(crate) fn selected_rule_codes( ) -> PyResult> { let py: Python<'_> = config.py(); let request: Bound<'_, PyDict> = PyDict::new(py); - request.set_item( - "version", - py.import(RULES_CONSTANTS_MODULE)? - .getattr("RULES_NATIVE_API_VERSION")?, - )?; + request.set_item("version", RULES_NATIVE_API_VERSION)?; request.set_item( "config", py.import(DATACLASSES_MODULE)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index c8172a16bf..878902bc88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -12,11 +12,11 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule_cache_path}; const NATIVE_MODULE: &str = "sqlbuild._native"; const RULES_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; -const LINT_CONSTANTS_MODULE: &str = "sqlbuild.lint.constants"; const LINT_EXCEPTIONS_MODULE: &str = "sqlbuild.lint.exceptions"; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; @@ -568,7 +568,7 @@ fn violation_findings<'py>( let engine_error: String = imported(py, LINT_EXCEPTIONS_MODULE, "NativeLintError")? .getattr("code")? .extract()?; - let safe_fixes: Bound<'py, PyAny> = imported(py, LINT_CONSTANTS_MODULE, "SAFE_RULE_FIX_CODES")?; + let safe_fixes: Bound<'py, PyAny> = PyFrozenSet::new(py, SAFE_RULE_FIX_CODES)?.into_any(); let resolved: Bound<'py, PyAny> = run.project_dir.call_method0("resolve")?; let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; let no_paths: BTreeSet = BTreeSet::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 372315ecd9..50cae7b73c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -18,30 +18,24 @@ use crate::bindings::_helpers::scopes::scope_cache::{ use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; +use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; const NATIVE_MODULE: &str = "sqlbuild._native"; -const SCOPE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.scopes.constants"; +const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; +const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild._native"; const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const MACRO_INVENTORY_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.user_macros"; /// Where the scope index cache file of a project lives: `(directory, file name)`. fn cache_location(project_dir: &Bound<'_, PyAny>) -> PyResult<(PathBuf, String)> { - let py: Python<'_> = project_dir.py(); - let directory: Bound<'_, PyAny> = cache_directory(project_dir)?.div(imported( - py, - SCOPE_CONSTANTS_MODULE, - "SCOPE_CACHE_DIRECTORY_NAME", - )?)?; - Ok(( - directory.extract()?, - imported(py, SCOPE_CONSTANTS_MODULE, "SCOPE_CACHE_FILENAME")?.extract()?, - )) + let directory: Bound<'_, PyAny> = + cache_directory(project_dir)?.div(SCOPE_CACHE_DIRECTORY_NAME)?; + Ok((directory.extract()?, SCOPE_CACHE_FILENAME.to_owned())) } /// Whether `error` is a project input error that selects the tolerant index. @@ -158,11 +152,7 @@ fn tolerant_index<'py>( py, [ py.None().into_bound(py), - imported( - py, - COMPILE_CONSTANTS_MODULE, - "SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM", - )?, + pyo3::types::PyString::new(py, SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM).into_any(), ], )?; let relationship_faults: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index aa22330e58..9161c7c459 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -1,11 +1,12 @@ //! The Python models, enums and compile helpers the semantic completion stage reads and builds. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: &str = "P009"; +const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const TYPE_RECOVERY_MODULE: &str = "sqlbuild._native"; @@ -13,7 +14,6 @@ const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const NATIVE_MODULE: &str = "sqlbuild._native"; const NORMALIZE_TYPE_MODULE: &str = "sqlbuild.adapter.type_system.main.normalize_type"; const DATACLASSES_MODULE: &str = "dataclasses"; -const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; /// Imported once per stage run. @@ -60,9 +60,7 @@ impl<'py> StageClasses<'py> { model_resource: resource.getattr("MODEL")?, source_resource: resource.getattr("SOURCE")?, sql_test_resource: resource.getattr("SQL_TEST")?, - opt_out_code: py - .import(COMPILE_CONSTANTS_MODULE)? - .getattr("UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE")?, + opt_out_code: PyString::new(py, UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE).into_any(), replace: py.import(DATACLASSES_MODULE)?.getattr("replace")?, project_with: py.import(PROJECT_WITH_MODULE)?.getattr("project_with")?, with_type_recovery: py @@ -75,9 +73,7 @@ impl<'py> StageClasses<'py> { normalize_type: py .import(NORMALIZE_TYPE_MODULE)? .getattr("normalize_type")?, - sql_analysis_key: py - .import(DISCOVERY_CONSTANTS_MODULE)? - .getattr("SQL_ANALYSIS_CONFIG_KEY")?, + sql_analysis_key: PyString::new(py, SQL_ANALYSIS_CONFIG_KEY).into_any(), udf_kind: kinds.getattr("UDF")?, table_function_kind: kinds.getattr("TABLE_FUNCTION")?, }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index a95d1998cc..0bad28def1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -11,7 +11,6 @@ use crate::bindings::_helpers::sql_test_glue::assembly::{ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const GLUE_MODELS_MODULE: &str = "sqlbuild.compiler.sql_test_glue.models"; const SPEC_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; /// Fields a direct-logic compiled payload copies from its input payload, in order. @@ -74,9 +73,7 @@ impl<'py> Classes<'py> { assembly: glue.getattr("NativeSqlTestAssembly")?, failure: glue.getattr("NativeSqlTestFailure")?, location: py.import(SPEC_MODELS_MODULE)?.getattr("SourceLocation")?, - code: py - .import(COMPILE_CONSTANTS_MODULE)? - .getattr("SQL_TEST_HELPER_REFERENCE_CODE")?, + code: PyString::new(py, super::constants::SQL_TEST_HELPER_REFERENCE_CODE).into_any(), models, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index b78aa93e68..28afc8a867 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -24,3 +24,9 @@ pub(super) const DEFAULT_INTEGER_END: &str = "1000000000000000"; pub(super) const CURSOR_START_KEY: &str = "cursor_start"; pub(super) const CURSOR_END_KEY: &str = "cursor_end"; pub(super) const SENTINEL: &str = "__SQLBUILD_CALL_SUFFIX__"; +/// The diagnostic code of a SQL test that references a helper it may not read. +pub(crate) const SQL_TEST_HELPER_REFERENCE_CODE: &str = "P013"; +/// The scan store namespace of expanded SQL-test CTEs. +pub(crate) const SQL_TEST_CTE_SCAN_ALGORITHM: &str = "expanded-sql-test-ctes-v1"; +/// The scan store namespace of the models SQL tests expect. +pub(crate) const SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: &str = "sql-test-expected-models-v1"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index e1a4f4766c..17e0dd54a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,7 +1,7 @@ pub(crate) mod assembly; pub(crate) mod authored_ctes; mod compiled_tests; -mod constants; +pub(crate) mod constants; mod cursor_window; mod fixture_validation; mod plan_assembly; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs index dabf3065ca..bb8bf430a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_errors.rs @@ -75,14 +75,7 @@ pub(super) fn planning_error( kwargs.set_item("location", cte_call_location(py, &test, (&cte, &call))?)?; "SqlTestReferenceError" } else { - kwargs.set_item( - "code", - imported( - py, - "sqlbuild.compiler.compile.constants", - "SQL_TEST_HELPER_REFERENCE_CODE", - )?, - )?; + kwargs.set_item("code", super::constants::SQL_TEST_HELPER_REFERENCE_CODE)?; "CompileInputError" }; return Ok(PyErr::from_value( diff --git a/crates/sqlbuild-scopes/src/scope_index/constants.rs b/crates/sqlbuild-scopes/src/scope_index/constants.rs index e8bc8ac087..162456a17b 100644 --- a/crates/sqlbuild-scopes/src/scope_index/constants.rs +++ b/crates/sqlbuild-scopes/src/scope_index/constants.rs @@ -1,4 +1,4 @@ -//! Scope layout names, mirroring the Python scope builder and `sqlbuild.compiler.scopes.constants`. +//! Scope layout names. use crate::scope_index::models::ResourceKind; diff --git a/python-boundary.toml b/python-boundary.toml index d1055ee399..e1a29e157a 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 12699 +remaining_lines = 12470 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/comparing/compare.py b/scripts/compiler_differential/_helpers/comparing/compare.py index 6d88ca21aa..17996d1679 100644 --- a/scripts/compiler_differential/_helpers/comparing/compare.py +++ b/scripts/compiler_differential/_helpers/comparing/compare.py @@ -5,9 +5,12 @@ import json from typing import cast -from scripts.compiler_differential.constants import MISSING_VALUE, VALUE_PREVIEW_CHARACTERS +from scripts.compiler_differential.constants import ( + MISSING_VALUE, + STAGE_CAPTURE_SHARED_MARKER, + VALUE_PREVIEW_CHARACTERS, +) from scripts.compiler_differential.models import Divergence -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SHARED_MARKER def first_json_difference(*, left: object, right: object, pointer: str = "") -> Divergence | None: diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index fcdfecef9c..8fbbbcf835 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -18,6 +18,7 @@ from scripts.compiler_differential.constants import ( ANALYSIS_CODE_SCAN_ROOTS, ANALYSIS_NATIVE_CODE_SCAN_ROOTS, + BUILT_IN_AUDIT_SHADOW_CODE, COLD_COMPILE, CORPUS_FAILURES, DIAGNOSTIC_CODE_PATTERN, @@ -38,7 +39,6 @@ FailureCase, ProjectRun, ) -from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE from sqlbuild.compiler.compile import constants as compile_constants from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 3b20207b54..637a7900d4 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -22,6 +22,7 @@ ) from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( + BUILT_IN_AUDIT_NAMES, COMPILE_INPUTS_CAPTURE_TYPE, RENDER_CALLABLE_CAPTURE_FIELDS, RENDER_DETAIL_KINDS, @@ -31,7 +32,6 @@ RENDER_STAGE_CAPTURE_SUFFIX, ) from sqlbuild._native import STAGE_CAPTURE_OMITTED_ATTRIBUTES -from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_NAMES from sqlbuild.compiler.compile.models import CompileProjectInputs _MACRO_CALL: re.Pattern[str] = re.compile(r"@([A-Za-z_]\w*)\(") diff --git a/scripts/compiler_differential/classes/capture_file.py b/scripts/compiler_differential/classes/capture_file.py index bc73aaa489..3017d571fd 100644 --- a/scripts/compiler_differential/classes/capture_file.py +++ b/scripts/compiler_differential/classes/capture_file.py @@ -7,7 +7,7 @@ from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest from scripts.compiler_differential._helpers.comparing.normalize import normalize_artifact_text -from sqlbuild.compiler.frontier.constants import ( +from scripts.compiler_differential.constants import ( STAGE_CAPTURE_ROOT_KEY, STAGE_CAPTURE_SHARED_NODES_KEY, ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index d05df9c59f..06cd15e799 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -693,3 +693,10 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ), "B219": _NEVER_REPORTED_BY_POLYGLOT, } +BUILT_IN_AUDIT_NAMES: frozenset[str] = frozenset( + {"accepted_values", "not_null", "relationships", "unique"} +) +BUILT_IN_AUDIT_SHADOW_CODE: str = "P003" +STAGE_CAPTURE_ROOT_KEY: str = "__capture__" +STAGE_CAPTURE_SHARED_MARKER: str = "__shared__" +STAGE_CAPTURE_SHARED_NODES_KEY: str = "__shared_nodes__" diff --git a/src/sqlbuild/compiler/analysis_session/constants.py b/src/sqlbuild/compiler/analysis_session/constants.py deleted file mode 100644 index cc6670f3f8..0000000000 --- a/src/sqlbuild/compiler/analysis_session/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -"""Native model analysis session constants.""" - -LINEAGE_NATIVE: str = "native" -NATIVE_ANALYSIS_STORE_FILE_NAME: str = "model-analyses.bin" diff --git a/src/sqlbuild/compiler/auditing/constants.py b/src/sqlbuild/compiler/auditing/constants.py deleted file mode 100644 index f61adcee46..0000000000 --- a/src/sqlbuild/compiler/auditing/constants.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Stable constants for audit rendering.""" - -BUILT_IN_AUDIT_NAMES: frozenset[str] = frozenset( - {"accepted_values", "not_null", "relationships", "unique"} -) -BUILT_IN_AUDIT_SHADOW_CODE: str = "P003" diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 5f7b168b20..7ab946ed48 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -3,66 +3,11 @@ from __future__ import annotations from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.python_nodes.types import SqlResourceRefKind -UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" -SQL_TEST_HELPER_REFERENCE_CODE: str = "P013" -DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { - "model_files": "model", - "scenario_files": "scenario", - "seed_files": "seed", - "source_files": "source", - "sql_function_files": "function", - "python_function_files": "function", - "sql_hook_files": "hook", - "hook_functions": "hook", - "loader_functions": "loader", - "task_functions": "task", - "asset_functions": "asset", - "check_functions": "check", - "providers": "provider", -} -DESCRIPTION_EXEMPT_INPUTS: dict[str, str] = { - "project_config": "project configuration, not a named resource", - "local_config": "local configuration, not a named resource", - "project_dir": "project location, not a named resource", - "adapter_file": "project adapter override, not a named resource", - "enum_files": "typed value declarations used inside SQL, not graph resources", - "constant_files": "typed value declarations used inside SQL, not graph resources", - "macro_files": "SQL helpers expanded inline, not graph resources", - "model_schema_files": "shared column schemas; models that use one are checked", - "schema_files": "seed YAML declarations; each declared seed is checked", - "test_files": "SQL unit tests are named by their subject and expected behaviour", - "audit_files": "audits are assertions named by what they check", - "audit_factories": "generate audits, which are assertions", - "materialization_files": "materialization strategies, not graph resources", - "event_exporters": "runtime event outputs, not graph resources", - "command_output_sinks": "runtime command outputs, not graph resources", - "native_session": "native parse state kept for later stages, not a project input", -} MIGRATE_FROM_CONFIG_KEY: str = "migrate_from" MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" -EXPECTED_TEST_CTE_PREFIX: str = "__expected__" -ASSERT_TEST_CTE_PREFIX: str = "__assert__" -REF_TEST_CTE_PREFIX: str = SqlReferenceKind.REF.fixture_cte_prefix -SOURCE_TEST_CTE_PREFIX: str = SqlReferenceKind.SOURCE.fixture_cte_prefix -SEED_TEST_CTE_PREFIX: str = SqlReferenceKind.SEED.fixture_cte_prefix -DBT_REF_TEST_CTE_PREFIX: str = SqlReferenceKind.DBT_REF.fixture_cte_prefix -TABLE_FN_TEST_CTE_PREFIX: str = SqlReferenceKind.TABLE_FUNCTION.fixture_cte_prefix DEFAULT_SQL_TEST_MODE: SqlTestMode = SqlTestMode.MODEL -COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" -COMPILE_CACHE_DISABLE_VALUE: str = "1" -RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" -RETIRED_REFERENCE_CACHE_DIRECTORY_NAME: str = "references-v3" -SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" -SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: str = "sql-test-expected-models-v1" -EXPLICIT_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = { - SqlReferenceKind.REF.value: SqlResourceRefKind.MODEL, - SqlReferenceKind.SOURCE.value: SqlResourceRefKind.SOURCE, - SqlReferenceKind.SEED.value: SqlResourceRefKind.SEED, -} diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 2f1f7cef23..4eca4a8a6c 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -3,123 +3,14 @@ from __future__ import annotations PROJECT_CONFIG_FILENAME: str = "sqlbuild_project.toml" -LOCAL_CONFIG_FILENAME: str = "sqlbuild_local.toml" -YAML_FILE_SUFFIXES: frozenset[str] = frozenset({".yml", ".yaml"}) PYTHON_INIT_MODULE_STEM: str = "__init__" PYTHON_NODE_ROOT: str = "python" -SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( - {"arguments", "returns", "database", "schema", "tags", "description"} -) -SQL_AUDIT_HEADER_KEYS: frozenset[str] = frozenset( - { - "name", - "severity", - "run_scope", - "always_run", - "evaluation", - "value", - "sample_count", - "sample_unit", - "thresholds", - "minimum_samples", - "sql_analysis", - } -) -SQL_HOOK_HEADER_KEYS: frozenset[str] = frozenset({"description"}) -SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( - { - "alias", - "append_cursor_inclusive", - "audit_factories", - "audits", - "batch_concurrency", - "batch_size", - "check_columns", - "columns", - "config", - "constants", - "contract", - "cursor", - "cursor_end", - "cursor_future_action", - "cursor_future_max_distance", - "cursor_grain", - "cursor_inputs", - "cursor_start", - "cursor_start_max_action", - "cursor_start_max_ahead", - "cursor_type", - "cursor_watermark_mode", - "database", - "description", - "dynamic_columns", - "enabled", - "enums", - "full_refresh", - "historical_input", - "incremental_mode", - "incremental_strategy", - "initial_valid_from", - "invalidate_hard_deletes", - "lookback", - "materialized", - "max_microbatches", - "merge_exclude_columns", - "microbatch_limit", - "microbatch_strategy", - "migrate_force", - "migrate_from", - "model_schema", - "observed_at", - "old_name_view", - "on_schema_change", - "placeholders", - "post_hooks", - "pre_hooks", - "replay_on_change", - "row_diff_exclude_columns", - "row_diff_sample_rows", - "row_diff_sample_seed", - "row_diff_tolerances", - "schema", - "snapshot_full_refresh", - "snapshot_schema_change", - "snapshot_strategy", - "sql_analysis", - "sql_validation", - "table_type", - "tags", - "time_travel_retention", - "unaccounted_partition_policy", - "unique_key", - "updated_at", - "valid_from_column", - "valid_to_column", - } -) -REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) -SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" -SQL_TEST_HEADER_KEYS: frozenset[str] = frozenset( - {"name", "mode", "parameters", "cases", "cursor_start", "cursor_end", SQL_ANALYSIS_CONFIG_KEY} -) -SQL_SCENARIO_HEADER_KEYS: frozenset[str] = frozenset({"description", "tags"}) -SQL_ANALYSIS_SETTING_KEY: str = "sql_analysis" SETTINGS_SECTION: str = "settings" TABLE_PROMOTION_MODE_SETTING_KEY: str = "table_promotion_mode" STAGED_PROMOTION_MODE: str = "staged" SQL_TESTS_OWNERSHIP_ROOT: str = "tests/unit" SQL_SCENARIOS_OWNERSHIP_ROOT: str = "tests/scenarios" - - -SQL_HOOK_OUTPUT_FIELDS: tuple[str, ...] = ( - "statement", - "name", - "relative_path", - "definition_sql", - "kwargs", - "description", -) diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index cd8407ff47..02232e4b6c 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -4,19 +4,6 @@ import itertools -NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" -RETIRED_CACHE_NAMESPACE_SUFFIXES: tuple[str, ...] = ( - "", - NATIVE_CACHE_NAMESPACE_SUFFIX, - "-native-preview-v1", -) -TARGET_DIRECTORY_NAME: str = "target" -CACHE_DIRECTORY_NAME: str = "cache" -COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" COMPILE_REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" -STAGE_CAPTURE_SUFFIX: str = ".json" -STAGE_CAPTURE_SHARED_MARKER: str = "__shared__" -STAGE_CAPTURE_ROOT_KEY: str = "__capture__" -STAGE_CAPTURE_SHARED_NODES_KEY: str = "__shared_nodes__" STAGE_CAPTURE_SEQUENCE: itertools.count[int] = itertools.count(1) diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 6016efad5f..f70fcd3439 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -9,11 +9,7 @@ COMPILE_REUSE_DISABLE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR, ) -from sqlbuild.compiler.references.types import SqlReferenceKind -GENERATED_REFERENCE_MARKERS: tuple[str, ...] = tuple( - f"{kind.function_name}(" for kind in SqlReferenceKind -) ACTIVE_MACRO_BRIDGE: ContextVar[object | None] = ContextVar( "sqlbuild_active_macro_bridge", default=None ) diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py deleted file mode 100644 index 922ac5749e..0000000000 --- a/src/sqlbuild/compiler/scopes/constants.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Constants for the compiler-owned scope domain.""" - -from __future__ import annotations - -SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" -SCOPE_CACHE_FILENAME: str = "scope-index.json" -CURRENT_PATH_COMPONENT: str = "." -DECLARATION_LEXICAL_STEM: str = "__declaration__.sql" diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index a0dcf02871..c7e0b21087 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -15,5 +15,3 @@ POLYGLOT_COMPLEXITY_GUARD_OPTION: str = "complexityGuard" POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION: str = "maxFunctionCallDepth" POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 - -TYPE_CHECKED_DIALECTS: frozenset[str] = frozenset({"duckdb", "snowflake", "postgresql", "bigquery"}) diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 60e87c3258..f9c86a5037 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -4,24 +4,8 @@ from sqlbuild.lint.types import LintSeverity -LINT_ENGINE_SQLBUILD: str = "sqlbuild" - VIOLATION_SEVERITY_FAULT: LintSeverity = LintSeverity.FAULT VIOLATION_SEVERITY_WARNING: LintSeverity = LintSeverity.WARNING DEFAULT_MAX_LITERAL_LENGTH: int = 100 DEFAULT_MAX_RANKING_ORDER_BY: int = 5 -MAX_LITERAL_LENGTH_THRESHOLD: str = "max_literal_length" -MAX_RANKING_ORDER_BY_THRESHOLD: str = "max_ranking_order_by" -SAFE_RULE_FIX_CODES: frozenset[str] = frozenset( - { - "SQBRSQL002", - "SQBRSQL003", - "SQBRSQL005", - "SQBRSQL006", - "SQBRSQL008", - "SQBRSQL017", - "SQBRSQL025", - "SQBRSQL042", - } -) diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index f599d97893..c448dbf8f3 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -1,12 +1,5 @@ """Rules engine constants.""" -RULES_SKILL_PATHS: tuple[str, ...] = ( - ".agents/skills/sqlbuild-rules/SKILL.md", - ".claude/skills/sqlbuild-rules/SKILL.md", - ".opencode/skills/sqlbuild-rules/SKILL.md", -) - -RULES_NATIVE_API_VERSION: int = 1 CUSTOM_HOST_PROTOCOL_VERSION: int = 1 CUSTOM_HOST_RUNTIME_VERSION: str = "sqlbuild-rules-custom-v2" CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 @@ -72,15 +65,11 @@ TYPE_PROOF_RULE_CODES: frozenset[str] = frozenset({"SQBRCONTRACT105"}) -TARGET_DIRECTORY_NAME: str = "target" RULE_DECORATOR_TOKEN: str = "@rule" INIT_MODULE_NAME: str = "__init__" PARENT_DIRECTORY_TOKEN: str = ".." MIN_CUSTOM_RULE_TEST_CASES: str = "min_custom_rule_test_cases" -RULE_FIX_AVAILABLE_NOTE: str = ( - "Fix available: `sqb format --fix` applies it after compiler verification" -) FACT_PROJECT_MODELS: str = "project.models" FACT_PROJECT_SOURCES: str = "project.sources" FACT_PROJECT_SEEDS: str = "project.seeds" diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index fa9fd13c6e..d20a1ff03e 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -6,4 +6,3 @@ DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() ZERO_DAY_CURSOR_DURATION: str = "0d" -DEFAULT_DIFF_MAX_FULL_ROWS: int = 10_000_000 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 15334996b8..0b756cad53 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -45,20 +45,11 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.compiler.analysis_session.constants import NATIVE_ANALYSIS_STORE_FILE_NAME -from sqlbuild.compiler.compile.constants import ( - RETIRED_FACT_CACHE_DIRECTORY_NAME, - RETIRED_REFERENCE_CACHE_DIRECTORY_NAME, -) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.frontier.constants import ( COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR, ) -from sqlbuild.compiler.frontier.constants import ( - COMPILER_CACHE_DIRECTORY_NAME, - RETIRED_CACHE_NAMESPACE_SUFFIXES, -) from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -77,6 +68,12 @@ contract_statuses, ) +NATIVE_ANALYSIS_STORE_FILE_NAME: str = "model-analyses.bin" +RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" +RETIRED_REFERENCE_CACHE_DIRECTORY_NAME: str = "references-v3" +COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" +RETIRED_CACHE_NAMESPACE_SUFFIXES: tuple[str, ...] = ("", "-native-v1", "-native-preview-v1") + NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" NATIVE_REUSE_SUFFIX: str = ".store" RETIRED_REUSE_DIRECTORY_NAME: str = "project-reuse-v1" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index 211d9c61e4..eab3e5a839 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -11,7 +11,6 @@ import pytest -from sqlbuild.compiler.compile.constants import DESCRIPTION_REQUIRED_INPUT_KINDS from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FormatDescriptionResolutionE2ECase, PathDefaultDescriptionCase, @@ -29,6 +28,22 @@ ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { + "model_files": "model", + "scenario_files": "scenario", + "seed_files": "seed", + "source_files": "source", + "sql_function_files": "function", + "python_function_files": "function", + "sql_hook_files": "hook", + "hook_functions": "hook", + "loader_functions": "loader", + "task_functions": "task", + "asset_functions": "asset", + "check_functions": "check", + "providers": "provider", +} + _SEED_CSV: str = "product_type_id,label\n1,waffle\n" _SEED_COLUMNS: str = ( " columns:\n - name: product_type_id\n type: INTEGER\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py index 064500b673..ac589f5907 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/lineage/test_lineage.py @@ -7,7 +7,6 @@ import pytest -from sqlbuild.compiler.frontier.constants import NATIVE_CACHE_NAMESPACE_SUFFIX from tests.e2e.src.sqlbuild.cli.commands.main.lineage._test_types import ( ColumnLineageCacheCliTestCase, DiamondLineageTreeCliTestCase, @@ -30,6 +29,8 @@ run_sqb, ) +NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" + _TREE_LINE: re.Pattern[str] = re.compile(r"^[│ ]*[├└]── .*$", re.MULTILINE) _EXPANDED_DIAMOND_NODE: re.Pattern[str] = re.compile( r"^[│ ]*[├└]── (?![^\n]*\(already shown\))[^\n]*?(orders_(?:left_|right_)?\d+(?:\.amount)?)", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index 3bcbe1fb76..3d11199743 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -9,9 +9,11 @@ from time import perf_counter from sqlbuild._native import compiler_cache_directory -from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import REPO_ROOT +SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" +SCOPE_CACHE_FILENAME: str = "scope-index.json" + def run_scope_alias( *, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index ab7a1ba08f..49860e9287 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -26,7 +26,6 @@ from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.models import NativeQueryColumns -from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact @@ -34,6 +33,8 @@ RepeatedJsonParseTestCase, ) +STAGE_CAPTURE_SUFFIX: str = ".json" + def staging_directories(target_dir: Path) -> list[str]: """Return hidden entries left in target_dir, such as abandoned artifact staging.""" diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 9ad5adf381..a33213c95a 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -30,7 +30,6 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main -from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, @@ -59,6 +58,8 @@ ) from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry +LINEAGE_NATIVE: str = "native" + _PROJECT_TOML: str = ( 'name = "orders_analysis"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' ) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 1c873934ed..fde1966b25 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -32,11 +32,6 @@ discover_sql_hook_files, render_stage_capture, ) -from sqlbuild.compiler.discovery.constants import ( - SQL_AUDIT_HEADER_KEYS, - SQL_FUNCTION_HEADER_KEYS, - SQL_HOOK_HEADER_KEYS, -) from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSourceFile, @@ -53,6 +48,26 @@ ) from tests.integration.src.sqlbuild.compiler.helpers import random_float, random_text +SQL_AUDIT_HEADER_KEYS: frozenset[str] = frozenset( + { + "always_run", + "evaluation", + "minimum_samples", + "name", + "run_scope", + "sample_count", + "sample_unit", + "severity", + "sql_analysis", + "thresholds", + "value", + } +) +SQL_FUNCTION_HEADER_KEYS: frozenset[str] = frozenset( + {"arguments", "database", "description", "returns", "schema", "tags"} +) +SQL_HOOK_HEADER_KEYS: frozenset[str] = frozenset({"description"}) + REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[6] EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".venv", "node_modules", "target", ".git"}) PYTHON_ERROR: dict[str, str] = {"error": "rejected"} diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 04f67da526..577cc71dea 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -4,11 +4,6 @@ from pathlib import Path -from sqlbuild.compiler.compile.constants import ( - EXPECTED_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, -) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledModelSqlTestPayload, @@ -36,6 +31,10 @@ reference_names_by_kind, ) +EXPECTED_TEST_CTE_PREFIX: str = "__expected__" +REF_TEST_CTE_PREFIX: str = "__ref__" +SOURCE_TEST_CTE_PREFIX: str = "__source__" + _STUB_TEST_FILE: DiscoveredSqlTestFile = DiscoveredSqlTestFile( file_path=Path("tests/unit/test_chain.sql"), relative_path=Path("tests/unit/test_chain.sql"), diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 0a0558e330..3c32f6e416 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -35,7 +35,6 @@ DiscoveredProjectInputs, EnumDeclaration, ) -from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, DECLARATION_LEXICAL_STEM from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, @@ -54,6 +53,9 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry +CURRENT_PATH_COMPONENT: str = "." +DECLARATION_LEXICAL_STEM: str = "__declaration__.sql" + PROJECT_CONFIG: str = ( 'name = "scope_parity"\nadapter = "duckdb"\n\n[connection]\ndatabase = "scope.duckdb"\n' ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py index ac13c897b2..af860d16b1 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -6,15 +6,45 @@ import pytest -from sqlbuild.compiler.compile.constants import ( - DESCRIPTION_EXEMPT_INPUTS, - DESCRIPTION_REQUIRED_INPUT_KINDS, -) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DescriptionInputClassificationCase, ) +DESCRIPTION_EXEMPT_INPUTS: dict[str, str] = { + "project_config": "project configuration, not a named resource", + "local_config": "local configuration, not a named resource", + "project_dir": "project location, not a named resource", + "adapter_file": "project adapter override, not a named resource", + "enum_files": "typed value declarations used inside SQL, not graph resources", + "constant_files": "typed value declarations used inside SQL, not graph resources", + "macro_files": "SQL helpers expanded inline, not graph resources", + "model_schema_files": "shared column schemas; models that use one are checked", + "schema_files": "seed YAML declarations; each declared seed is checked", + "test_files": "SQL unit tests are named by their subject and expected behaviour", + "audit_files": "audits are assertions named by what they check", + "audit_factories": "generate audits, which are assertions", + "materialization_files": "materialization strategies, not graph resources", + "event_exporters": "runtime event outputs, not graph resources", + "command_output_sinks": "runtime command outputs, not graph resources", + "native_session": "native parse state kept for later stages, not a project input", +} +DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { + "model_files": "model", + "scenario_files": "scenario", + "seed_files": "seed", + "source_files": "source", + "sql_function_files": "function", + "python_function_files": "function", + "sql_hook_files": "hook", + "hook_functions": "hook", + "loader_functions": "loader", + "task_functions": "task", + "asset_functions": "asset", + "check_functions": "check", + "providers": "provider", +} + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index 19f4aba554..fb6510bd32 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -1,10 +1,6 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.discovery.constants import ( - REMOVED_SQL_MODEL_HEADER_KEYS, - SQL_MODEL_HEADER_KEYS, -) from sqlbuild.compiler.discovery.models import ( DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, @@ -17,6 +13,79 @@ SourceLocation, ) +REMOVED_SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) +SQL_MODEL_HEADER_KEYS: frozenset[str] = frozenset( + { + "alias", + "append_cursor_inclusive", + "audit_factories", + "audits", + "batch_concurrency", + "batch_size", + "check_columns", + "columns", + "config", + "constants", + "contract", + "cursor", + "cursor_end", + "cursor_future_action", + "cursor_future_max_distance", + "cursor_grain", + "cursor_inputs", + "cursor_start", + "cursor_start_max_action", + "cursor_start_max_ahead", + "cursor_type", + "cursor_watermark_mode", + "database", + "description", + "dynamic_columns", + "enabled", + "enums", + "full_refresh", + "historical_input", + "incremental_mode", + "incremental_strategy", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "max_microbatches", + "merge_exclude_columns", + "microbatch_limit", + "microbatch_strategy", + "migrate_force", + "migrate_from", + "model_schema", + "observed_at", + "old_name_view", + "on_schema_change", + "placeholders", + "post_hooks", + "pre_hooks", + "replay_on_change", + "row_diff_exclude_columns", + "row_diff_sample_rows", + "row_diff_sample_seed", + "row_diff_tolerances", + "schema", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "sql_analysis", + "sql_validation", + "table_type", + "tags", + "time_travel_retention", + "unaccounted_partition_policy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", + } +) + def write_lifecycle_event_sink_project(*, project_dir: Path, exporter_config: str) -> None: (project_dir / "sinks").mkdir() diff --git a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py index a5ba58ee08..853a33d646 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/main/test_serialize_hook_entries.py @@ -5,12 +5,20 @@ import pytest from sqlbuild._native import serialize_hook_entries -from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import ( SerializeHookEntriesTestCase, ) +SQL_HOOK_OUTPUT_FIELDS: tuple[str, ...] = ( + "statement", + "name", + "relative_path", + "definition_sql", + "kwargs", + "description", +) + _NAMED_SQL_HOOK: SqlHookEntry = SqlHookEntry( statement="INSERT INTO order_audit SELECT 'orders'", name="record_order_access", diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index aa456abf66..9a50df5ea6 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -10,10 +10,7 @@ from scripts.compiler_differential.classes.capture_file import expand_capture_text from sqlbuild._native import compile_frontier, render_stage_capture -from sqlbuild.compiler.frontier.constants import ( - STAGE_CAPTURE_DIR_ENV_VAR, - STAGE_CAPTURE_SHARED_NODES_KEY, -) +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerStage from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( FrontierCaptureTestCase, @@ -34,6 +31,8 @@ repeated_orders, ) +STAGE_CAPTURE_SHARED_NODES_KEY: str = "__shared_nodes__" + _COMPILE_MODELS: str = "sqlbuild.compiler.compile.models" _COMPILE_TYPES: str = "sqlbuild.compiler.compile.types" _LINEAGE_TYPES: str = "sqlbuild.compiler.lineage.types" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index c17745d1c6..53c4009afe 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -18,15 +18,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile.constants import ( - ASSERT_TEST_CTE_PREFIX, - DBT_REF_TEST_CTE_PREFIX, - EXPECTED_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - TABLE_FN_TEST_CTE_PREFIX, -) from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledModel, @@ -64,6 +55,14 @@ PlanTestChainTestCase, ) +ASSERT_TEST_CTE_PREFIX: str = "__assert__" +DBT_REF_TEST_CTE_PREFIX: str = "__dbt_ref__" +EXPECTED_TEST_CTE_PREFIX: str = "__expected__" +REF_TEST_CTE_PREFIX: str = "__ref__" +SEED_TEST_CTE_PREFIX: str = "__seed__" +SOURCE_TEST_CTE_PREFIX: str = "__source__" +TABLE_FN_TEST_CTE_PREFIX: str = "__table_fn__" + _STUB_TEST_FILE: DiscoveredSqlTestFile = DiscoveredSqlTestFile( file_path=Path("tests/unit/test_chain.sql"), relative_path=Path("tests/unit/test_chain.sql"), diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index 64ec4f7b5f..dcf794f89a 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -14,7 +14,6 @@ load_or_build_scope_index, scope_index_fingerprint, ) -from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME from sqlbuild.compiler.scopes.models import DeclarationIdentity, ScopeIndex from sqlbuild.compiler.scopes.types import ( DeclarationKind, @@ -42,6 +41,9 @@ write_wrong_version_cache, ) +SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" +SCOPE_CACHE_FILENAME: str = "scope-index.json" + _PROJECT: str = SCOPE_CACHE_PROJECT _MODEL: str = SCOPE_CACHE_MODEL diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index 4b08a93296..c741a3e7d7 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -7,7 +7,7 @@ import pytest from sqlbuild._native import apply_lint_suppressions, run_lint -from sqlbuild.lint.constants import LINT_ENGINE_SQLBUILD, VIOLATION_SEVERITY_FAULT +from sqlbuild.lint.constants import VIOLATION_SEVERITY_FAULT from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from tests.unit.src.sqlbuild.lint._test_types import ( FixtureCeremonyFormatTestCase, @@ -23,6 +23,8 @@ violation_reports, ) +LINT_ENGINE_SQLBUILD: str = "sqlbuild" + CLEAN_MODEL: str = 'MODEL (\n materialized table,\n description "ok"\n);\nSELECT 1 AS x FROM t\n' NO_DESCRIPTION_MODEL: str = "MODEL (\n materialized table\n);\nSELECT 1 AS x FROM t\n" PROJECT_TOML: str = 'name = "demo"\nadapter = "duckdb"\n' diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 39123d7c0d..3c79ba9238 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -29,7 +29,7 @@ from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import custom_dispatch as native from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES, RULES_SKILL_PATHS +from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( CustomRulesOutcome, @@ -45,6 +45,12 @@ from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project +RULES_SKILL_PATHS: tuple[str, ...] = ( + ".agents/skills/sqlbuild-rules/SKILL.md", + ".claude/skills/sqlbuild-rules/SKILL.md", + ".opencode/skills/sqlbuild-rules/SKILL.md", +) + _MODEL_SQL: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final" diff --git a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py index c6df6b64a3..5a7e853bba 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/skills/test_skills.py @@ -6,7 +6,6 @@ import pytest from sqlbuild._native import install_rules_skills as install_skills -from sqlbuild.rule_engine.constants import RULES_SKILL_PATHS from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.models import RuleExemption, RulesConfig from tests.unit.src.sqlbuild.rule_engine.main.skills._test_types import ( @@ -17,6 +16,12 @@ ) from tests.unit.src.sqlbuild.rule_engine.main.skills.helpers import write_custom_rule +RULES_SKILL_PATHS: tuple[str, ...] = ( + ".agents/skills/sqlbuild-rules/SKILL.md", + ".claude/skills/sqlbuild-rules/SKILL.md", + ".opencode/skills/sqlbuild-rules/SKILL.md", +) + @pytest.mark.parametrize( "test_case", From ff4a7736f4ba46431cf11e5e67e5636324197089 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:50:30 +0100 Subject: [PATCH 03/23] refactor: move native row aliases and protocols into the extension stub --- python-boundary.toml | 2 +- .../_helpers/diagnostic_measurement.py | 12 +- src/sqlbuild/_native.pyi | 123 ++++++++++++++++-- .../compiler/analysis_session/models.py | 5 +- .../compiler/analysis_session/types.py | 13 -- src/sqlbuild/compiler/compile/models.py | 119 ----------------- src/sqlbuild/compiler/compile/types.py | 26 ++-- src/sqlbuild/compiler/fingerprints/types.py | 9 -- .../compiler/project_assembly/models.py | 25 ++-- .../compiler/project_assembly/types.py | 6 - src/sqlbuild/compiler/scopes/types.py | 9 -- src/sqlbuild/compiler/sql_analysis/types.py | 55 -------- src/sqlbuild/compiler/sql_test_glue/models.py | 4 +- src/sqlbuild/compiler/sql_test_glue/types.py | 27 ---- src/sqlbuild/lint/types.py | 4 - .../compiler/analysis_session/_test_types.py | 5 +- .../compiler/analysis_session/helpers.py | 6 +- .../compiler/semantic_checks/helpers.py | 6 +- .../classes/test_binding_catalog.py | 7 +- .../compiler/sql_analysis/main/_test_types.py | 4 +- .../compiler/sql_analysis/main/helpers.py | 26 ++-- .../src/sqlbuild/lint/_helpers/helpers.py | 6 +- 22 files changed, 191 insertions(+), 308 deletions(-) delete mode 100644 src/sqlbuild/compiler/analysis_session/types.py delete mode 100644 src/sqlbuild/compiler/fingerprints/types.py delete mode 100644 src/sqlbuild/compiler/project_assembly/types.py delete mode 100644 src/sqlbuild/compiler/sql_analysis/types.py delete mode 100644 src/sqlbuild/compiler/sql_test_glue/types.py diff --git a/python-boundary.toml b/python-boundary.toml index e1a29e157a..3ce9a218d6 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 12470 +remaining_lines = 12232 [[stays]] path = "executor" diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py index 3c07d85fe7..8ca95c915d 100644 --- a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py +++ b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py @@ -1,16 +1,20 @@ """Fresh-process and location-only measurements for diagnostic workloads.""" +from __future__ import annotations + import json import subprocess import sys from pathlib import Path from time import perf_counter -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import sqlbuild._native as _native from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic -from sqlbuild.compiler.sql_analysis.types import NativeDiagnosticRow, NativePositionsModule + +if TYPE_CHECKING: + from sqlbuild._native import NativeDiagnosticRow, NativePositionsModule def measure_diagnostic_compile( @@ -59,7 +63,7 @@ def measure_position_mapping(*, count: int, lines: int = 4000) -> float: diagnostic: SqlBindingDiagnostic = SqlBindingDiagnostic( code="B217", message="Incompatible comparison", start=offset, end=offset + len("amount > 5") ) - mapper: Any = cast(NativePositionsModule, _native).BindingPositions( + mapper: Any = cast("NativePositionsModule", _native).BindingPositions( { "authored": original, "query": original, @@ -94,7 +98,7 @@ def measure_function_name_scan(*, calls: int) -> float: ) + "\nFROM products" ) - native: NativePositionsModule = cast(NativePositionsModule, _native) + native: NativePositionsModule = cast("NativePositionsModule", _native) started: float = perf_counter() rows: list[NativeDiagnosticRow] = native.binding_diagnostics( sql=sql, dialect="snowflake", rows=[] diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2bc52aa41f..f01be7f71a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -9,7 +9,7 @@ from datetime import date, datetime from decimal import Decimal from pathlib import Path, PurePath from threading import Event -from typing import Any, Literal, TextIO, TypedDict +from typing import Any, Literal, Protocol, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -82,7 +82,6 @@ from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, CompileAdapterContext, CompileAnalysisSelection, - CompileAuditInput, CompiledAudit, CompiledFunction, CompiledLineageColumnFact, @@ -98,9 +97,6 @@ from sqlbuild.compiler.compile.models import ( CompiledSqlTest, CompileModelConfig, CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, - CompileSqlFunctionInput, CompileSqlReference, CompileSqlScenarioInput, CompileSqlTestCte, @@ -177,7 +173,6 @@ from sqlbuild.compiler.discovery.models import ( TolerantScopeDiscovery, ) from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet -from sqlbuild.compiler.fingerprints.types import FingerprintWriteProgress from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge, LineageGraphViews from sqlbuild.compiler.lineage.models import ( @@ -343,7 +338,9 @@ from sqlbuild.compiler.scopes.models import ( UsageRecord, VisibilityResolution, ) -from sqlbuild.compiler.scopes.types import DeclarationKind, JsonValue, ScopeQueryTarget +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, +) from sqlbuild.compiler.source_freshness.models import ( AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, @@ -363,13 +360,11 @@ from sqlbuild.compiler.sql_analysis.models import ( SqlLexicalSyntax, SqlSchemaValidationRequest, ) -from sqlbuild.compiler.sql_analysis.types import NativeProjectCatalog from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssembly, NativeSqlTestChainRequest, NativeSqlTestPlanningRequest, ) -from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow from sqlbuild.cost.models import ( CostResourceContext, CostRunRecord, @@ -442,7 +437,9 @@ from sqlbuild.lint.models import ( PreparedLintFiles, RuleFixResult, ) -from sqlbuild.lint.types import RelationKeyRow, RelationKeys +from sqlbuild.lint.types import ( + RelationKeyRow, +) from sqlbuild.presentation.models import DisplayOptions from sqlbuild.presentation.types import CompletionState, TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer @@ -603,6 +600,104 @@ def evaluate_rules_request(request: NativeRulesRequest) -> NativeRulesEvaluation def emit_orjson_report(report: object) -> str: ... def emit_json_report(report: object) -> str: ... +type ColumnRow = tuple[str, str | None, str] + +type DiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] + +type ProofRow = tuple[ + bool, list[ColumnRow], list[tuple[str, str | None]], list[str], str | None, bool +] + +type QueryColumnsMode = Literal["batch", "reanalysis", "parse", "legacy"] + +type CtePairRow = tuple[str, str] + +type NativeChainStepRow = tuple[ + str, + str, + str | None, + list[CtePairRow], + str | None, + list[str] | None, + list[CtePairRow], +] + +type NativeAssertionStepRow = tuple[str, str, list[CtePairRow], str | None] + +type NativePlanWarningRow = tuple[str | None, str, str] + +type NativeSqlTestPlanRow = tuple[ + str | None, + list[NativeChainStepRow], + list[NativeAssertionStepRow], + list[str], + list[NativePlanWarningRow], + list[str], +] + +type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] + +class FingerprintWriteProgress(Protocol): + def __call__(self, *, completed: int, total: int) -> None: ... + +type JsonValue = None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue] + +type ScopeQueryTarget = str | PurePath | ResourceIdentity | DeclarationIdentity + +type NativeDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] + +type NativeNormalizationRequest = tuple[str, dict[str, str], dict[str, str]] + +class NativeCompactAnalysisJob(Protocol): + def run(self) -> bytes: ... + +class NativeProjectCatalog(Protocol): + def inferred_schema( + self, *, sql: str, columns: Mapping[str, str], inputs: Mapping[str, Mapping[str, str]] + ) -> list[tuple[str, str]]: ... + def with_relations( + self, relations: Mapping[str, Mapping[str, str]] + ) -> NativeProjectCatalog: ... + def validation_payload( + self, requests: list[tuple[str, list[tuple[str, bool]], dict[str, Mapping[str, str]]]] + ) -> str: ... + def update_analysis( + self, relations: Mapping[str, tuple[Mapping[str, str], Mapping[str, str]]] + ) -> None: ... + def register_override(self, relations: Mapping[str, Mapping[str, str]]) -> int: ... + def prepare_compact(self, payload: bytes) -> NativeCompactAnalysisJob: ... + def update_relations(self, relations: Mapping[str, Mapping[str, str]]) -> None: ... + def normalize_analysis_sqls( + self, *, dialect: str, requests: list[NativeNormalizationRequest] + ) -> list[str | Exception]: ... + +class NativeBindingPositions(Protocol): + def position( + self, *, start: int | None, line: int | None, column: int | None, message: str + ) -> tuple[int | None, int | None]: ... + +class NativePositionsModule(Protocol): + def normalize_analysis_sqls( + self, *, dialect: str, requests: list[NativeNormalizationRequest] + ) -> list[str | Exception]: ... + def binding_diagnostics( + self, *, sql: str, dialect: str, rows: list[NativeDiagnosticRow] + ) -> list[NativeDiagnosticRow]: ... + def BindingPositions( + self, + request: dict[str, object], + ) -> NativeBindingPositions: ... + def normalize_analysis_sql(self, request: dict[str, object]) -> str: ... + def normalize_dialect_sql(self, *, sql: str, dialect: str) -> str: ... + +type RelationKeys = dict[str, tuple[tuple[str, ...], ...]] + +type NativePreparedSql = tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] + +type ObjectKeyRow = tuple[str, str] + +type NamespaceRow = tuple[str | None, str | None, str | None, str | None] + type _GraphKey = tuple[str, str] type _GraphEdges = list[tuple[_GraphKey, list[_GraphKey]]] type _GraphIndexes = tuple[ @@ -792,17 +887,17 @@ class NativeInputsView: @property def model_inputs(self) -> tuple[NativeModelInput, ...]: ... @property - def seed_inputs(self) -> tuple[CompileSeedInput, ...]: ... + def seed_inputs(self) -> tuple[SeedInput, ...]: ... @property - def source_inputs(self) -> tuple[CompileSourceInput, ...]: ... + def source_inputs(self) -> tuple[SourceInput, ...]: ... @property - def sql_function_inputs(self) -> tuple[CompileSqlFunctionInput, ...]: ... + def sql_function_inputs(self) -> tuple[FunctionInput, ...]: ... @property def test_inputs(self) -> tuple[CompileSqlTestInput, ...]: ... @property def scenario_inputs(self) -> tuple[CompileSqlScenarioInput, ...]: ... @property - def audit_inputs(self) -> tuple[CompileAuditInput, ...]: ... + def audit_inputs(self) -> tuple[AuditInput, ...]: ... @property def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... @property diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py index 11c5082f04..c2084e97c1 100644 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -3,10 +3,9 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any +from typing import Any, Literal from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import QueryColumnsMode from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, CompileSqlReference, @@ -59,7 +58,7 @@ class NativeColumnQuery: """One query outside the session whose output columns Python's analysis inferred.""" sql: str - mode: QueryColumnsMode + mode: Literal["batch", "reanalysis", "parse", "legacy"] placeholders: dict[str, str] | None = None references: tuple[CompileSqlReference, ...] = () recover_cte_facts: bool = True diff --git a/src/sqlbuild/compiler/analysis_session/types.py b/src/sqlbuild/compiler/analysis_session/types.py deleted file mode 100644 index 1c2694b5f1..0000000000 --- a/src/sqlbuild/compiler/analysis_session/types.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Native model analysis session row types.""" - -from __future__ import annotations - -from typing import Literal - -type ColumnRow = tuple[str, str | None, str] -type DiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] -type ProofRow = tuple[ - bool, list[ColumnRow], list[tuple[str, str | None]], list[str], str | None, bool -] - -type QueryColumnsMode = Literal["batch", "reanalysis", "parse", "legacy"] diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 2e7f47cc58..748300918b 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -43,11 +43,9 @@ DiscoveredLoaderFunction, DiscoveredMaterializationFile, DiscoveredProjectInputs, - DiscoveredPythonFunctionFile, DiscoveredSchemaFile, DiscoveredSeedFile, DiscoveredSourceFile, - DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, DiscoveredSqlModelFile, DiscoveredSqlScenarioFile, @@ -667,48 +665,6 @@ class FunctionReturnColumn: type: str -@dataclass(frozen=True) -class CompileSqlFunctionInput: - """One discovered SQL function with validated compile-time metadata.""" - - function_file: DiscoveredSqlFunctionFile | DiscoveredPythonFunctionFile - name: str - arguments: tuple[FunctionArgument, ...] - returns: str - body_sql: str - return_columns: tuple[FunctionReturnColumn, ...] = field(default_factory=tuple) - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - database: str | None = None - schema: str | None = None - logical_database: str | None = None - logical_schema: str | None = None - fingerprint_database: str | None = None - fingerprint_schema: str | None = None - fingerprint_logical_database: str | None = None - fingerprint_logical_schema: str | None = None - language: FunctionLanguage = FunctionLanguage.SQL - runtime_version: str | None = None - entry_point: str | None = None - packages: tuple[str, ...] = field(default_factory=tuple) - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - tags: tuple[str, ...] = field(default_factory=tuple) - description: str | None = None - - @property - def assembly_row(self) -> tuple[str, str, list[str], list[tuple[str, str, str | None]]]: - """`(node type, name, tags, references)`, what assembly resolves dependencies from.""" - - return ( - CompiledResourceType.TABLE_FN if self.return_columns else CompiledResourceType.UDF, - self.name, - list(self.tags), - [ - (str(reference.ref_kind), reference.ref_name, reference.ref_package) - for reference in self.references - ], - ) - - @dataclass(frozen=True) class CompileModelInput: """One discovered model file with its attached schema metadata, if any.""" @@ -789,31 +745,6 @@ class ModelSqlAnalysis: dynamic_column_contract: DynamicColumnContractProof | None = None -@dataclass(frozen=True) -class CompileSeedInput: - """One discovered seed file with its attached seed metadata.""" - - seed_file: DiscoveredSeedFile - schema_entry: SchemaSeedEntry - schema_file: DiscoveredSchemaFile - - -@dataclass(frozen=True) -class CompileSourceInput: - """One normalized source declaration with its source file provenance.""" - - source_entry: SourceEntry - source_file: DiscoveredSourceFile - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - - @property - def assembly_row(self) -> tuple[str, bool, str | None, str | None]: - """`(name, has a loader, database, schema)`, what assembly resolves a namespace from.""" - - entry: SourceEntry = self.source_entry - return (entry.name, entry.loader is not None, entry.database, entry.schema) - - @dataclass(frozen=True) class CompileSqlScenarioInput: """One discovered SQL-native scenario with compile-time SQL expansion applied.""" @@ -834,56 +765,6 @@ class CompileSqlScenarioInput: declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class CompileAuditInput: - """One discovered SQL audit block with compile-time SQL expansion applied.""" - - audit_file: DiscoveredAuditFile - audit_block: DiscoveredAuditBlock - sql_body: str - evaluation_mode: AuditEvaluationMode = AuditEvaluationMode.VIOLATIONS - measurement_contract: MeasurementContract | None = None - thresholds: MeasurementThresholds | None = None - minimum_samples: int | None = None - measure_sql: str | None = None - evidence_sql: str | None = None - evidence_limit: int | None = None - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - attached_target_kind: AttachedAuditTargetKind | str | None = None - attached_target_name: str | None = None - attached_column_name: str | None = None - severity: AuditSeverity | None = None - run_scope: str | None = None - always_run: bool = False - declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - name: str | None = None - description: str | None = None - - def __post_init__(self) -> None: - if self.attached_target_kind is not None: - object.__setattr__( - self, - "attached_target_kind", - AttachedAuditTargetKind(self.attached_target_kind), - ) - - @property - def assembly_row( - self, - ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: - """`(references, attached (kind, name))`, what assembly resolves scope deps from.""" - - return ( - [ - (str(reference.ref_kind), reference.ref_name, reference.ref_package) - for reference in self.references - ], - (str(self.attached_target_kind), self.attached_target_name) - if self.attached_target_kind is not None and self.attached_target_name is not None - else None, - ) - - @dataclass(frozen=True) class CompileProjectInputs: """Attached project view used as the first pre-semantic compile input snapshot.""" diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 2ef7d95c3a..65da2263d0 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -25,8 +25,6 @@ CompiledSqlTest, CompileModelConfig, CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, CompileSqlReference, CompileSqlScenarioInput, CompileSqlTestInput, @@ -74,7 +72,7 @@ class SourceInput(Protocol): - """One attached source: a native owner view, or a hand-built `CompileSourceInput`.""" + """One attached source, as the native owner view holds it.""" @property def source_entry(self) -> SourceEntry: ... @@ -102,7 +100,7 @@ def source_file(self) -> DiscoveredSourceFile: ... class SeedInput(Protocol): - """One seed paired with its declaration: a native owner view, or a `CompileSeedInput`.""" + """One seed paired with its declaration, as the native owner view holds it.""" @property def seed_file(self) -> DiscoveredSeedFile: ... @@ -134,7 +132,7 @@ def external(self) -> bool: ... class FunctionInput(Protocol): - """One attached function: a native owner view, or a hand-built `CompileSqlFunctionInput`.""" + """One attached function, as the native owner view holds it.""" @property def function_file(self) -> DiscoveredSqlFunctionFile | DiscoveredPythonFunctionFile: ... @@ -228,7 +226,7 @@ def fingerprint_destination(self) -> CompiledRelationLocation: ... class AuditInput(Protocol): - """One attached audit: a native owner view, or a hand-built `CompileAuditInput`.""" + """One attached audit, as the native owner view holds it.""" @property def audit_file(self) -> DiscoveredAuditFile: ... @@ -363,9 +361,9 @@ def public_constants(self) -> dict[str, ConstantDeclaration]: ... @property def model_inputs(self) -> tuple[ModelInput, ...]: ... @property - def seed_inputs(self) -> tuple[CompileSeedInput, ...]: ... + def seed_inputs(self) -> tuple[SeedInput, ...]: ... @property - def source_inputs(self) -> tuple[CompileSourceInput, ...]: ... + def source_inputs(self) -> tuple[SourceInput, ...]: ... @property def sql_function_inputs(self) -> tuple[FunctionInput, ...]: ... @property @@ -596,12 +594,6 @@ def render_typed_array(self, *, value: SqlValue) -> str: ... def render_typed_object(self, *, value: SqlValue) -> str: ... -class CompileInputsReadyCallback(Protocol): - """Invocation-local observer for the completed pre-semantic input snapshot.""" - - def __call__(self, *, inputs: CompileInputsView) -> None: ... - - class AttachedAuditTargetKind(StrEnum): MODEL = "model" SOURCE = "source" @@ -660,3 +652,9 @@ class SqlTestMode(StrEnum): type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] + + +class CompileInputsReadyCallback(Protocol): + """Invocation-local observer for the completed pre-semantic input snapshot.""" + + def __call__(self, *, inputs: CompileInputsView) -> None: ... diff --git a/src/sqlbuild/compiler/fingerprints/types.py b/src/sqlbuild/compiler/fingerprints/types.py deleted file mode 100644 index 88c3256fcd..0000000000 --- a/src/sqlbuild/compiler/fingerprints/types.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Fingerprint callable contracts.""" - -from __future__ import annotations - -from typing import Protocol - - -class FingerprintWriteProgress(Protocol): - def __call__(self, *, completed: int, total: int) -> None: ... diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py index b65cfeadd9..ca80153a2c 100644 --- a/src/sqlbuild/compiler/project_assembly/models.py +++ b/src/sqlbuild/compiler/project_assembly/models.py @@ -4,20 +4,17 @@ from dataclasses import dataclass -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.project_assembly.types import NamespaceRow, ObjectKeyRow - @dataclass(frozen=True, slots=True) class NativeProjectResources: """Per-resource facts in input order, and whether each model's SQL passed syntax validation.""" - model_deps: tuple[list[ObjectKeyRow], ...] + model_deps: tuple[list[tuple[str, str]], ...] model_syntax_valid: tuple[bool, ...] source_namespaces: tuple[tuple[str | None, str | None] | None, ...] - seed_namespaces: tuple[NamespaceRow, ...] - function_deps: tuple[list[ObjectKeyRow], ...] - audit_scope_deps: tuple[list[ObjectKeyRow], ...] + seed_namespaces: tuple[tuple[str | None, str | None, str | None, str | None], ...] + function_deps: tuple[list[tuple[str, str]], ...] + audit_scope_deps: tuple[list[tuple[str, str]], ...] failure: NativeResourceFailure | None = None @@ -34,8 +31,18 @@ class NativeResourceFailure: class NativeModelFacts: """One model's native facts; its dependencies as `(resource type, name)` rows.""" - deps: list[ObjectKeyRow] - dynamic_contract: ProofRow | None + deps: list[tuple[str, str]] + dynamic_contract: ( + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None + ) syntax_valid: bool = False diff --git a/src/sqlbuild/compiler/project_assembly/types.py b/src/sqlbuild/compiler/project_assembly/types.py deleted file mode 100644 index 65e981eb20..0000000000 --- a/src/sqlbuild/compiler/project_assembly/types.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Plain-data rows exchanged with the native project assembly.""" - -from __future__ import annotations - -type ObjectKeyRow = tuple[str, str] -type NamespaceRow = tuple[str | None, str | None, str | None, str | None] diff --git a/src/sqlbuild/compiler/scopes/types.py b/src/sqlbuild/compiler/scopes/types.py index c1286dd702..c9dfe6bcd3 100644 --- a/src/sqlbuild/compiler/scopes/types.py +++ b/src/sqlbuild/compiler/scopes/types.py @@ -3,15 +3,6 @@ from __future__ import annotations from enum import StrEnum -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from pathlib import PurePath - - from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity - -type JsonValue = None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue] -type ScopeQueryTarget = str | PurePath | ResourceIdentity | DeclarationIdentity class ResourceKind(StrEnum): diff --git a/src/sqlbuild/compiler/sql_analysis/types.py b/src/sqlbuild/compiler/sql_analysis/types.py deleted file mode 100644 index 16f0913380..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/types.py +++ /dev/null @@ -1,55 +0,0 @@ -"""SQL analysis type-layer declarations.""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import Protocol - -type NativeDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] -type NativeNormalizationRequest = tuple[str, dict[str, str], dict[str, str]] - - -class NativeCompactAnalysisJob(Protocol): - def run(self) -> bytes: ... - - -class NativeProjectCatalog(Protocol): - def inferred_schema( - self, *, sql: str, columns: Mapping[str, str], inputs: Mapping[str, Mapping[str, str]] - ) -> list[tuple[str, str]]: ... - def with_relations( - self, relations: Mapping[str, Mapping[str, str]] - ) -> NativeProjectCatalog: ... - def validation_payload( - self, requests: list[tuple[str, list[tuple[str, bool]], dict[str, Mapping[str, str]]]] - ) -> str: ... - def update_analysis( - self, relations: Mapping[str, tuple[Mapping[str, str], Mapping[str, str]]] - ) -> None: ... - def register_override(self, relations: Mapping[str, Mapping[str, str]]) -> int: ... - def prepare_compact(self, payload: bytes) -> NativeCompactAnalysisJob: ... - def update_relations(self, relations: Mapping[str, Mapping[str, str]]) -> None: ... - def normalize_analysis_sqls( - self, *, dialect: str, requests: list[NativeNormalizationRequest] - ) -> list[str | Exception]: ... - - -class NativeBindingPositions(Protocol): - def position( - self, *, start: int | None, line: int | None, column: int | None, message: str - ) -> tuple[int | None, int | None]: ... - - -class NativePositionsModule(Protocol): - def normalize_analysis_sqls( - self, *, dialect: str, requests: list[NativeNormalizationRequest] - ) -> list[str | Exception]: ... - def binding_diagnostics( - self, *, sql: str, dialect: str, rows: list[NativeDiagnosticRow] - ) -> list[NativeDiagnosticRow]: ... - def BindingPositions( - self, - request: dict[str, object], - ) -> NativeBindingPositions: ... - def normalize_analysis_sql(self, request: dict[str, object]) -> str: ... - def normalize_dialect_sql(self, *, sql: str, dialect: str) -> str: ... diff --git a/src/sqlbuild/compiler/sql_test_glue/models.py b/src/sqlbuild/compiler/sql_test_glue/models.py index 5c8ec02498..ebb8d1fce7 100644 --- a/src/sqlbuild/compiler/sql_test_glue/models.py +++ b/src/sqlbuild/compiler/sql_test_glue/models.py @@ -3,13 +3,13 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Literal from sqlbuild.compiler.compile.models import ( CompiledSqlTest, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.sql_test_glue.types import SqlTestAssemblyFailureKind @dataclass(frozen=True) @@ -43,7 +43,7 @@ class NativeSqlTestChainRequest: class NativeSqlTestFailure: """The error one test's assembly raises: its kind and message.""" - kind: SqlTestAssemblyFailureKind + kind: Literal["input", "internal", "decimal_overflow"] message: str diff --git a/src/sqlbuild/compiler/sql_test_glue/types.py b/src/sqlbuild/compiler/sql_test_glue/types.py deleted file mode 100644 index e90849a619..0000000000 --- a/src/sqlbuild/compiler/sql_test_glue/types.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Row shapes the native SQL-test glue returns.""" - -from __future__ import annotations - -from typing import Literal - -type CtePairRow = tuple[str, str] -type NativeChainStepRow = tuple[ - str, - str, - str | None, - list[CtePairRow], - str | None, - list[str] | None, - list[CtePairRow], -] -type NativeAssertionStepRow = tuple[str, str, list[CtePairRow], str | None] -type NativePlanWarningRow = tuple[str | None, str, str] -type NativeSqlTestPlanRow = tuple[ - str | None, - list[NativeChainStepRow], - list[NativeAssertionStepRow], - list[str], - list[NativePlanWarningRow], - list[str], -] -type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] diff --git a/src/sqlbuild/lint/types.py b/src/sqlbuild/lint/types.py index 1255e5989f..10a7fee765 100644 --- a/src/sqlbuild/lint/types.py +++ b/src/sqlbuild/lint/types.py @@ -18,8 +18,6 @@ class LintSeverity(StrEnum): type RuleFixStatus = Literal["applied", "refused", "unavailable"] -type RelationKeys = dict[str, tuple[tuple[str, ...], ...]] - type RelationKeyRow = tuple[ str, str, @@ -35,5 +33,3 @@ class LintSeverity(StrEnum): type DiagnosticIdentity = tuple[str, str, str, str] type NativeLintPreparationRequest = LintPreparationRequest - -type NativePreparedSql = tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index 7b53a6d733..eaf40bf223 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -1,11 +1,14 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import QueryColumnsMode from sqlbuild.compiler.lineage.types import ColumnLineageMode +if TYPE_CHECKING: + from sqlbuild._native import QueryColumnsMode + @dataclass(frozen=True) class GeneratedAnalysisTestCase: diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index a33213c95a..fef49b8785 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field, replace from itertools import chain from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import pytest @@ -35,7 +35,6 @@ NativeModelAnalysisRequest, NativePivotTables, ) -from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, @@ -58,6 +57,9 @@ ) from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry +if TYPE_CHECKING: + from sqlbuild._native import ProofRow + LINEAGE_NATIVE: str = "native" _PROJECT_TOML: str = ( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index d81bc4f55a..d236fd3411 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, replace from itertools import chain from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import pytest @@ -16,13 +16,15 @@ from sqlbuild._native import build_project_graph, discover_project_inputs, project_with from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( generated_analysis_files, ) +if TYPE_CHECKING: + from sqlbuild._native import DiagnosticRow + _RELATIONS: dict[str, tuple[str, ...]] = { '__source("raw_orders")': ( "order_id", diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/classes/test_binding_catalog.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/classes/test_binding_catalog.py index 7a1e16d19d..cfd863e4b3 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/classes/test_binding_catalog.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/classes/test_binding_catalog.py @@ -1,11 +1,13 @@ """Prepared compact analysis jobs keep their catalog snapshot and run beside catalog updates.""" +from __future__ import annotations + from concurrent.futures import Future, ThreadPoolExecutor +from typing import TYPE_CHECKING import pytest from sqlbuild._native import BindingCatalog -from sqlbuild.compiler.sql_analysis.types import NativeCompactAnalysisJob from tests.unit.src.sqlbuild.compiler.sql_analysis.classes._test_types import ( CompactJobRerunCase, CompactJobSnapshotCase, @@ -18,6 +20,9 @@ retype_orders, ) +if TYPE_CHECKING: + from sqlbuild._native import NativeCompactAnalysisJob + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py index d3b072c465..ddc30ccb81 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py @@ -2,8 +2,10 @@ from collections.abc import Callable from dataclasses import dataclass +from typing import TYPE_CHECKING -from sqlbuild.compiler.sql_analysis.types import NativeProjectCatalog +if TYPE_CHECKING: + from sqlbuild._native import NativeProjectCatalog @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py index 040d0a2090..8dce30ffd0 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/helpers.py @@ -1,17 +1,21 @@ """Helpers for SQL analysis public-entry tests.""" +from __future__ import annotations + from collections.abc import Callable, Mapping, Sequence -from typing import cast +from typing import TYPE_CHECKING, cast import sqlbuild._native as _native from sqlbuild._native import BindingCatalog from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlBindingResult -from sqlbuild.compiler.sql_analysis.types import ( - NativeDiagnosticRow, - NativeNormalizationRequest, - NativePositionsModule, - NativeProjectCatalog, -) + +if TYPE_CHECKING: + from sqlbuild._native import ( + NativeDiagnosticRow, + NativeNormalizationRequest, + NativePositionsModule, + NativeProjectCatalog, + ) def nested_coalesce_sql(*, function_depth: int) -> str: @@ -44,7 +48,7 @@ def mapped_binding_result( ) -> SqlBindingResult: """Native polyglot diagnostic rows mapped to SQLBuild binding diagnostics for `sql`.""" - mapped: list[NativeDiagnosticRow] = cast(NativePositionsModule, _native).binding_diagnostics( + mapped: list[NativeDiagnosticRow] = cast("NativePositionsModule", _native).binding_diagnostics( sql=sql, dialect=dialect, rows=rows ) return SqlBindingResult(diagnostics=tuple(SqlBindingDiagnostic(*row) for row in mapped)) @@ -61,7 +65,7 @@ def normalize_analysis_sql_results( normalize: Callable[..., list[str | Exception]] = getattr( catalog, "normalize_analysis_sqls", - cast(NativePositionsModule, _native).normalize_analysis_sqls, + cast("NativePositionsModule", _native).normalize_analysis_sqls, ) batch: list[NativeNormalizationRequest] = [ (sql, dict(stubs or {}), dict(placeholders or {})) for sql, stubs, placeholders in requests @@ -72,7 +76,7 @@ def normalize_analysis_sql_results( def normalize_sql_for_polyglot(*, sql: str, dialect: str | None) -> str: """Return analysis-equivalent SQL for Polyglot's supported dialect surface.""" - return cast(NativePositionsModule, _native).normalize_dialect_sql( + return cast("NativePositionsModule", _native).normalize_dialect_sql( sql=sql, dialect=dialect or "generic" ) @@ -86,7 +90,7 @@ def normalize_analysis_sql( ) -> str: """Normalize one analysis SQL natively.""" - return cast(NativePositionsModule, _native).normalize_analysis_sql( + return cast("NativePositionsModule", _native).normalize_analysis_sql( { "sql": sql, "dialect": dialect or "generic", diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index 99430dead4..2d9277d037 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING from sqlbuild._native import ( lint_native_sql_bodies, @@ -18,7 +19,10 @@ LintConfig, LintViolation, ) -from sqlbuild.lint.types import NativeLintPreparationRequest, NativePreparedSql + +if TYPE_CHECKING: + from sqlbuild._native import NativePreparedSql + from sqlbuild.lint.types import NativeLintPreparationRequest def lint_bodies_for(*, file_path: Path, contents: str) -> tuple[LintBody, ...]: From a2ad560836d90f90eb081be06adf7881a1867eb8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:04:02 +0100 Subject: [PATCH 04/23] refactor: return native project assembly facts as extension classes --- .../_helpers/compile_driver/assembly_facts.rs | 38 ++++++----- .../bindings/_helpers/project_assembly/mod.rs | 1 + .../_helpers/project_assembly/models.rs | 44 ++++++++++++ .../project_assembly/native_resources.rs | 67 ++++++++----------- .../_helpers/project_assembly/registration.rs | 9 ++- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 53 ++++++++++++++- .../compiler/project_assembly/__init__.py | 1 - .../compiler/project_assembly/models.py | 54 --------------- .../compile/test_native_analysis_seams.py | 3 +- 10 files changed, 154 insertions(+), 118 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs delete mode 100644 src/sqlbuild/compiler/project_assembly/__init__.py delete mode 100644 src/sqlbuild/compiler/project_assembly/models.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index a50bdb50a0..aa9e48ede4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; const NATIVE_MODULE: &str = "sqlbuild._native"; const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const PROJECT_ASSEMBLY_MODELS_MODULE: &str = "sqlbuild.compiler.project_assembly.models"; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( @@ -60,8 +60,6 @@ pub(crate) fn native_project_facts<'py>( context, session, )?; - let model_facts: Bound<'py, PyAny> = - imported(py, PROJECT_ASSEMBLY_MODELS_MODULE, "NativeModelFacts")?; let deps: Bound<'py, PyAny> = resources.getattr("model_deps")?; let syntax_valid: Bound<'py, PyAny> = resources.getattr("model_syntax_valid")?; let mut models: Vec> = Vec::new(); @@ -70,22 +68,26 @@ pub(crate) fn native_project_facts<'py>( Some(proof) => proof, None => py.None().into_bound(py), }; - models.push(call_with( - &model_facts, - &[ - ("deps", deps.get_item(index)?), - ("dynamic_contract", proof), - ("syntax_valid", syntax_valid.get_item(index)?), - ], - )?); + models.push( + Bound::new( + py, + NativeModelFacts { + deps: deps.get_item(index)?.unbind(), + dynamic_contract: proof.unbind(), + syntax_valid: syntax_valid.get_item(index)?.unbind(), + }, + )? + .into_any(), + ); } - call_with( - &imported(py, PROJECT_ASSEMBLY_MODELS_MODULE, "NativeProjectFacts")?, - &[ - ("models", PyTuple::new(py, models)?.into_any()), - ("resources", resources), - ], - ) + Ok(Bound::new( + py, + NativeProjectFacts { + models: PyTuple::new(py, models)?.into_any().unbind(), + resources: resources.unbind(), + }, + )? + .into_any()) } /// Each standalone pivot model's dynamic contract proof, by model position. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs index b58d1f0cc0..f672104c84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod declared_relations; +pub(crate) mod models; pub(crate) mod native_resources; pub(crate) mod registration; pub(crate) mod resources; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs new file mode 100644 index 0000000000..a369286d57 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs @@ -0,0 +1,44 @@ +//! The resource and model facts the native project assembly hands to the compile driver. + +use pyo3::prelude::{Py, PyAny}; +use pyo3::pyclass; + +/// Per-resource facts in input order, and whether each model's SQL passed syntax validation. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct NativeProjectResources { + pub(crate) model_deps: Py, + pub(crate) model_syntax_valid: Py, + pub(crate) source_namespaces: Py, + pub(crate) seed_namespaces: Py, + pub(crate) function_deps: Py, + pub(crate) audit_scope_deps: Py, + /// The `NativeResourceFailure` Python's assembly raises at, or `None`. + pub(crate) failure: Py, +} + +/// The resource Python's assembly raises at, and the exception it raises there. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct NativeResourceFailure { + pub(crate) slot: Py, + pub(crate) index: Py, + pub(crate) error: Py, +} + +/// One model's native facts; its dependencies as `(resource type, name)` rows. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct NativeModelFacts { + pub(crate) deps: Py, + pub(crate) dynamic_contract: Py, + pub(crate) syntax_valid: Py, +} + +/// Every model's native facts and the resource facts. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct NativeProjectFacts { + pub(crate) models: Py, + pub(crate) resources: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 87c54458bf..4253a2d8c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -18,12 +18,14 @@ use sqlbuild_model_config::templates::models::Scalar; use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::_helpers::model_config::template_data::record_reads; +use crate::bindings::_helpers::project_assembly::models::{ + NativeProjectResources, NativeResourceFailure, +}; use crate::bindings::types::CompilerDetach; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; -const ASSEMBLY_MODELS_MODULE: &str = "sqlbuild.compiler.project_assembly.models"; const OS_MODULE: &str = "os"; const PROJECT_ASSEMBLY_CONTEXT: &str = "native project assembly"; const GENERIC_DIALECT: &str = "generic"; @@ -379,14 +381,15 @@ fn failure_object<'py>( ResourceSlot::Seed => SEED_SLOT, ResourceSlot::Audit => AUDIT_SLOT, }; - call_with( - &imported(py, ASSEMBLY_MODELS_MODULE, "NativeResourceFailure")?, - &[ - ("slot", PyString::new(py, slot).into_any()), - ("index", failure.index.into_pyobject(py)?.into_any()), - ("error", failure_error(py, failure.error, held)?), - ], - ) + Ok(Bound::new( + py, + NativeResourceFailure { + slot: PyString::new(py, slot).into_any().unbind(), + index: failure.index.into_pyobject(py)?.into_any().unbind(), + error: failure_error(py, failure.error, held)?.unbind(), + }, + )? + .into_any()) } fn namespace_row( @@ -426,36 +429,20 @@ fn resources_object<'py>( None => py.None().into_bound(py), }; let seeds: Vec<_> = resources.seeds.into_iter().map(namespace_row).collect(); - call_with( - &imported(py, ASSEMBLY_MODELS_MODULE, "NativeProjectResources")?, - &[ - ( - "model_deps", - tuple(resources.model_deps.into_pyobject(py)?.into_any())?, - ), - ( - "model_syntax_valid", - tuple(resources.model_syntax_valid.into_pyobject(py)?.into_any())?, - ), - ( - "source_namespaces", - tuple(resources.sources.into_pyobject(py)?.into_any())?, - ), - ( - "seed_namespaces", - tuple(seeds.into_pyobject(py)?.into_any())?, - ), - ( - "function_deps", - tuple(resources.function_deps.into_pyobject(py)?.into_any())?, - ), - ( - "audit_scope_deps", - tuple(resources.audit_deps.into_pyobject(py)?.into_any())?, - ), - ("failure", failure), - ], - ) + Ok(Bound::new( + py, + NativeProjectResources { + model_deps: tuple(resources.model_deps.into_pyobject(py)?.into_any())?.unbind(), + model_syntax_valid: tuple(resources.model_syntax_valid.into_pyobject(py)?.into_any())? + .unbind(), + source_namespaces: tuple(resources.sources.into_pyobject(py)?.into_any())?.unbind(), + seed_namespaces: tuple(seeds.into_pyobject(py)?.into_any())?.unbind(), + function_deps: tuple(resources.function_deps.into_pyobject(py)?.into_any())?.unbind(), + audit_scope_deps: tuple(resources.audit_deps.into_pyobject(py)?.into_any())?.unbind(), + failure: failure.unbind(), + }, + )? + .into_any()) } /// Return the resource facts and the resource Python's assembly raises at, if any. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 9823ce1309..4b6c8aa7b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -1,12 +1,19 @@ //! Register the native project assembly functions. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use crate::bindings::_helpers::project_assembly::models::{ + NativeModelFacts, NativeProjectFacts, NativeProjectResources, NativeResourceFailure, +}; use crate::bindings::_helpers::project_assembly::{ declared_relations, native_resources, resources, syntax_checks, syntax_validation, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; declared_relations::register(module)?; native_resources::register(module)?; syntax_validation::register(module)?; diff --git a/python-boundary.toml b/python-boundary.toml index 3ce9a218d6..e2b041020e 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 12232 +remaining_lines = 12177 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index f01be7f71a..46a6519840 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -303,7 +303,6 @@ from sqlbuild.compiler.planner.types import ( PlanAction, PlanReason, ) -from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonSqlRunLifecyclePlan, @@ -698,6 +697,58 @@ type ObjectKeyRow = tuple[str, str] type NamespaceRow = tuple[str | None, str | None, str | None, str | None] +class NativeProjectResources: + @property + def model_deps(self) -> tuple[list[tuple[str, str]], ...]: ... + @property + def model_syntax_valid(self) -> tuple[bool, ...]: ... + @property + def source_namespaces(self) -> tuple[tuple[str | None, str | None] | None, ...]: ... + @property + def seed_namespaces( + self, + ) -> tuple[tuple[str | None, str | None, str | None, str | None], ...]: ... + @property + def function_deps(self) -> tuple[list[tuple[str, str]], ...]: ... + @property + def audit_scope_deps(self) -> tuple[list[tuple[str, str]], ...]: ... + @property + def failure(self) -> NativeResourceFailure | None: ... + +class NativeResourceFailure: + @property + def slot(self) -> str: ... + @property + def index(self) -> int: ... + @property + def error(self) -> Exception: ... + +class NativeModelFacts: + @property + def deps(self) -> list[tuple[str, str]]: ... + @property + def dynamic_contract( + self, + ) -> ( + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None + ): ... + @property + def syntax_valid(self) -> bool: ... + +class NativeProjectFacts: + @property + def models(self) -> tuple[NativeModelFacts, ...]: ... + @property + def resources(self) -> NativeProjectResources: ... + type _GraphKey = tuple[str, str] type _GraphEdges = list[tuple[_GraphKey, list[_GraphKey]]] type _GraphIndexes = tuple[ diff --git a/src/sqlbuild/compiler/project_assembly/__init__.py b/src/sqlbuild/compiler/project_assembly/__init__.py deleted file mode 100644 index f3b7b4892e..0000000000 --- a/src/sqlbuild/compiler/project_assembly/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native compiled project assembly.""" diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py deleted file mode 100644 index ca80153a2c..0000000000 --- a/src/sqlbuild/compiler/project_assembly/models.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Resource facts the native project assembly returns, as Python's assembly builds them.""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True, slots=True) -class NativeProjectResources: - """Per-resource facts in input order, and whether each model's SQL passed syntax validation.""" - - model_deps: tuple[list[tuple[str, str]], ...] - model_syntax_valid: tuple[bool, ...] - source_namespaces: tuple[tuple[str | None, str | None] | None, ...] - seed_namespaces: tuple[tuple[str | None, str | None, str | None, str | None], ...] - function_deps: tuple[list[tuple[str, str]], ...] - audit_scope_deps: tuple[list[tuple[str, str]], ...] - failure: NativeResourceFailure | None = None - - -@dataclass(frozen=True, slots=True) -class NativeResourceFailure: - """The resource Python's assembly raises at, and the exception it raises there.""" - - slot: str - index: int - error: Exception - - -@dataclass(frozen=True, slots=True) -class NativeModelFacts: - """One model's native facts; its dependencies as `(resource type, name)` rows.""" - - deps: list[tuple[str, str]] - dynamic_contract: ( - tuple[ - bool, - list[tuple[str, str | None, str]], - list[tuple[str, str | None]], - list[str], - str | None, - bool, - ] - | None - ) - syntax_valid: bool = False - - -@dataclass(frozen=True, slots=True) -class NativeProjectFacts: - """Every model's native facts and the resource facts.""" - - models: tuple[NativeModelFacts, ...] - resources: NativeProjectResources diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index ab8d8c437d..e614d69543 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,11 +8,10 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import NativeProjectResources, ProjectColumnLineage from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.planner.models import NativeSqlTestArtifact -from sqlbuild.compiler.project_assembly.models import NativeProjectResources from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, From 1c40f76f3839ed0ec5d35289ccc19fef2c78ca27 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:26:04 +0100 Subject: [PATCH 05/23] refactor: host planner constants natively and drop dead planner declarations --- .../bindings/_helpers/cost_command/bounds.rs | 7 +- .../_helpers/plan_output/migrations.rs | 8 +- .../_helpers/planner_entries/model_entries.rs | 7 +- .../_helpers/planner_entries/values.rs | 3 +- .../planner_migrations/column_warnings.rs | 26 ++-- .../_helpers/planner_migrations/constants.rs | 6 + .../planner_migrations/model_handover.rs | 7 +- .../planner_migrations/model_warnings.rs | 21 +--- .../planner_migrations/planning_origin.rs | 8 +- .../planner_resolution/maximum_safety.rs | 40 ++++++- .../_helpers/planner_scenario/artifacts.rs | 10 +- .../bindings/_helpers/planner_scenario/cli.rs | 5 +- .../_helpers/planner_scenario/constants.rs | 10 ++ .../_helpers/planner_scenario/entries.rs | 3 +- .../_helpers/planner_scenario/execution.rs | 3 +- .../_helpers/planner_scenario/fixtures.rs | 3 +- .../_helpers/planner_scenario/graph.rs | 8 +- .../_helpers/planner_scenario/relations.rs | 7 +- .../bindings/_helpers/planner_scenario/sql.rs | 9 +- .../_helpers/planner_scenario/values.rs | 6 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 5 +- src/sqlbuild/compiler/planner/constants.py | 111 ------------------ src/sqlbuild/compiler/planner/exceptions.py | 29 +---- src/sqlbuild/compiler/planner/models.py | 32 +---- src/sqlbuild/compiler/planner/types.py | 31 ----- src/sqlbuild/cursor_algebra/constants.py | 5 - .../main/test_dropped_relation_first_run.py | 9 +- .../_helpers/test_warehouse_snapshot.py | 2 +- .../planner/_helpers/changes/test_detect.py | 3 +- .../planner/_helpers/migrations/helpers.py | 5 +- .../_helpers/test_scenario_artifacts.py | 8 +- 32 files changed, 142 insertions(+), 297 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs index 924dcecfe5..142313c100 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyInt}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cost_command::constants::ISO_DATE_LENGTH; use sqlbuild_core::cost_command::errors::CostCommandError; @@ -15,7 +15,7 @@ use crate::bindings::_helpers::cost_command::refusal::cost_error; const DATETIME_MODULE: &str = "datetime"; const DURATION_MODELS: &str = "sqlbuild.cursor_algebra.models"; -const DURATION_CONSTANTS: &str = "sqlbuild.cursor_algebra.constants"; +const MINUTE_TO_DAY_DURATION_UNITS: [&str; 3] = ["m", "h", "d"]; fn utc(py: Python<'_>) -> PyResult> { imported(py, DATETIME_MODULE, "UTC") @@ -105,8 +105,7 @@ pub(crate) fn parse_since<'py>(py: Python<'py>, value: &str) -> PyResult = - imported(py, DURATION_CONSTANTS, "MINUTE_TO_DAY_DURATION_UNITS")?; + let units: Bound<'py, PyAny> = PyFrozenSet::new(py, MINUTE_TO_DAY_DURATION_UNITS)?.into_any(); if !duration .call_method1("is_single_unit_in", (units,))? .is_truthy()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs index f835c7cf12..f9bba2e78e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs @@ -15,6 +15,8 @@ use sqlbuild_core::plan_display::main::{ }; use std::collections::BTreeMap; +const MANUAL_RENAME_HINT: &str = "matched by unchanged definition; sqb rename rewrites references and declares migrate_from so the move does not depend on discovery"; + fn label(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); @@ -116,11 +118,7 @@ pub(crate) fn migration_lines( .map(column) .collect::>()?, display: display_limit(display_options)?, - manual_rename_hint: plan - .py() - .import("sqlbuild.compiler.planner.constants")? - .getattr("MANUAL_RENAME_HINT")? - .extract()?, + manual_rename_hint: MANUAL_RENAME_HINT.to_owned(), }; Ok(format_model_migrations::format_model_migrations( lines, &report, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 82e93fa088..fdaf09081a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -2,8 +2,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::planner_entries::values::{ - COMPILE_EXCEPTIONS, COMPILE_TYPES, PLANNER_CONSTANTS, PLANNER_EXCEPTIONS, PLANNER_TYPES, - boolean, error_of, native, none, planner_model, planner_type, replaced, string, + COMPILE_EXCEPTIONS, COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, PLANNER_EXCEPTIONS, + PLANNER_TYPES, boolean, error_of, native, none, planner_model, planner_type, replaced, string, }; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, @@ -333,8 +333,7 @@ fn check_microbatch_concurrency<'py>( let settings = project.getattr("settings")?; if !configured.is_empty() && !settings.getattr("microbatch_concurrency")?.is_truthy()? { let guidance = native(py, "microbatch_guidance", &[])?; - let prefix: String = - imported(py, PLANNER_CONSTANTS, "MICROBATCH_CONCURRENCY_MESSAGE")?.extract()?; + let prefix: &str = MICROBATCH_CONCURRENCY_MESSAGE; return Err(error_of( py, (COMPILE_EXCEPTIONS, "CompileInputError"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index 694d1742f3..2da73008f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -9,7 +9,8 @@ pub(super) const NATIVE_MODULE: &str = "sqlbuild._native"; pub(super) const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; pub(super) const PLANNER_EXCEPTIONS: &str = "sqlbuild.compiler.planner.exceptions"; -pub(super) const PLANNER_CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; +pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = + "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; pub(super) const COMPILE_EXCEPTIONS: &str = "sqlbuild.compiler.compile.exceptions"; pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; pub(super) const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs index 9c8d7c5db8..e5553c1664 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs @@ -2,7 +2,11 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; -use crate::bindings::_helpers::planner_migrations::constants::DONE; +use crate::bindings::_helpers::planner_migrations::constants::{ + DONE, MISSING_ORIGIN_ALLOW_OUTCOME, MISSING_ORIGIN_CONFIRMATION_OUTCOME, + MISSING_ORIGIN_DENY_OUTCOME, +}; +use pyo3::exceptions::PyKeyError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::planner_migrations::main::column_warning::column_warning; use sqlbuild_analysis::planner_migrations::models::ColumnWarning; @@ -31,7 +35,6 @@ pub(super) fn warning<'py>( } pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult>> { - let py = entry.py(); let destination = entry.getattr("destination")?; let qualified = destination.getattr("qualified_name")?; let relation: String = if qualified.is_truthy()? { @@ -59,13 +62,8 @@ pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult( })?; warning(entry, code, &message, false) } + +/// The build outcome a missing migration origin has under its policy. +pub(super) fn missing_origin_outcome(policy: &Bound<'_, PyAny>) -> PyResult<&'static str> { + match policy.extract::()?.as_str() { + "allow" => Ok(MISSING_ORIGIN_ALLOW_OUTCOME), + "require_confirmation" => Ok(MISSING_ORIGIN_CONFIRMATION_OUTCOME), + "deny" => Ok(MISSING_ORIGIN_DENY_OUTCOME), + _ => Err(PyKeyError::new_err(policy.clone().unbind())), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs index b686ee5250..110d7f3ace 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/constants.rs @@ -15,3 +15,9 @@ pub(super) const GET_COLUMNS_FOR_INSPECTION: (&str, &str) = ( "sqlbuild.adapter.relations.main._get_columns_for_inspection", "get_columns_for_inspection", ); +pub(super) const QUALIFIED_RELATION_MAX_PARTS: usize = 3; +pub(super) const MIGRATION_MODEL_NAME_METADATA_KEY: &str = "model_name"; +pub(super) const HIDDEN_ORIGIN_REMEDY: &str = "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)"; +pub(super) const MISSING_ORIGIN_ALLOW_OUTCOME: &str = "nothing is migrated and the build continues as if migrate_from were absent; remove migrate_from once no target still needs it"; +pub(super) const MISSING_ORIGIN_CONFIRMATION_OUTCOME: &str = "building without it needs confirmation (missing_migration_origin = require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if it is no longer needed"; +pub(super) const MISSING_ORIGIN_DENY_OUTCOME: &str = "the build stops (missing_migration_origin = deny); if the migration already happened elsewhere or is no longer needed, remove migrate_from"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index e059dae71b..2abe787222 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_migrations::column_objects::{native, replace, text}; +use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_MODEL_NAME_METADATA_KEY; use crate::bindings::_helpers::state_rows::timestamps::presumed_utc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; @@ -37,11 +38,7 @@ pub(super) fn handover<'py>( } Err(error) => return Err(error), }; - let key = imported( - py, - "sqlbuild.compiler.planner.constants", - "MIGRATION_MODEL_NAME_METADATA_KEY", - )?; + let key = MIGRATION_MODEL_NAME_METADATA_KEY; if let Ok(payload) = payload.cast::() && payload.contains(&key)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs index ddae3fd5a5..35267749c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_warnings.rs @@ -1,7 +1,9 @@ //! Adapt diagnostic display facts while preserving policy and code ordering. -use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_migrations::column_warnings::warning; +use crate::bindings::_helpers::planner_migrations::column_warnings::{ + missing_origin_outcome, warning, +}; +use crate::bindings::_helpers::planner_migrations::constants::HIDDEN_ORIGIN_REMEDY; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyString; use sqlbuild_analysis::planner_migrations::main::model_warning::model_warning; @@ -21,19 +23,8 @@ pub(super) fn entry_warning<'py>(entry: &Bound<'py, PyAny>) -> PyResult( .call_method1("strip", ("\"`",))?, ); } - let maximum: usize = imported( - py, - "sqlbuild.compiler.planner.constants", - "QUALIFIED_RELATION_MAX_PARTS", - )? - .extract()?; + let maximum: usize = QUALIFIED_RELATION_MAX_PARTS; let mut invalid = parts.len() > maximum; for part in &parts { invalid |= !part.is_truthy()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs index 86631c049f..a14a30a3ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/maximum_safety.rs @@ -221,11 +221,11 @@ fn apply_maximum_start_policy<'py>( Err(PyErr::from_value(call_with( &imported(py, EXCEPTIONS, "MaximumAutomaticStartError")?, &[ - ("evidence", evidence.clone()), ( - "non_idempotent", - non_idempotent.into_pyobject(py)?.to_owned().into_any(), + "message", + string(py, &exceeded_message(&evidence, non_idempotent)?), ), + ("evidence", evidence.clone()), ], )?)) }; @@ -246,6 +246,40 @@ fn apply_maximum_start_policy<'py>( ) } +fn rendered(value: &Bound<'_, PyAny>) -> PyResult { + call_with( + &imported(value.py(), "sqlbuild._native", "cursor_render")?, + &[("value", value.clone())], + )? + .extract() +} + +/// The message of an automatic start that exceeds its horizon. +fn exceeded_message(evidence: &Bound<'_, PyAny>, non_idempotent: bool) -> PyResult { + let detail: &str = if non_idempotent { + " on a non-idempotent materialization" + } else { + "" + }; + let highest = evidence.getattr("highest_eligible_target_max")?; + let highest: String = if highest.is_none() { + "None".to_owned() + } else { + string(evidence.py(), &rendered(&highest)?) + .repr()? + .to_string() + }; + Ok(format!( + "maximum automatic start policy exceeded{detail}: physical target MAX='{}', highest eligible MAX={highest}, effective post-lookback start='{}', horizon='{}', action={}, input={}.{}", + rendered(&evidence.getattr("physical_target_max")?)?, + rendered(&evidence.getattr("effective_start")?)?, + rendered(&evidence.getattr("maximum_allowed_start")?)?, + evidence.getattr("action")?.getattr("value")?.str()?, + evidence.getattr("target_relation")?.str()?, + evidence.getattr("cursor_column")?.str()?, + )) +} + pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(maximum_allowed_start, module)?)?; module.add_function(wrap_pyfunction!(apply_maximum_start_policy, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs index 20a3aac300..0b77cfebcd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -3,6 +3,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_scenario::constants::{ + SCENARIO_PLAN_HASH_COLLISION, SCENARIO_PLAN_INVALID_HASH_PREFIX, + SCENARIO_PLAN_RELATION_COLLISION, +}; use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, error, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -43,7 +47,7 @@ fn compute_prefix( return Err(error( py, "Scenario hash prefix length must be at least 1", - "SCENARIO_PLAN_INVALID_HASH_PREFIX", + SCENARIO_PLAN_INVALID_HASH_PREFIX, None, )?); } @@ -89,7 +93,7 @@ pub(crate) fn build_scenario_hash_index<'py>( &format!( "Scenario artifact hash collision: scenarios '{existing}' and '{name}' both map to hash prefix '{prefix}'. Rename one scenario file so SQLBuild can generate distinct warehouse artifact prefixes." ), - "SCENARIO_PLAN_HASH_COLLISION", + SCENARIO_PLAN_HASH_COLLISION, Some( "Rename one scenario file so SQLBuild can generate distinct artifact prefixes.", ), @@ -239,7 +243,7 @@ pub(crate) fn build_scenario_relation_map<'py>( &format!( "Scenario relation name collision in scenario '{scenario}'. Both scenario artifacts map to '{physical}': {old_kind} {old_name} and {new_kind} {new_name}. Rename one artifact or shorten one model/source name." ), - "SCENARIO_PLAN_RELATION_COLLISION", + SCENARIO_PLAN_RELATION_COLLISION, Some("Rename or shorten one of the colliding scenario artifacts."), )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 7d7bc1c889..9448426a6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -3,9 +3,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; -use crate::bindings::_helpers::planner_scenario::values::{CONSTANTS, MODELS, TYPES, string}; +use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -152,7 +153,7 @@ fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { code = Some(if value.is_truthy()? { value } else { - imported(py, CONSTANTS, "SCENARIO_PLAN_GRAPH_VALIDATION")? + string(py, SCENARIO_PLAN_GRAPH_VALIDATION) }); } messages.push(warning.getattr("message")?.extract::()?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs index 22fd61edbd..d88424ebf5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/constants.rs @@ -4,3 +4,13 @@ pub(super) const REF: &str = "ref"; pub(super) const SOURCE: &str = "source"; pub(super) const SOURCE_MARKER: usize = 2; pub(super) const DBT_REF: &str = "dbt_ref"; +pub(super) const SCENARIO_PLAN_INVALID_HASH_PREFIX: &str = "S501"; +pub(super) const SCENARIO_PLAN_HASH_COLLISION: &str = "S502"; +pub(super) const SCENARIO_PLAN_RELATION_COLLISION: &str = "S503"; +pub(super) const SCENARIO_PLAN_GRAPH_VALIDATION: &str = "S504"; +pub(super) const SCENARIO_PLAN_UNKNOWN_SEED: &str = "S507"; +pub(super) const SCENARIO_PLAN_MISSING_FIXTURE_SQL: &str = "S508"; +pub(super) const SCENARIO_PLAN_MISSING_RELATION_TARGET: &str = "S509"; +pub(super) const SCENARIO_PLAN_INVALID_FIXTURE: &str = "S510"; +pub(super) const SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER: &str = "S511"; +pub(super) const SCENARIO_PLAN_INTERNAL: &str = "S599"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs index 22748240c3..59729c0370 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/entries.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_UNKNOWN_SEED; use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, sql::required_target, @@ -32,7 +33,7 @@ pub(super) fn seed_entries<'py>( return Err(error( py, &format!("Scenario '{scenario}' requires unknown seed '{name}'"), - "SCENARIO_PLAN_UNKNOWN_SEED", + SCENARIO_PLAN_UNKNOWN_SEED, None, )?); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index e82d4e2fda..cf00568184 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::by_attribute; +use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_INVALID_FIXTURE; use crate::bindings::_helpers::planner_scenario::{ entries::{function_entry, seed_entries}, fixtures::{extract_fixtures, fixture_plans}, @@ -239,7 +240,7 @@ fn complete_fixtures<'py>( context.label, details.join("\n") ), - "SCENARIO_PLAN_INVALID_FIXTURE", + SCENARIO_PLAN_INVALID_FIXTURE, None, )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs index 786df6d4ac..8d21bc9df3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/fixtures.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_MISSING_FIXTURE_SQL; use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, sql::{required_target, resolve_sql, source_label}, @@ -75,7 +76,7 @@ pub(super) fn fixture_plans<'py>( return Err(error( py, &format!("Scenario is missing {kind} fixture SQL '{name}'"), - "SCENARIO_PLAN_MISSING_FIXTURE_SQL", + SCENARIO_PLAN_MISSING_FIXTURE_SQL, None, )?); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index fe9dbb626a..d9f2d68a51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -2,7 +2,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; -use crate::bindings::_helpers::planner_scenario::values::{CONSTANTS, MODELS, TYPES, string}; +use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; +use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyTuple; use sqlbuild_analysis::planner_scenario::main::graph::graph; @@ -101,10 +102,7 @@ pub(crate) fn plan_scenario_graph<'py>( ("message", string(py, &message)), ]; if coded { - values.push(( - "code", - imported(py, CONSTANTS, "SCENARIO_PLAN_GRAPH_VALIDATION")?, - )); + values.push(("code", string(py, SCENARIO_PLAN_GRAPH_VALIDATION))); } warnings.push(call_with(&imported(py, MODELS, "PlanWarning")?, &values)?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index 13763c301f..d5aca78b04 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -4,6 +4,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; +use crate::bindings::_helpers::planner_scenario::constants::{ + SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, +}; use crate::bindings::_helpers::planner_scenario::values::{MODELS, error}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; @@ -90,7 +93,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( return Err(error( py, &format!("Scenario relation plan is missing seed target '{name}'"), - "SCENARIO_PLAN_MISSING_RELATION_TARGET", + SCENARIO_PLAN_MISSING_RELATION_TARGET, Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), )?); }; @@ -124,7 +127,7 @@ fn target_for_artifact<'py>( return Err(error( py, &format!("Scenario relation map is missing {kind} artifact '{name}'"), - "SCENARIO_PLAN_INTERNAL", + SCENARIO_PLAN_INTERNAL, Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), )?); }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs index e3b8b69279..5fceaafc9c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/sql.rs @@ -3,6 +3,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; +use crate::bindings::_helpers::planner_scenario::constants::{ + SCENARIO_PLAN_MISSING_RELATION_TARGET, SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, +}; use crate::bindings::_helpers::planner_scenario::{ models::SqlContext, values::{error, string}, @@ -48,7 +51,7 @@ pub(super) fn resolve_sql( if target.is_none() { let marker = string(py, marker).repr()?.to_str()?.to_owned(); return Err(error(py, &format!("Scenario SQL in {} references {marker}, which is not a relation in this scenario", context.label), - "SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER", Some("Reference a model, seed, source, or dbt ref the scenario builds or provides as a fixture."))?); + SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, Some("Reference a model, seed, source, or dbt ref the scenario builds or provides as a fixture."))?); } target.extract::() }).map_err(|failure| match failure { @@ -56,7 +59,7 @@ pub(super) fn resolve_sql( MarkerError::Scan(unclosed) => { let construct = match unclosed {Unclosed::BlockComment => "block comment", Unclosed::Quote => "quoted string", Unclosed::Parenthesis => "parenthesis"}; error(py, &format!("Scenario SQL in {} could not be scanned for relation markers: SQL reference resolution contains an unclosed {construct}", context.label), - "SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER", Some("Close every quoted string and block comment using the project dialect's quoting and comment rules.")).unwrap_or_else(|error| error) + SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER, Some("Close every quoted string and block comment using the project dialect's quoting and comment rules.")).unwrap_or_else(|error| error) } })?; } @@ -106,7 +109,7 @@ pub(super) fn required_target<'py>( return Err(error( relations.py(), &format!("Scenario relation plan is missing {kind} target '{name}'"), - "SCENARIO_PLAN_MISSING_RELATION_TARGET", + SCENARIO_PLAN_MISSING_RELATION_TARGET, Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs index c51cc792de..a5e5f5805d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/values.rs @@ -6,7 +6,6 @@ use pyo3::types::PyString; pub(super) const MODELS: &str = "sqlbuild.compiler.planner.models"; pub(super) const TYPES: &str = "sqlbuild.compiler.planner.types"; -pub(super) const CONSTANTS: &str = "sqlbuild.compiler.planner.constants"; pub(super) fn string<'py>(py: Python<'py>, text: &str) -> Bound<'py, PyAny> { PyString::new(py, text).into_any() @@ -18,10 +17,7 @@ pub(super) fn error( code: &str, help: Option<&str>, ) -> PyResult { - let mut args = vec![ - ("message", string(py, message)), - ("code", imported(py, CONSTANTS, code)?), - ]; + let mut args = vec![("message", string(py, message)), ("code", string(py, code))]; if let Some(help) = help { args.push(("help", string(py, help))); } diff --git a/python-boundary.toml b/python-boundary.toml index e2b041020e..8623b5cd59 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 12177 +remaining_lines = 11981 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 46a6519840..7651f99654 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -296,8 +296,6 @@ from sqlbuild.compiler.planner.models import ( from sqlbuild.compiler.planner.types import ( CursorGrain, CursorType, - FixtureGroups, - InputColumns, MaterializationType, OnSchemaChange, PlanAction, @@ -749,6 +747,9 @@ class NativeProjectFacts: @property def resources(self) -> NativeProjectResources: ... +type FixtureKey = tuple[CompiledResourceType, str] +type InputColumns = Callable[[str, str], frozenset[str] | None] +type FixtureGroups = tuple[tuple[CompiledResourceType, dict[str, str]], ...] type _GraphKey = tuple[str, str] type _GraphEdges = list[tuple[_GraphKey, list[_GraphKey]]] type _GraphIndexes = tuple[ diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index f849e52563..cffbab89c0 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -4,121 +4,10 @@ from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy -PATH_SELECTOR_EXPLICIT_ROOT_ERROR: str = ( - "path selectors require an explicit root: use 'models/' or 'python/'" -) -EMPTY_FINGERPRINT_METADATA_JSON: str = "{}" -RECORDED_RELATION_MISSING_WARNING_CODE: str = "RECORDED_RELATION_MISSING" -MODEL_SELECTOR_ROOT: str = "models" -SQL_FILE_SELECTOR_SUFFIX: str = ".sql" -MODEL_SELECTOR_ROOT_PREFIX: str = "models/" -SELECTOR_MISSING_NAME_ERROR_FRAGMENT: str = "no name" -PATH_SELECTOR_SEPARATOR: str = "~" SELECTOR_KIND_SEPARATOR: str = ":" -SELECTOR_PATH_SEPARATOR: str = "/" -EMPTY_SELECTOR_PATH: str = "" -POLYGLOT_CUSTOM_DATA_TYPE_NAME: str = "CUSTOM" -SQL_FUNCTION_CALL_OPEN_PAREN: str = "(" -SQL_ALIAS_BOUNDARY_CHARACTERS: frozenset[str] = frozenset("),;") -SQL_ALIAS_KEYWORD: str = "AS" -SQL_IDENTIFIER_LEADING_CHARACTERS: frozenset[str] = frozenset("[_") -SOURCE_ALIAS_BOUNDARY_CHARACTERS: frozenset[str] = frozenset(",);") -SQL_BRACKETED_IDENTIFIER_START: str = "[" -SQL_QUOTED_IDENTIFIER_DELIMITERS: frozenset[str] = frozenset({'"', "`"}) -SOURCE_DEFERRAL_CONTEXT_FIELDS: frozenset[str] = frozenset({"schema", "database"}) -UNIFIED_DIFF_ADDITION_PREFIX: str = "+" -UNIFIED_DIFF_ADDITION_HEADER_PREFIX: str = "+++" -UNIFIED_DIFF_REMOVAL_PREFIX: str = "-" -UNIFIED_DIFF_REMOVAL_HEADER_PREFIX: str = "---" -SELECTOR_EXPANSION_MARKER: str = "+" -SELECTOR_SUGGESTION_CUTOFF: float = 0.8 -SELECTOR_SUGGESTION_LIMIT: int = 3 -UNIT_TEST_SELECTOR_ERROR_CODE: str = "S013" -UNKNOWN_SELECTOR_ERROR_CODE: str = "S007" -UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD: str = ( - "only `sqb test` and `sqb build` accept unit-test selectors" -) -MODEL_CUSTOM_CONFIG_KEY: str = "config" -MODEL_PLACEHOLDERS_CONFIG_KEY: str = "placeholders" -MODEL_PRE_HOOKS_CONFIG_KEY: str = "pre_hooks" -MODEL_POST_HOOKS_CONFIG_KEY: str = "post_hooks" MICROBATCH_START_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.START) MICROBATCH_END_SENTINEL: str = sentinel_to_token(sentinel=BoundSentinel.END) -METADATA_NAME_FILTER_LIMIT: int = 250 -SCENARIO_ARTIFACT_PREFIX: str = "__sqb_" -SCENARIO_HASH_PREFIX_LENGTH: int = 12 -SCENARIO_ARTIFACT_KINDS: tuple[str, ...] = ("source", "ref", "seed", "dbt_ref", "model") -SCENARIO_DEFAULT_IDENTIFIER_LIMIT: int = 63 -QUERY_FINGERPRINT_FAILED: str = "S025" -SCENARIO_PLAN_INVALID_HASH_PREFIX: str = "S501" -SCENARIO_PLAN_HASH_COLLISION: str = "S502" -SCENARIO_PLAN_RELATION_COLLISION: str = "S503" -SCENARIO_PLAN_GRAPH_VALIDATION: str = "S504" -SCENARIO_PLAN_UNKNOWN_SEED: str = "S507" -SCENARIO_PLAN_MISSING_FIXTURE_SQL: str = "S508" -SCENARIO_PLAN_MISSING_RELATION_TARGET: str = "S509" -SCENARIO_PLAN_INVALID_FIXTURE: str = "S510" -SCENARIO_PLAN_UNRESOLVED_RELATION_MARKER: str = "S511" -SCENARIO_PLAN_INTERNAL: str = "S599" - -WHOLE_DAY_CURSOR_GRAINS: frozenset[str] = frozenset({"day", "month", "year"}) -VIEW_RELATION_TYPE_MARKER: str = "VIEW" -QUALIFIED_RELATION_MAX_PARTS: int = 3 -MIGRATION_MODEL_NAME_METADATA_KEY: str = "model_name" MIGRATION_FINGERPRINT_METADATA_KEY: str = "migration_fingerprint" -DECLARED_COLUMNS_METADATA_KEY: str = "declared_columns_hash" -LOCAL_FUNCTION_HASHES_METADATA_KEY: str = "local_function_hashes" -FULL_REFRESH_DISABLED_REBUILD_CODE: str = "S203" -MIGRATION_REF_PLACEHOLDER_PREFIX: str = "__sqb_migration_ref__" -MIGRATION_LOCAL_NAME_PREFIX: str = "__sqb_local_" -MIGRATION_FINGERPRINT_ALGORITHM: str = "migration-fingerprint-v1" -MIGRATION_FINGERPRINT_EXCLUDED_CONFIG_KEYS: frozenset[str] = frozenset( - { - "batch_size", - "full_refresh", - "incremental_mode", - "incremental_strategy", - "lookback", - "materialized", - "max_microbatches", - "microbatch_limit", - "microbatch_strategy", - "on_schema_change", - "snapshot_full_refresh", - "snapshot_schema_change", - } -) SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" -SOURCE_INPUT_FUNCTION: str = "__source" -REF_INPUT_FUNCTION: str = "__ref" -MANUAL_RENAME_HINT: str = ( - "matched by unchanged definition; sqb rename rewrites references and " - "declares migrate_from so the move does not depend on discovery" -) -HIDDEN_ORIGIN_REMEDY: str = ( - "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore " - "access to it, or remove migrate_from to build the model fresh deliberately " - "(missing_migration_origin does not apply)" -) -MISSING_ORIGIN_OUTCOMES: dict[MissingMigrationOriginPolicy, str] = { - MissingMigrationOriginPolicy.ALLOW: ( - "nothing is migrated and the build continues as if migrate_from were absent; remove " - "migrate_from once no target still needs it" - ), - MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION: ( - "building without it needs confirmation (missing_migration_origin = " - "require_confirmation); pass --allow-missing-migration-origin, or remove migrate_from if " - "it is no longer needed" - ), - MissingMigrationOriginPolicy.DENY: ( - "the build stops (missing_migration_origin = deny); if the migration already happened " - "elsewhere or is no longer needed, remove migrate_from" - ), -} -SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE: str = "source freshness unknown" -MICROBATCH_CONCURRENCY_MESSAGE: str = ( - "batch_concurrency > 1 requires concurrent microbatches; selected model(s): " -) diff --git a/src/sqlbuild/compiler/planner/exceptions.py b/src/sqlbuild/compiler/planner/exceptions.py index fb72dc94a5..18882d134f 100644 --- a/src/sqlbuild/compiler/planner/exceptions.py +++ b/src/sqlbuild/compiler/planner/exceptions.py @@ -4,8 +4,6 @@ from typing import Any -from sqlbuild._native import cursor_render as render - class PlannerInputError(ValueError): """Raised when planner inputs cannot be resolved safely.""" @@ -30,29 +28,6 @@ class FutureCursorSafetyError(PlannerInputError): class MaximumAutomaticStartError(PlannerInputError): """Raised when automatic cursor recovery would exceed its configured horizon.""" - def __init__( - self, - message: str | None = None, - *, - evidence: Any | None = None, - non_idempotent: bool = False, - ) -> None: + def __init__(self, message: str, *, evidence: Any | None = None) -> None: + super().__init__(message) self.evidence = evidence - if message is None and evidence is not None: - detail: str = " on a non-idempotent materialization" if non_idempotent else "" - message = ( - "maximum automatic start policy exceeded" - f"{detail}: physical target MAX='{render(value=evidence.physical_target_max)}', " - "highest eligible MAX=" - f"{self._render_optional_repr(evidence.highest_eligible_target_max)}, " - "effective post-lookback start=" - f"'{render(value=evidence.effective_start)}', " - f"horizon='{render(value=evidence.maximum_allowed_start)}', " - f"action={evidence.action.value}, " - f"input={evidence.target_relation}.{evidence.cursor_column}" - ) - super().__init__(message or "maximum automatic start policy exceeded") - - @staticmethod - def _render_optional_repr(value: Any | None) -> str: - return "None" if value is None else repr(render(value=value)) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 6a8ba4b1f8..424088c573 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -7,7 +7,7 @@ from datetime import UTC, datetime from decimal import Decimal, InvalidOperation from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from sqlbuild._native import MigrationFingerprintCache from sqlbuild._native import cursor_parse as parse @@ -52,7 +52,6 @@ CursorGrain, CursorType, CursorWatermarkMode, - FixtureKey, GraphResourceKind, MaterializationType, OnSchemaChange, @@ -98,6 +97,9 @@ ) from sqlbuild.sql_values.models import SqlValue +if TYPE_CHECKING: + from sqlbuild._native import FixtureKey + @dataclass(frozen=True) class NativeSqlTestArtifact: @@ -156,14 +158,6 @@ class RelationFixturePlanningContext: ) -@dataclass(frozen=True) -class GraphNodeKey: - """Neutral graph key matching fingerprint identity fields.""" - - node_type: str - node_name: str - - @dataclass(frozen=True) class SelectorExpansion: """One selector split into graph expansion flags and core text.""" @@ -182,16 +176,6 @@ class ParsedScenarioArtifactName: logical_name: str -@dataclass(frozen=True) -class GraphIdentityNode: - """Neutral node input for dependency-aware identity resolution.""" - - key: GraphNodeKey - resource_kind: GraphResourceKind - upstream_keys: tuple[GraphNodeKey, ...] - local_hash: str | None - - @dataclass(frozen=True, kw_only=True, init=False) class CursorOverrides: """Cursor override values from CLI flags; untyped `start`/`end` await planner typing.""" @@ -786,14 +770,6 @@ class SqlTestSelection: models_selected: bool = True -@dataclass(frozen=True) -class UnitTestSelectorNames: - """Names selectors can match: unit tests, and every other selectable resource.""" - - tests: frozenset[str] - resources: frozenset[str] - - @dataclass(frozen=True) class UnitTestSelectorSplit: """Resource selectors left after unit-test selectors moved into a test selection.""" diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index b203914048..026265d449 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -2,32 +2,7 @@ from __future__ import annotations -from collections.abc import Callable from enum import StrEnum -from typing import TYPE_CHECKING, Protocol - -from sqlbuild.compiler.compile.types import CompiledResourceType - -type FixtureKey = tuple[CompiledResourceType, str] -type InputColumns = Callable[[str, str], frozenset[str] | None] -type FixtureGroups = tuple[tuple[CompiledResourceType, dict[str, str]], ...] - -if TYPE_CHECKING: - from sqlbuild.compiler.planner.models import GraphNodeKey - - -class GraphIdentityComposer(Protocol): - def __call__( - self, - *, - local_hash: str, - upstream_hashes: tuple[tuple[GraphNodeKey, str], ...], - ) -> str: ... - - -class CursorInputRole(StrEnum): - FILTER = "filter" - WATERMARK = "watermark" class SelectorKind(StrEnum): @@ -172,12 +147,6 @@ class InitialValidFrom(StrEnum): EXECUTION_TIME = "execution_time" -class SnapshotFullRefreshPolicy(StrEnum): - DENY = "deny" - REQUIRE_CONFIRMATION = "require_confirmation" - ALLOW = "allow" - - class SnapshotSchemaChangePolicy(StrEnum): DENY = "deny" REQUIRE_CONFIRMATION = "require_confirmation" diff --git a/src/sqlbuild/cursor_algebra/constants.py b/src/sqlbuild/cursor_algebra/constants.py index 9d38c029db..4c93c60836 100644 --- a/src/sqlbuild/cursor_algebra/constants.py +++ b/src/sqlbuild/cursor_algebra/constants.py @@ -11,14 +11,9 @@ CursorGrain.YEAR: 5, } - DURATION_YEAR_UNIT: str = "y" DURATION_MONTH_UNIT: str = "mo" DURATION_DAY_UNIT: str = "d" DURATION_HOUR_UNIT: str = "h" DURATION_MINUTE_UNIT: str = "m" DURATION_SECOND_UNIT: str = "s" - -MINUTE_TO_DAY_DURATION_UNITS: frozenset[str] = frozenset( - {DURATION_MINUTE_UNIT, DURATION_HOUR_UNIT, DURATION_DAY_UNIT} -) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py index 236231f93d..2320657d71 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_dropped_relation_first_run.py @@ -7,7 +7,6 @@ import pytest -from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( AbsentMicrobatchFullRefreshTestCase, DroppedIncrementalFirstRunRangeTestCase, @@ -145,16 +144,12 @@ def test_given_relation_dropped_outside_sqlbuild_when_planning_and_building_then model["built_version_present"], model["identity_status"], ) == (None, False, "missing") - assert plan_warning_identities(plan=plan) == ( - ("orders", RECORDED_RELATION_MISSING_WARNING_CODE), - ) + assert plan_warning_identities(plan=plan) == (("orders", "RECORDED_RELATION_MISSING"),) assert rebuild_exit_code == 0, rebuild_output assert rebuilt_rows == test_case.expected_rows assert steady_exit_code == 0, steady_output assert query_duckdb_rows(db_path=db_path, sql=_BUILD_ARTIFACTS_SQL) == () - assert ("orders", RECORDED_RELATION_MISSING_WARNING_CODE) not in plan_warning_identities( - plan=steady_plan - ) + assert ("orders", "RECORDED_RELATION_MISSING") not in plan_warning_identities(plan=steady_plan) assert plan_model(plan=steady_plan, name="orders")["identity_status"] == "current" diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 3e2f071bd2..7b5d5fbdec 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -25,7 +25,6 @@ CompiledResourceType, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorOverrides, @@ -57,6 +56,7 @@ with_leading_enforced_model_contracts, ) +METADATA_NAME_FILTER_LIMIT: int = 250 _INCREMENTAL_MODEL: _IncrementalModelSpec = _IncrementalModelSpec( name="fact_orders", schema="staging", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index c4099e9a61..ee1c1effcb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -18,7 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.constants import RECORDED_RELATION_MISSING_WARNING_CODE from sqlbuild.compiler.planner.models import ( BackfillResult, ChangeDetectionResult, @@ -465,7 +464,7 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir False, (), ) - assert [warning.code for warning in warnings] == [RECORDED_RELATION_MISSING_WARNING_CODE] + assert [warning.code for warning in warnings] == ["RECORDED_RELATION_MISSING"] assert warnings[0].severity == WarningSeverity.WARNING assert warnings[0].model_name == "orders" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 516ac3c56d..1c613cdf19 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -4,12 +4,15 @@ import json from datetime import UTC, datetime +from typing import TYPE_CHECKING from sqlbuild._native import build_migration_fingerprint, parse_query_shape from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import OldNameViewAction from sqlbuild.compiler.planner.models import OldNameViewPlanEntry, QueryShape -from sqlbuild.compiler.planner.types import InputColumns + +if TYPE_CHECKING: + from sqlbuild._native import InputColumns BASE_CONFIG: dict[str, object] = { "materialized": "incremental", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index e106f642a7..c04e12e040 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -10,10 +10,6 @@ is_scenario_artifact_physical_name, parse_scenario_artifact_physical_name, ) -from sqlbuild.compiler.planner.constants import ( - SCENARIO_PLAN_HASH_COLLISION, - SCENARIO_PLAN_RELATION_COLLISION, -) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ParsedScenarioArtifactName, @@ -102,7 +98,7 @@ def test_given_colliding_scenario_hashes_when_building_index_then_raises_clear_e prefix_length=test_case.prefix_length, run_namespace=test_case.run_namespace, ) - assert exc_info.value.code == SCENARIO_PLAN_HASH_COLLISION + assert exc_info.value.code == "S502" @pytest.mark.parametrize( @@ -315,4 +311,4 @@ def test_given_colliding_scenario_artifacts_when_building_relation_map_then_rais artifacts=test_case.artifacts, normalize_identifier=test_case.normalize_identifier, ) - assert exc_info.value.code == SCENARIO_PLAN_RELATION_COLLISION + assert exc_info.value.code == "S503" From ca9e417388d0dcc2b26ac18e89bab0649b708f6e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:09:42 +0100 Subject: [PATCH 06/23] refactor: host planner value records natively --- .../compile_driver/python_plan_entries.rs | 3 +- .../src/bindings/_helpers/mod.rs | 1 + .../_helpers/planner_changes/actions.rs | 5 +- .../_helpers/planner_changes/detect.rs | 8 +- .../_helpers/planner_changes/renames.rs | 3 +- .../_helpers/planner_changes/results.rs | 2 +- .../_helpers/planner_entries/clone_plans.rs | 11 +- .../_helpers/planner_entries/display_plan.rs | 2 +- .../planner_entries/execution_plan.rs | 8 +- .../_helpers/planner_entries/model_entries.rs | 14 +- .../_helpers/planner_entries/plan_output.rs | 6 +- .../_helpers/planner_identity/graph.rs | 3 +- .../_helpers/planner_identity/identities.rs | 3 +- .../_helpers/planner_identity/staleness.rs | 6 +- .../planner_migrations/column_overlay.rs | 4 +- .../planner_migrations/column_planning.rs | 7 +- .../planner_migrations/column_warnings.rs | 5 +- .../_helpers/planner_migrations/columns.rs | 4 +- .../planner_migrations/compatibility.rs | 5 +- .../_helpers/planner_migrations/discovery.rs | 8 +- .../planner_migrations/discovery_requests.rs | 7 +- .../planner_migrations/model_handover.rs | 2 +- .../planner_migrations/model_overlay.rs | 5 +- .../planner_migrations/model_planning.rs | 10 +- .../planner_migrations/old_name_planning.rs | 4 +- .../planner_migrations/old_name_references.rs | 4 +- .../_helpers/planner_migrations/old_names.rs | 4 +- .../planner_migrations/planning_boundary.rs | 4 +- .../planner_migrations/projections.rs | 6 +- .../bindings/_helpers/planner_models/mod.rs | 5 + .../_helpers/planner_models/record.rs | 214 ++++ .../_helpers/planner_models/records.rs | 775 ++++++++++++++ .../_helpers/planner_models/registration.rs | 79 ++ .../planner_phases/execution_changes.rs | 12 +- .../_helpers/planner_phases/identities.rs | 8 +- .../planner_phases/output_assembly.rs | 4 +- .../_helpers/planner_phases/scopes.rs | 4 +- .../planner_phases/warehouse_state.rs | 6 +- .../planner_registration/registration.rs | 7 +- .../planner_resolution/cursor_warnings.rs | 3 +- .../planner_resolution/orchestration.rs | 6 +- .../_helpers/planner_scenario/artifacts.rs | 4 +- .../bindings/_helpers/planner_scenario/cli.rs | 4 +- .../_helpers/planner_scenario/execution.rs | 6 +- .../_helpers/planner_scenario/graph.rs | 9 +- .../planner_selection/model_selectors.rs | 3 +- .../_helpers/planner_selection/unit_tests.rs | 2 +- .../planner_strategy/fixture_values.rs | 2 +- .../planner_strategy/microbatch_limits.rs | 6 +- .../_helpers/planner_strategy/strategy.rs | 4 +- .../_helpers/planner_strategy/values.rs | 4 +- .../_helpers/planner_warehouse/snapshot.rs | 10 +- .../planner_warehouse/source_freshness.rs | 4 +- .../_helpers/sql_test_glue/plan_assembly.rs | 4 +- .../_helpers/sql_test_glue/plan_driver.rs | 3 +- .../_helpers/sql_test_glue/plan_rows.rs | 4 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 998 +++++++++++++++++- src/sqlbuild/compiler/planner/models.py | 622 +---------- .../compile/test_native_analysis_seams.py | 3 +- .../src/sqlbuild/cli/commands/main/helpers.py | 7 +- .../compiler/planner/_helpers/_test_types.py | 7 +- .../_helpers/fixtures/test_type_scope.py | 6 +- .../compiler/planner/_helpers/helpers.py | 2 +- .../planner/_helpers/resolve/helpers.py | 8 +- .../_helpers/resolve/test_orchestration.py | 10 +- .../planner/_helpers/resolve/test_resolve.py | 7 +- .../planner/_helpers/test_fixture_planning.py | 4 +- .../_helpers/test_warehouse_cursor_reads.py | 8 +- .../_helpers/test_warehouse_snapshot.py | 4 +- .../planner/main/test_build_execution_plan.py | 2 +- .../test_microbatch_reconciliation.py | 6 +- .../src/sqlbuild/executor/scenario/helpers.py | 4 +- .../sqlbuild/cli/output/main/plan/helpers.py | 10 +- .../cli/output/main/plan/test_plan_text.py | 6 +- .../compiler/planner/_helpers/_test_types.py | 12 +- .../planner/_helpers/changes/_test_helpers.py | 11 +- .../planner/_helpers/changes/_test_types.py | 8 +- .../planner/_helpers/changes/test_detect.py | 12 +- .../changes/test_reference_renames.py | 7 +- .../compiler/planner/_helpers/helpers.py | 10 +- .../planner/_helpers/identity/helpers.py | 2 +- .../test_merge_recomputed_model_changes.py | 4 +- .../planner/_helpers/migrations/helpers.py | 10 +- .../test_column_migration_candidates.py | 10 +- .../migrations/test_column_renames.py | 2 +- .../_helpers/migrations/test_query_shape.py | 7 +- .../planner/_helpers/planning/_test_types.py | 4 +- .../planner/_helpers/planning/helpers.py | 8 +- .../_helpers/planning/test_full_refresh.py | 8 +- .../_helpers/planning/test_identities.py | 8 +- .../_helpers/planning/test_retention.py | 11 +- .../planner/_helpers/planning/test_scopes.py | 6 +- .../planner/_helpers/resolve/helpers.py | 4 +- .../resolve/test_cursor_intrinsics.py | 10 +- .../planner/_helpers/resolve/test_hooks.py | 6 +- .../selection/test_unit_test_selectors.py | 9 +- .../_helpers/sql_test_assembly/helpers.py | 6 +- .../sql_test_assembly/test_plan_test.py | 6 +- .../planner/_helpers/test_buildability.py | 8 +- .../_helpers/test_cursor_type_check.py | 6 +- .../_helpers/test_fixture_planning_work.py | 10 +- .../planner/_helpers/test_microbatch_limit.py | 4 +- .../test_plan_entry_cursor_overrides.py | 9 +- .../_helpers/test_scenario_artifacts.py | 6 +- .../planner/_helpers/test_scenario_cli.py | 9 +- .../planner/_helpers/test_scenario_graph.py | 6 +- .../_helpers/test_scenario_relations.py | 2 +- .../_helpers/test_selection_staleness.py | 14 +- .../planner/_helpers/test_strategy_action.py | 9 +- .../_helpers/test_strategy_schema_actions.py | 9 +- .../_helpers/test_strategy_warnings.py | 6 +- .../planner/classes/test_sql_test_planning.py | 10 +- .../compiler/planner/main/_test_types.py | 6 +- .../sqlbuild/compiler/planner/main/helpers.py | 4 +- .../compiler/planner/main/identity/helpers.py | 5 +- .../main/identity/test_deep_graph_identity.py | 4 +- .../test_version_identity_function_hashes.py | 6 +- .../planner/main/test_selection_staleness.py | 4 +- .../planner/main/test_selector_expansion.py | 6 +- .../executor/pipeline/_helpers/helpers.py | 4 +- .../executor/scenario/_helpers/helpers.py | 4 +- .../executor/scenario/main/helpers.py | 4 +- .../src/sqlbuild/integrations/dbt/helpers.py | 5 +- 124 files changed, 2447 insertions(+), 956 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index de3bb1e189..9cf7e18501 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -351,7 +350,7 @@ fn skipped_ingress_warnings<'py>( let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; let selected: Bound<'py, PyAny> = run_selection.getattr("python_node_names")?; let upstream: HashMap> = upstream_names(graph)?; - let warning_class: Bound<'py, PyAny> = imported(py, PLANNER_MODELS_MODULE, "PlanWarning")?; + let warning_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "PlanWarning")?; let severity: Bound<'py, PyAny> = imported(py, PLANNER_TYPES_MODULE, "WarningSeverity")?.getattr("WARNING")?; let mut warnings: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 33931214d4..5b0a588488 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -50,6 +50,7 @@ pub(crate) mod planner_changes; pub(crate) mod planner_entries; pub(crate) mod planner_identity; pub(crate) mod planner_migrations; +pub(crate) mod planner_models; pub(crate) mod planner_phases; pub(crate) mod planner_registration; pub(crate) mod planner_resolution; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index 404fbe3599..baad948897 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -5,7 +5,6 @@ use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; @@ -21,7 +20,7 @@ fn resolve_model_actions<'py>( let py: Python<'py> = scope.py(); let model_kind: Bound<'py, PyAny> = imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; - let action_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "ResolvedModelAction")?; + let action_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "ResolvedModelAction")?; let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; let models_by_name: Bound<'py, PyAny> = scope.getattr("models_by_name")?; let model_changes: Bound<'py, PyAny> = changes.getattr("models")?; @@ -43,7 +42,7 @@ fn resolve_model_actions<'py>( )?; } call_with( - &imported(py, MODELS_MODULE, "PlannerResolvedActions")?, + &imported(py, "sqlbuild._native", "PlannerResolvedActions")?, &[("models", resolved.into_any())], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 3269ee20cd..0ee8cbe412 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::planner_changes::renames::origin_names; use crate::bindings::_helpers::planner_changes::results::{ - MODELS_MODULE, backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, + backfill_result, change_kind, planner_enum, replay_policy_error, schema_findings, }; use crate::bindings::_helpers::planner_identity::hashing::{ function_definition_hash, model_query_hash, @@ -86,7 +86,7 @@ fn detect_changes<'py>( ), )?; let functions: Bound<'py, PyDict> = PyDict::new(py); - let result_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "FunctionChangeResult")?; + let result_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "FunctionChangeResult")?; let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; let recorded: Bound<'py, PyAny> = snapshot.getattr("fingerprints")?.getattr("functions")?; let query_change_tracking: bool = project @@ -132,7 +132,7 @@ fn detect_changes<'py>( functions.set_item(name, call_with(&result_class, &arguments)?)?; } call_with( - &imported(py, MODELS_MODULE, "PlannerChangeResults")?, + &imported(py, "sqlbuild._native", "PlannerChangeResults")?, &[ ("models", models.into_any()), ("functions", functions.into_any()), @@ -758,7 +758,7 @@ impl<'py> ChangeResult<'py> { .map(|flag| flag.to_owned().into_any()) }; call_with( - &imported(py, MODELS_MODULE, "ChangeDetectionResult")?, + &imported(py, "sqlbuild._native", "ChangeDetectionResult")?, &[ ("model_name", self.model_name), ("change_kind", kind), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs index 32dfca7039..729f2e9f83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/renames.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_identity::hashing::model_query_hash; const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -187,7 +186,7 @@ fn plan_reference_renames<'py>( let target_name: Bound<'py, PyAny> = runtime .getattr("project")? .getattr("effective_target_name")?; - let rename_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "ReferenceRename")?; + let rename_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "ReferenceRename")?; let mut renames: Vec> = Vec::new(); for item in planned.call_method0("items")?.try_iter()? { let item: Bound<'py, PyAny> = item?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs index 113cab7398..727df386ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -45,7 +45,7 @@ pub(crate) fn backfill_result<'py>( BackfillAction::ForwardOnly => "forward", }; call_with( - &imported(py, MODELS_MODULE, "BackfillResult")?, + &imported(py, "sqlbuild._native", "BackfillResult")?, &[ ("action", planner_enum(py, "BackfillAction", action)?), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index 514364147f..c7327b2447 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -180,7 +180,7 @@ fn clone_model_entries<'py>( let function = function?; functions.set_item(function.getattr("name")?, function)?; } - let context = planner_model( + let context = native( py, "ModelPlanContext", &[ @@ -206,11 +206,11 @@ fn clone_model_entries<'py>( &[ ("adapter", adapter.clone()), ("model", model.clone()), - ("snapshot", planner_model(py, "WarehouseSnapshot", &[])?), + ("snapshot", native(py, "WarehouseSnapshot", &[])?), ("context", context), ( "backfill", - planner_model( + native( py, "BackfillResult", &[( @@ -220,10 +220,7 @@ fn clone_model_entries<'py>( )?, ), ("full_refresh", PyBool::new(py, false).to_owned().into_any()), - ( - "cursor_overrides", - planner_model(py, "CursorOverridePair", &[])?, - ), + ("cursor_overrides", native(py, "CursorOverridePair", &[])?), ], )? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs index ff15696e1c..c938ed6c98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/display_plan.rs @@ -104,7 +104,7 @@ fn display_entry<'py>( ("cursor_type", config(model, "cursor_type")?), ( "backfill", - planner_model( + native( py, "BackfillResult", &[( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index a7b06a8daa..fdedc2eb8d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -4,7 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; use crate::bindings::_helpers::planner_entries::values::{ - COMPILE_TYPES, boolean, native, planner_model, replaced, string, + COMPILE_TYPES, boolean, native, replaced, string, }; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; @@ -28,7 +28,7 @@ fn sql_test_planning<'py>( if !enabled { future.call_method1( "set_result", - (planner_model( + (native( py, "PlannedSqlTests", &[ @@ -130,7 +130,7 @@ fn change_results<'py>( ], )?; if !policies.getattr("selection_diagnostics")?.is_truthy()? { - let empty = planner_model( + let empty = native( py, "PlannerChangeResults", &[ @@ -351,7 +351,7 @@ fn build_execution_plan<'py>( let project = args.value("project")?; let adapter = args.value("adapter")?; let connection = args.value("connection")?; - let runtime = planner_model( + let runtime = native( py, "PlannerRuntime", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index fdaf09081a..b9a0afdb50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::planner_entries::values::{ COMPILE_EXCEPTIONS, COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, PLANNER_EXCEPTIONS, - PLANNER_TYPES, boolean, error_of, native, none, planner_model, planner_type, replaced, string, + PLANNER_TYPES, boolean, error_of, native, none, planner_type, replaced, string, }; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, @@ -39,7 +39,7 @@ fn plan_model_from_change<'py>( let py = model.py(); let full_refresh = boolean(py, request.full_refresh)?; let backfill = if request.full_refresh { - planner_model( + native( py, "BackfillResult", &[("action", planner_type(py, "BackfillAction", "FULL")?)], @@ -106,7 +106,7 @@ fn plan_model_from_change<'py>( ], )?; } - let pair = planner_model( + let pair = native( py, "CursorOverridePair", &[ @@ -371,7 +371,7 @@ fn check_microbatch_concurrency<'py>( return Ok(None); } let count = concurrent.len(); - planner_model( + native( py, "PlanWarning", &[ @@ -414,7 +414,7 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult let full_refresh = args.value("full_refresh")?; let inputs = args.value("build_inputs")?; let inputs = if inputs.is_none() { - planner_model(py, "PlanEntryBuildInputs", &[])? + native(py, "PlanEntryBuildInputs", &[])? } else { inputs }; @@ -439,7 +439,7 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult } let settings = project.getattr("settings")?; let models_by_name = scope.getattr("models_by_name")?; - let context = planner_model( + let context = native( py, "ModelPlanContext", &[ @@ -567,7 +567,7 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries)? { warnings.push(warning); } - planner_model( + native( py, "PlannerModelEntryResults", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index c0d6f9153b..e8715b7dda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -157,7 +157,7 @@ fn provider_usages<'py>( let mut usages: Vec> = Vec::new(); for usage in provider_usages.try_iter()? { let usage = usage?; - usages.push(planner_model( + usages.push(native( py, "PlanProviderUsage", &[ @@ -527,7 +527,7 @@ fn planned_sql_tests<'py>( ) -> PyResult> { let py = project.py(); let selected = selected_test_entries((project, adapter), selected_keys, selection, &none(py))?; - planner_model( + native( py, "PlannedSqlTests", &[ @@ -549,7 +549,7 @@ fn plan_output<'py>(args: &KeywordArguments<'py>) -> PyResult> let model_entry_results = args.value("model_entry_results")?; let extras = args.value("extras")?; let extras = if extras.is_none() { - planner_model(py, "PlanOutputExtras", &[])? + native(py, "PlanOutputExtras", &[])? } else { extras }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index eb2556c564..7659f048da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -27,7 +27,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::graph::algorithms::{closure, inverted}; use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_identity::hashing::graph_key; const EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -370,7 +369,7 @@ fn check_buildability<'py>( Some(name_set(&deferred_relations)?) }, }; - let entry_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "MissingUpstream")?; + let entry_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "MissingUpstream")?; let mut entries: Vec> = Vec::new(); for missing in missing_upstreams(&selected, &deps, &available) { entries.push(call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 0f06e11456..63bf9d947f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -17,7 +17,6 @@ use sqlbuild_analysis::planner_identity::models::{GraphKey, IdentityGraph, Ident use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_changes::results::MODELS_MODULE; use crate::bindings::_helpers::planner_identity::hashing::{graph_key, model_query_hash}; use crate::bindings::_helpers::planner_identity::metadata::{ function_local_hashes, model_version_identity_metadata_json, seed_identity, @@ -183,7 +182,7 @@ fn build_direct_model_version_identities<'py>( } } call_with( - &imported(py, MODELS_MODULE, "DirectModelVersionIdentities")?, + &imported(py, "sqlbuild._native", "DirectModelVersionIdentities")?, &[ ("function_local_hashes", function_hashes.into_any()), ("seed_version_hashes", seed_hashes.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index f1a0332f64..f3041d9f4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; -use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_changes::results::planner_enum; use crate::bindings::_helpers::planner_identity::hashing::graph_key; /// Change kinds that mean the model's own identity changed. @@ -143,7 +143,7 @@ fn plan_warnings<'py>( py: Python<'py>, warnings: Vec, ) -> PyResult> { - let warning_class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "PlanWarning")?; + let warning_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "PlanWarning")?; let severity: Bound<'py, PyAny> = planner_enum(py, "WarningSeverity", "warning")?; let mut rendered: Vec> = Vec::with_capacity(warnings.len()); for warning in warnings { @@ -258,7 +258,7 @@ fn classify_selection_staleness_warnings<'py>( }, }; let warning_class: Bound<'py, PyAny> = - imported(py, MODELS_MODULE, "SelectionStalenessWarning")?; + imported(py, "sqlbuild._native", "SelectionStalenessWarning")?; let mut warnings: Vec> = Vec::new(); for warning in classify_selection_staleness(&staleness) { warnings.push(call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 62862ac921..2fb09ff0e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -4,7 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, definition_hash, lower, model, native, replace, + config, definition_hash, lower, native, replace, }; use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -97,7 +97,7 @@ pub(super) fn overlay<'py>( ("existing_columns", columns), ( "fingerprints", - model( + native( py, "WarehouseFingerprints", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 761bc780a7..d3e566786d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -3,7 +3,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, replace}; +use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; @@ -42,7 +43,7 @@ fn plan_column_migrations<'py>( &adapter.call_method0("sql_analysis_dialect")?, )?; if candidates.is_empty() { - return model(py, "ColumnMigrationPlanning", &[("snapshot", snapshot)]); + return native(py, "ColumnMigrationPlanning", &[("snapshot", snapshot)]); } let database = call_with( &imported(py, "sqlbuild._native", "migration_planning_database")?, @@ -107,7 +108,7 @@ fn plan_column_migrations<'py>( } let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; let snapshot = super::column_overlay::cursors(&args, overlaid, &entries)?; - model( + native( py, "ColumnMigrationPlanning", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs index e5553c1664..a205870f8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs @@ -1,7 +1,8 @@ //! Preserve column-migration diagnostic codes and public messages. use crate::bindings::_helpers::compile_driver::python_calls::imported; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, text}; +use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::column_objects::text; use crate::bindings::_helpers::planner_migrations::constants::{ DONE, MISSING_ORIGIN_ALLOW_OUTCOME, MISSING_ORIGIN_CONFIRMATION_OUTCOME, MISSING_ORIGIN_DENY_OUTCOME, @@ -18,7 +19,7 @@ pub(super) fn warning<'py>( error: bool, ) -> PyResult> { let py = entry.py(); - model( + native( py, "PlanWarning", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs index a48b0a41be..a3c0fe63e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/columns.rs @@ -15,8 +15,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; - fn projections(shape: &Bound<'_, PyAny>) -> PyResult> { let mut result: Vec = Vec::new(); for item in shape.getattr("projections")?.try_iter()? { @@ -121,7 +119,7 @@ fn rename_hints<'py>( let mut result: Vec> = Vec::new(); for hint in hints { result.push(call_with( - &imported(py, MODELS_MODULE, "ColumnRenameHint")?, + &imported(py, "sqlbuild._native", "ColumnRenameHint")?, &[ ("model_name", arguments.value("model_name")?), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index 5babeb15e4..b9aa0e3417 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -10,7 +10,6 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::compatibility_findings::compatibility_findings; use sqlbuild_analysis::planner_migrations::models::CompatibilityFinding; -const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { @@ -143,7 +142,7 @@ fn check_migration_compatibility<'py>( let columns = PyDict::new(py); columns.set_item(model.getattr("name")?, arguments.value("origin_columns")?)?; let snapshot = call_with( - &imported(py, MODELS_MODULE, "WarehouseSnapshot")?, + &imported(py, "sqlbuild._native", "WarehouseSnapshot")?, &[ ("existing_relations", relations.into_any()), ("existing_columns", columns.into_any()), @@ -213,7 +212,7 @@ fn result<'py>( )? .call1((status,))?; call_with( - &imported(py, MODELS_MODULE, "MigrationCompatibilityResult")?, + &imported(py, "sqlbuild._native", "MigrationCompatibilityResult")?, &[("status", status), ("findings", findings.into_any())], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs index 32c1eb2f2b..7b4aa59682 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery.rs @@ -1,7 +1,7 @@ //! Native migration discovery over existing inspection and fingerprint cache boundaries. use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, native}; +use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::{ discovery_events, discovery_models, discovery_requests, }; @@ -46,7 +46,7 @@ fn discover_model_migrations<'py>( let triggered = discovery_models::triggered(&models, &scope, &args.value("snapshot")?, &declared_names)?; if !triggered && !manual.is_truthy()? { - return model(py, "ModelMigrationDiscovery", &[]); + return native(py, "ModelMigrationDiscovery", &[]); } let metadata = discovery_models::metadata(&runtime, &models)?; let dialect = runtime @@ -79,7 +79,7 @@ fn discover_model_migrations<'py>( ("fingerprints", cache), ], )?; - return model( + return native( py, "ModelMigrationDiscovery", &[("requests", manual), ("destination_fingerprints", hashes)], @@ -140,7 +140,7 @@ fn discover_model_migrations<'py>( ("fingerprints", cache), ], )?; - model( + native( py, "ModelMigrationDiscovery", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index d678acd4f0..ed9e4b42d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -1,7 +1,8 @@ //! Live candidate inspection and automatic request materialization. use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, model}; +use crate::bindings::_helpers::planner_migrations::column_objects::config; +use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::discovery_events::{ identity, key, newest, relation, }; @@ -137,7 +138,7 @@ pub(super) fn automatic<'py>( None }; if let Some((location, origin_model)) = origin { - requests.push(model( + requests.push(native( py, "ModelMigrationRequest", &[ @@ -183,7 +184,7 @@ pub(super) fn warnings<'py>( let mut warnings: Vec> = Vec::new(); for (name, detail) in ambiguous { if selected.contains(&name)? { - warnings.push(model(py, "PlanWarning", &[ + warnings.push(native(py, "PlanWarning", &[ ("model_name", PyString::new(py, &name).into_any()), ("severity", imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")?.getattr("WARNING")?), ("message", PyString::new(py, &format!("'{name}' matches {detail}; no automatic migration was inferred. Add migrate_from to choose one.")).into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs index 2abe787222..e968b917a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_handover.rs @@ -40,7 +40,7 @@ pub(super) fn handover<'py>( }; let key = MIGRATION_MODEL_NAME_METADATA_KEY; if let Ok(payload) = payload.cast::() - && payload.contains(&key)? + && payload.contains(key)? { payload.set_item(key, model.getattr("name")?)?; let kwargs = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index 2f099ab2ea..56eab7c306 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -3,7 +3,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_migrations::column_objects::{config, model, replace}; +use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; @@ -109,7 +110,7 @@ pub(super) fn overlay<'py>( ("existing_columns", columns.into_any()), ( "fingerprints", - model( + native( py, "WarehouseFingerprints", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 23c5718b55..c9ca422d39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -3,9 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_migrations::column_objects::{ - config, model, native, replace, -}; +use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; use crate::bindings::_helpers::planner_migrations::constants::MIGRATION_TYPES; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{ @@ -21,7 +19,7 @@ fn declared<'py>(item: &Bound<'py, PyAny>) -> PyResult> return Ok(None); } let materialization = config(item, "materialized")?; - Ok(Some(model( + Ok(Some(native( py, "ModelMigrationRequest", &[ @@ -181,7 +179,7 @@ fn discover<'py>( ("state", state.clone()), ], )?; - declarations.push(model( + declarations.push(native( py, "ModelMigrationDeclaration", &[ @@ -341,7 +339,7 @@ fn finish<'py>( if !entries.is_empty() { fields.push(("renamed_models", PyFrozenSet::new(py, renamed)?.into_any())); } - model( + native( py, "ModelMigrationPlanning", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index aff9f276f8..eb1ea8fe69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, native}; +use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; @@ -179,7 +179,7 @@ fn plan_old_name_views<'py>( warnings.extend(super::old_name_references::errors( &runtime, &views, &entries, )?); - model( + native( py, "OldNameViewPlanning", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs index c87e7b0639..c2fd95fd9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -1,7 +1,7 @@ //! Diagnose literal references to compatibility-only old names. use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; +use crate::bindings::_helpers::planner_migrations::column_objects::{native, text}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; @@ -121,7 +121,7 @@ pub(super) fn errors<'py>( .to_str()? .to_owned(); let line = literal.getattr("line")?.str()?.to_str()?.to_owned(); - errors.push(model( + errors.push(native( py, "PlanWarning", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 43f7f0a255..0cc97d4143 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -12,8 +12,6 @@ use sqlbuild_analysis::planner_migrations::main::{ old_name_skip_reason, }; -const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; - #[pyfunction] fn declared_column_aliases<'py>( py: Python<'py>, @@ -139,7 +137,7 @@ fn old_name_claim_errors<'py>( let message = claim_message(&name.extract::()?, &view, &arguments.value("now")?)?; errors.push(call_with( - &imported(py, MODELS_MODULE, "PlanWarning")?, + &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", name.clone()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index b3dc9a2486..0ad044005b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_migrations::column_objects::model; +use crate::bindings::_helpers::planner_migrations::column_objects::native; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; @@ -52,7 +52,7 @@ fn migration_redirected_cursor_snapshots<'py>( } else { config.getattr("cursors")?.getattr("start")? }; - let cursor_scope = model( + let cursor_scope = native( py, "CursorSnapshotScope", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs index b872bedc9a..c16a6abc8f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/projections.rs @@ -11,8 +11,6 @@ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_migrations::main::{query_shape, renames_explain_change as proof}; use sqlbuild_analysis::planner_migrations::models::{QueryProjection, QueryShape}; -const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; - fn read_shape(shape: &Bound<'_, PyAny>) -> PyResult { let mut projections: Vec = Vec::new(); for item in shape.getattr("projections")?.try_iter()? { @@ -56,7 +54,7 @@ fn parse_query_shape<'py>( let mut projections: Vec> = Vec::new(); for item in shape.projections { projections.push(call_with( - &imported(py, MODELS_MODULE, "QueryProjection")?, + &imported(py, "sqlbuild._native", "QueryProjection")?, &[ ("name", item.name.into_pyobject(py)?.into_any()), ("expression", item.expression.into_pyobject(py)?.into_any()), @@ -76,7 +74,7 @@ fn parse_query_shape<'py>( None => py.None().into_bound(py), }; call_with( - &imported(py, MODELS_MODULE, "QueryShape")?, + &imported(py, "sqlbuild._native", "QueryShape")?, &[ ("projections", PyTuple::new(py, projections)?.into_any()), ("body", shape.body.into_pyobject(py)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs new file mode 100644 index 0000000000..ab99095c4b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs @@ -0,0 +1,5 @@ +//! Planner value records hosted natively. + +pub(crate) mod record; +pub(crate) mod records; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs new file mode 100644 index 0000000000..f9cac51993 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/record.rs @@ -0,0 +1,214 @@ +//! Frozen value records with dataclass construction, equality, hashing and repr. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use pyo3::IntoPyObject; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; + +/// How a record field is filled when the caller leaves it out. +pub(crate) enum FieldDefault { + /// The caller must pass the field. + Required, + /// `None`. + NoneValue, + /// A boolean. + Bool(bool), + /// A string. + Text(&'static str), + /// A new empty tuple. + EmptyTuple, + /// A new empty frozenset. + EmptyFrozenSet, + /// A new empty dict. + EmptyDict, + /// An enum member, by class and member name. + Member((&'static str, &'static str), &'static str), + /// A new instance of a class called without arguments. + Class((&'static str, &'static str)), + /// A value built by a function. + Factory(for<'py> fn(Python<'py>) -> PyResult>), +} + +/// One record field. +pub(crate) struct FieldSpec { + pub(crate) name: &'static str, + pub(crate) default: FieldDefault, + pub(crate) compare: bool, + pub(crate) repr: bool, +} + +/// The fields of one record class. +pub(crate) struct RecordSpec { + pub(crate) name: &'static str, + pub(crate) kw_only: bool, + pub(crate) fields: &'static [FieldSpec], +} + +fn default_value<'py>(py: Python<'py>, default: &FieldDefault) -> PyResult> { + match default { + FieldDefault::Required => Ok(py.None().into_bound(py)), + FieldDefault::NoneValue => Ok(py.None().into_bound(py)), + FieldDefault::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().into_any()), + FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), + FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), + FieldDefault::EmptyFrozenSet => Ok(PyFrozenSet::empty(py)?.into_any()), + FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), + FieldDefault::Member((module, class), member) => { + imported(py, module, class)?.getattr(*member) + } + FieldDefault::Class((module, class)) => imported(py, module, class)?.call0(), + FieldDefault::Factory(build) => build(py), + } +} + +/// Bind constructor arguments to the record's fields in order, as a dataclass `__init__` does. +pub(crate) fn bind_fields( + spec: &RecordSpec, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult>> { + let py = args.py(); + if args.len() > 0 && spec.kw_only { + return Err(PyTypeError::new_err(format!( + "{}.__init__() takes 1 positional argument but {} were given", + spec.name, + args.len() + 1 + ))); + } + if args.len() > spec.fields.len() { + return Err(PyTypeError::new_err(format!( + "{}.__init__() takes {} positional arguments but {} were given", + spec.name, + spec.fields.len() + 1, + args.len() + 1 + ))); + } + let mut values: Vec>> = args.iter().map(Some).collect(); + values.resize_with(spec.fields.len(), || None); + if let Some(kwargs) = kwargs { + for (key, value) in kwargs.iter() { + let key: String = key.extract()?; + let Some(index) = spec.fields.iter().position(|field| field.name == key) else { + return Err(PyTypeError::new_err(format!( + "{}.__init__() got an unexpected keyword argument '{key}'", + spec.name + ))); + }; + if values[index].is_some() { + return Err(PyTypeError::new_err(format!( + "{}.__init__() got multiple values for argument '{key}'", + spec.name + ))); + } + values[index] = Some(value); + } + } + spec.fields + .iter() + .zip(values) + .map(|(field, value)| match value { + Some(value) => Ok(value.unbind()), + None if matches!(field.default, FieldDefault::Required) => { + Err(PyTypeError::new_err(format!( + "{}.__init__() missing 1 required argument: '{}'", + spec.name, field.name + ))) + } + None => Ok(default_value(py, &field.default)?.unbind()), + }) + .collect() +} + +/// The current field values with `changes` applied, as `dataclasses.replace` binds them. +pub(crate) fn replaced_fields( + spec: &RecordSpec, + current: Vec>, + changes: Option<&Bound<'_, PyDict>>, +) -> PyResult>> { + let mut values: Vec> = current.into_iter().map(Bound::unbind).collect(); + let Some(changes) = changes else { + return Ok(values); + }; + for (key, value) in changes.iter() { + let key: String = key.extract()?; + let Some(index) = spec.fields.iter().position(|field| field.name == key) else { + return Err(PyTypeError::new_err(format!( + "{}.__init__() got an unexpected keyword argument '{key}'", + spec.name + ))); + }; + values[index] = value.unbind(); + } + Ok(values) +} + +fn compared<'py>( + py: Python<'py>, + spec: &RecordSpec, + values: &[Bound<'py, PyAny>], +) -> PyResult> { + let kept: Vec> = spec + .fields + .iter() + .zip(values) + .filter(|(field, _)| field.compare) + .map(|(_, value)| value.clone()) + .collect(); + PyTuple::new(py, kept) +} + +/// Dataclass equality: same class and equal compared fields. +pub(crate) fn record_eq<'py>( + spec: &RecordSpec, + (left, right): (&[Bound<'py, PyAny>], Option<&[Bound<'py, PyAny>]>), + py: Python<'py>, +) -> PyResult> { + let Some(right) = right else { + return Ok(py.NotImplemented()); + }; + let equal: bool = compared(py, spec, left)?.eq(compared(py, spec, right)?)?; + Ok(equal.into_pyobject(py)?.to_owned().into_any().unbind()) +} + +/// Dataclass hash: the hash of the compared fields. +pub(crate) fn record_hash<'py>( + spec: &RecordSpec, + values: &[Bound<'py, PyAny>], + py: Python<'py>, +) -> PyResult { + compared(py, spec, values)?.hash() +} + +/// Dataclass repr: `Name(field=value, ...)` over the represented fields. +pub(crate) fn record_repr(spec: &RecordSpec, values: &[Bound<'_, PyAny>]) -> PyResult { + let mut parts: Vec = Vec::new(); + for (field, value) in spec.fields.iter().zip(values) { + if field.repr { + parts.push(format!("{}={}", field.name, value.repr()?)); + } + } + Ok(format!("{}({})", spec.name, parts.join(", "))) +} + +/// Give a record class the dataclass field table `dataclasses.replace`, `fields` and `asdict` read. +pub(crate) fn install_dataclass_fields( + class: &Bound<'_, PyAny>, + spec: &RecordSpec, +) -> PyResult<()> { + let py = class.py(); + let dataclasses = py.import("dataclasses")?; + let field_kind = dataclasses.getattr("_FIELD")?; + let table = PyDict::new(py); + for field in spec.fields { + let options = PyDict::new(py); + options.set_item("compare", field.compare)?; + options.set_item("repr", field.repr)?; + let described = dataclasses.call_method("field", (), Some(&options))?; + described.setattr("name", field.name)?; + described.setattr("type", "object")?; + described.setattr("_field_type", &field_kind)?; + table.set_item(field.name, described)?; + } + class.setattr("__dataclass_fields__", table) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs new file mode 100644 index 0000000000..ae17c63bdd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs @@ -0,0 +1,775 @@ +//! Planner value records the native planner builds and reads. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_models::record::FieldDefault; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; + +/// Declares a frozen record class over `Py` fields. +macro_rules! planner_record { + (@convert $py:ident, $values:expr) => {{ + let _ = $py; + $values + }}; + (@convert $py:ident, $values:expr, $convert:path) => { $convert($py, $values)? }; + ( + $(#[$meta:meta])* + $name:ident, $kw_only:literal, [ + $(($field:ident = $field_name:literal, $default:expr, $compare:literal, $repr:literal)),* $(,)? + ] + $(, convert = $convert:path)? + $(, methods { $($methods:tt)* })? + ) => { + $(#[$meta])* + #[pyo3::pyclass(module = "sqlbuild._native", frozen)] + pub(crate) struct $name { + $( + #[pyo3(get)] + $field: pyo3::Py, + )* + } + + impl $name { + const SPEC: $crate::bindings::_helpers::planner_models::record::RecordSpec = + $crate::bindings::_helpers::planner_models::record::RecordSpec { + name: stringify!($name), + kw_only: $kw_only, + fields: &[$( + $crate::bindings::_helpers::planner_models::record::FieldSpec { + name: $field_name, + default: $default, + compare: $compare, + repr: $repr, + }, + )*], + }; + + /// Add the class to `module` with its dataclass field table. + pub(crate) fn register_class( + module: &pyo3::Bound<'_, pyo3::types::PyModule>, + ) -> pyo3::PyResult<()> { + use pyo3::types::{PyAnyMethods, PyModuleMethods}; + module.add_class::()?; + $crate::bindings::_helpers::planner_models::record::install_dataclass_fields( + &module.getattr(stringify!($name))?, + &Self::SPEC, + ) + } + + fn values<'py>(&self, py: pyo3::Python<'py>) -> Vec> { + vec![$(self.$field.bind(py).clone()),*] + } + + fn from_values(values: Vec>) -> pyo3::PyResult { + let mut values = values.into_iter(); + Ok(Self { + $( + $field: values.next().ok_or_else(|| { + pyo3::exceptions::PyTypeError::new_err(concat!("missing ", $field_name)) + })?, + )* + }) + } + } + + #[pyo3::pymethods] + impl $name { + #[new] + #[pyo3(signature = (*args, **kwargs))] + fn __new__( + args: &pyo3::Bound<'_, pyo3::types::PyTuple>, + kwargs: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, + ) -> pyo3::PyResult { + let py = args.py(); + let values = $crate::bindings::_helpers::planner_models::record::bind_fields( + &Self::SPEC, + args, + kwargs, + )?; + Self::from_values(planner_record!(@convert py, values $(, $convert)?)) + } + + #[pyo3(signature = (**changes))] + fn __replace__( + &self, + py: pyo3::Python<'_>, + changes: Option<&pyo3::Bound<'_, pyo3::types::PyDict>>, + ) -> pyo3::PyResult { + let values = $crate::bindings::_helpers::planner_models::record::replaced_fields( + &Self::SPEC, + self.values(py), + changes, + )?; + Self::from_values(planner_record!(@convert py, values $(, $convert)?)) + } + + fn __eq__( + &self, + other: &pyo3::Bound<'_, pyo3::PyAny>, + ) -> pyo3::PyResult> { + let py = other.py(); + let right = match other.cast::() { + Ok(right) => Some(right.get().values(py)), + Err(_) => None, + }; + $crate::bindings::_helpers::planner_models::record::record_eq( + &Self::SPEC, + (&self.values(py), right.as_deref()), + py, + ) + } + + fn __hash__(&self, py: pyo3::Python<'_>) -> pyo3::PyResult { + $crate::bindings::_helpers::planner_models::record::record_hash( + &Self::SPEC, + &self.values(py), + py, + ) + } + + fn __repr__(&self, py: pyo3::Python<'_>) -> pyo3::PyResult { + $crate::bindings::_helpers::planner_models::record::record_repr( + &Self::SPEC, + &self.values(py), + ) + } + + $($($methods)*)? + } + }; +} + +fn forward_only_backfill(py: Python<'_>) -> PyResult> { + let kwargs = PyDict::new(py); + kwargs.set_item( + "action", + imported(py, "sqlbuild.compiler.planner.types", "BackfillAction")? + .getattr("FORWARD_ONLY")?, + )?; + imported(py, "sqlbuild._native", "BackfillResult")?.call((), Some(&kwargs)) +} + +fn utc_now(py: Python<'_>) -> PyResult> { + imported(py, "datetime", "datetime")?.call_method1("now", (imported(py, "datetime", "UTC")?,)) +} + +fn scenario_artifact_kind(py: Python<'_>, mut values: Vec>) -> PyResult>> { + if let Some(kind) = values.first_mut() { + *kind = imported( + py, + "sqlbuild.compiler.planner.types", + "ScenarioArtifactKind", + )? + .call1((kind.bind(py),))? + .unbind(); + } + Ok(values) +} + +fn rename_hint_message( + identical: &Bound<'_, PyAny>, + candidates: &Bound<'_, PyAny>, + added: &Bound<'_, PyAny>, +) -> PyResult { + let relation: &str = if identical.is_truthy()? { + "same expression as" + } else { + "similar to" + }; + let candidates: Vec = candidates + .try_iter()? + .map(|candidate| Ok(candidate?.str()?.to_string())) + .collect::>()?; + let origin: &str = match candidates.as_slice() { + [only] => only, + _ => "", + }; + Ok(format!( + "{relation} {}; if this is a rename, add {} (migrate_from {origin})", + candidates.join(", "), + added.str()? + )) +} + +planner_record! { + /// One native-planned SQL-test artifact and its distinct planning error messages. + NativeSqlTestArtifact, false, [ + (sql = "sql", FieldDefault::Required, true, true), + (model_names = "model_names", FieldDefault::Required, true, true), + (error_messages = "error_messages", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// One statically provable invalid fixture condition. + RelationFixtureDiagnostic, false, [ + (key = "key", FieldDefault::Required, true, true), + (message = "message", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Completed fixture SQL plus metadata reused by SQL-test validation. + RelationFixtureCompletion, false, [ + (fixture_sql_by_key = "fixture_sql_by_key", FieldDefault::Required, true, true), + (inferred_by_fixture = "inferred_by_fixture", FieldDefault::Required, true, true), + (expected_types = "expected_types", FieldDefault::Required, true, true), + (diagnostics = "diagnostics", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Known type and nullability for one relation fixture column. + FixtureColumnMetadata, false, [ + (name = "name", FieldDefault::Required, true, true), + (r#type = "type", FieldDefault::Required, true, true), + (nullable = "nullable", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Known columns and whether they form an authoritative relation shape. + FixtureRelationMetadata, false, [ + (columns = "columns", FieldDefault::Required, true, true), + (authoritative_names = "authoritative_names", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Project-wide fixture metadata shared across test and scenario planning. + RelationFixturePlanningContext, false, [ + (models_by_name = "models_by_name", FieldDefault::Required, true, true), + (relations = "relations", FieldDefault::Required, true, true), + (authoritative_columns = "authoritative_columns", FieldDefault::Required, true, true), + (expected_types = "expected_types", FieldDefault::Required, true, true), + (fixture_inferences = "fixture_inferences", FieldDefault::Class(("sqlbuild.compiler.planner.classes.fixture_column_inferences", "FixtureColumnInferences")), false, true), + ] +} + +planner_record! { + /// One selector split into graph expansion flags and core text. + SelectorExpansion, false, [ + (core = "core", FieldDefault::Required, true, true), + (upstream = "upstream", FieldDefault::Bool(false), true, true), + (downstream = "downstream", FieldDefault::Bool(false), true, true), + ] +} + +planner_record! { + /// Parsed physical name for one planner-owned scenario artifact. + ParsedScenarioArtifactName, false, [ + (hash_prefix = "hash_prefix", FieldDefault::Required, true, true), + (kind = "kind", FieldDefault::Required, true, true), + (logical_name = "logical_name", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// One upstream dependency missing from both scope and warehouse. + MissingUpstream, false, [ + (key = "key", FieldDefault::Required, true, true), + (required_by = "required_by", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Latest direct fingerprints grouped by node type. + WarehouseFingerprints, false, [ + (models = "models", FieldDefault::EmptyDict, true, true), + (functions = "functions", FieldDefault::EmptyDict, true, true), + (seeds = "seeds", FieldDefault::EmptyDict, true, true), + (python_nodes = "python_nodes", FieldDefault::EmptyDict, true, true), + (unfiltered_schemas = "unfiltered_schemas", FieldDefault::EmptyDict, false, false), + (unfiltered_database = "unfiltered_database", FieldDefault::NoneValue, false, false), + ] +} + +planner_record! { + /// Execution selection used for cursor validation and runtime producer resolution. + CursorSnapshotScope, false, [ + (model_keys = "model_keys", FieldDefault::Required, true, true), + (runtime_producer_keys = "runtime_producer_keys", FieldDefault::Required, true, true), + (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), + (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), + (cursor_overrides = "cursor_overrides", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// One model rename that downstream references may have been rewritten for. + ReferenceRename, false, [ + (new_name = "new_name", FieldDefault::Required, true, true), + (origin_name = "origin_name", FieldDefault::Required, true, true), + (recorded_at = "recorded_at", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// Frozen point-in-time picture of warehouse state for planning. + WarehouseSnapshot, false, [ + (existing_relations = "existing_relations", FieldDefault::EmptyDict, true, true), + (existing_columns = "existing_columns", FieldDefault::EmptyDict, true, true), + (fingerprints = "fingerprints", FieldDefault::Class(("sqlbuild._native", "WarehouseFingerprints")), true, true), + (cursor_snapshots = "cursor_snapshots", FieldDefault::EmptyDict, true, true), + (source_freshness_state_schemas = "source_freshness_state_schemas", FieldDefault::EmptyFrozenSet, true, true), + (column_dialect = "column_dialect", FieldDefault::NoneValue, true, true), + (renamed_models = "renamed_models", FieldDefault::EmptyFrozenSet, true, true), + (reference_renames = "reference_renames", FieldDefault::EmptyTuple, true, true), + (migration_state_schemas = "migration_state_schemas", FieldDefault::EmptyFrozenSet, true, true), + (old_name_view_state_schemas = "old_name_view_state_schemas", FieldDefault::EmptyFrozenSet, true, true), + (listed_state_schemas = "listed_state_schemas", FieldDefault::EmptyFrozenSet, true, true), + ] +} + +planner_record! { + /// Neutral node identity for shared selection-aware staleness classification. + SelectionStalenessNodeKey, false, [ + (resource_type = "resource_type", FieldDefault::Required, true, true), + (name = "name", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Neutral graph inputs for selected-model stale warning classification. + SelectionStalenessGraph, false, [ + (upstream_deps = "upstream_deps", FieldDefault::Required, true, true), + (selected_model_names = "selected_model_names", FieldDefault::Required, true, true), + (run_model_names = "run_model_names", FieldDefault::Required, true, true), + (run_seed_names = "run_seed_names", FieldDefault::Required, true, true), + (run_source_names = "run_source_names", FieldDefault::Required, true, true), + (changed_model_names = "changed_model_names", FieldDefault::Required, true, true), + (changed_seed_names = "changed_seed_names", FieldDefault::Required, true, true), + (changed_source_names = "changed_source_names", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Neutral stale warning classification for one selected model. + SelectionStalenessWarning, false, [ + (model_name = "model_name", FieldDefault::Required, true, true), + (trigger_names = "trigger_names", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Resolved backfill action from a change detection policy. + BackfillResult, false, [ + (action = "action", FieldDefault::Required, true, true), + (duration = "duration", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// Per-model output from change detection and policy resolution. + ChangeDetectionResult, false, [ + (model_name = "model_name", FieldDefault::Required, true, true), + (change_kind = "change_kind", FieldDefault::Required, true, true), + (query_changed = "query_changed", FieldDefault::Bool(false), true, true), + (config_changed = "config_changed", FieldDefault::Bool(false), true, true), + (fingerprint_metadata_json = "fingerprint_metadata_json", FieldDefault::NoneValue, true, true), + (previous_metadata_json = "previous_metadata_json", FieldDefault::NoneValue, true, true), + (fingerprint_version_hash = "fingerprint_version_hash", FieldDefault::NoneValue, true, true), + (previous_version_hash = "previous_version_hash", FieldDefault::NoneValue, true, true), + (schema_findings = "schema_findings", FieldDefault::EmptyTuple, true, true), + (backfill = "backfill", FieldDefault::Factory(forward_only_backfill), true, true), + (recorded_build_relation_missing = "recorded_build_relation_missing", FieldDefault::Bool(false), true, true), + (changed_functions = "changed_functions", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Resource selectors left after unit-test selectors moved into a test selection. + UnitTestSelectorSplit, false, [ + (select = "select", FieldDefault::Required, true, true), + (exclude = "exclude", FieldDefault::Required, true, true), + (sql_test_selection = "sql_test_selection", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Relation lookups and source columns for building one model plan entry. + ModelPlanContext, false, [ + (model_locations = "model_locations", FieldDefault::Required, true, true), + (models_by_name = "models_by_name", FieldDefault::Required, true, true), + (seed_locations = "seed_locations", FieldDefault::Required, true, true), + (function_locations = "function_locations", FieldDefault::Required, true, true), + (source_map = "source_map", FieldDefault::Required, true, true), + (source_warehouse_columns = "source_warehouse_columns", FieldDefault::Required, true, true), + (star_exclude_keyword = "star_exclude_keyword", FieldDefault::Required, true, true), + (functions_by_name = "functions_by_name", FieldDefault::EmptyDict, true, true), + (runtime_cursor_producer_names = "runtime_cursor_producer_names", FieldDefault::EmptyFrozenSet, true, true), + (future_cursor_config = "future_cursor_config", FieldDefault::NoneValue, true, true), + (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), + (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// Resolved per-model cursor start/end overrides for plan entry construction. + CursorOverridePair, false, [ + (start_cursor_override = "start_cursor_override", FieldDefault::NoneValue, true, true), + (end_cursor_override = "end_cursor_override", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// Blocked models and cursor overrides for plan entry building. + PlanEntryBuildInputs, false, [ + (source_freshness_blocked_model_names = "source_freshness_blocked_model_names", FieldDefault::EmptyFrozenSet, true, true), + (external_blocked_model_names = "external_blocked_model_names", FieldDefault::EmptyFrozenSet, true, true), + (start_cursor_override = "start_cursor_override", FieldDefault::NoneValue, true, true), + (end_cursor_override = "end_cursor_override", FieldDefault::NoneValue, true, true), + (future_cursor_config = "future_cursor_config", FieldDefault::NoneValue, true, true), + (start_cursor_config = "start_cursor_config", FieldDefault::NoneValue, true, true), + (invocation_time = "invocation_time", FieldDefault::NoneValue, true, true), + (max_microbatches = "max_microbatches", FieldDefault::NoneValue, true, true), + (max_microbatches_is_override = "max_microbatches_is_override", FieldDefault::Bool(false), true, true), + (microbatch_limit_action = "microbatch_limit_action", FieldDefault::Member(("sqlbuild.spec.contracts.types", "MicrobatchLimitAction"), "ERROR"), true, true), + ] +} + +planner_record! { + /// Per-function output from change detection. + FunctionChangeResult, false, [ + (fingerprint_sql = "fingerprint_sql", FieldDefault::Required, true, true), + (reason = "reason", FieldDefault::Member(("sqlbuild.compiler.planner.types", "PlanReason"), "NO_CHANGE"), true, true), + ] +} + +planner_record! { + /// Change detection output for selected planner resources. + PlannerChangeResults, false, [ + (models = "models", FieldDefault::Required, true, true), + (functions = "functions", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Effective model change and backfill decided from the model's own change. + ResolvedModelAction, false, [ + (change = "change", FieldDefault::Required, true, true), + (backfill = "backfill", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Per-model planning decisions keyed by model name. + PlannerResolvedActions, false, [ + (models = "models", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Current direct model version identity values by model name. + DirectModelVersionIdentities, false, [ + (function_local_hashes = "function_local_hashes", FieldDefault::Required, true, true), + (seed_version_hashes = "seed_version_hashes", FieldDefault::Required, true, true), + (seed_metadata_jsons = "seed_metadata_jsons", FieldDefault::Required, true, true), + (model_metadata_jsons = "model_metadata_jsons", FieldDefault::Required, true, true), + (model_local_hashes = "model_local_hashes", FieldDefault::Required, true, true), + (model_version_hashes = "model_version_hashes", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Model plan-entry phase output. + PlannerModelEntryResults, false, [ + (entries = "entries", FieldDefault::Required, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// One concrete schema change action to apply to a target table. + SchemaAction, false, [ + (kind = "kind", FieldDefault::Required, true, true), + (column_name = "column_name", FieldDefault::Required, true, true), + (column_type = "column_type", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// One warning or error produced during plan resolution. + PlanWarning, false, [ + (model_name = "model_name", FieldDefault::Required, true, true), + (severity = "severity", FieldDefault::Required, true, true), + (message = "message", FieldDefault::Required, true, true), + (code = "code", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// A removed column that an added column may have been renamed from. + ColumnRenameHint, false, [ + (model_name = "model_name", FieldDefault::Required, true, true), + (added_column = "added_column", FieldDefault::Required, true, true), + (candidate_columns = "candidate_columns", FieldDefault::Required, true, true), + (identical = "identical", FieldDefault::Bool(false), true, true), + ], + methods { + /// The one-line hint shown beside the added column. + #[getter] + fn message(&self, py: Python<'_>) -> PyResult { + rename_hint_message( + self.identical.bind(py), + self.candidate_columns.bind(py), + self.added_column.bind(py), + ) + } + } +} + +planner_record! { + /// One named top-level output column of a query and its canonical expression. + QueryProjection, false, [ + (name = "name", FieldDefault::Required, true, true), + (expression = "expression", FieldDefault::Required, true, true), + (references = "references", FieldDefault::Required, true, true), + (passthrough = "passthrough", FieldDefault::Bool(false), true, true), + ], + methods { + /// The case-insensitive output name. + #[getter] + fn key<'py>(&self, py: Python<'py>) -> PyResult> { + self.name.bind(py).call_method0("lower") + } + } +} + +planner_record! { + /// Canonical top-level projections and the rest of one query, ignoring formatting. + QueryShape, false, [ + (projections = "projections", FieldDefault::Required, true, true), + (body = "body", FieldDefault::Required, true, true), + (alias_clauses = "alias_clauses", FieldDefault::Text("{}"), true, true), + (positional = "positional", FieldDefault::Bool(false), true, true), + (input_relations = "input_relations", FieldDefault::NoneValue, true, true), + (local_input_columns = "local_input_columns", FieldDefault::EmptyFrozenSet, true, true), + (clause_references = "clause_references", FieldDefault::EmptyFrozenSet, true, true), + ] +} + +planner_record! { + /// Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly. + PlanOutputExtras, false, [ + (seed_version_hashes = "seed_version_hashes", FieldDefault::NoneValue, true, true), + (seed_metadata_jsons = "seed_metadata_jsons", FieldDefault::NoneValue, true, true), + (seed_plan_reasons = "seed_plan_reasons", FieldDefault::NoneValue, true, true), + (planned_sql_tests = "planned_sql_tests", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// SQL test entries planned for one selection; reusable only for the same selection. + PlannedSqlTests, false, [ + (selected_keys = "selected_keys", FieldDefault::Required, true, true), + (sql_test_selection = "sql_test_selection", FieldDefault::Class(("sqlbuild.compiler.planner.models", "SqlTestSelection")), true, true), + (entries = "entries", FieldDefault::EmptyTuple, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Native chain and assertion steps for one SQL test, with SQL when rendered. + NativeSqlTestPlan, false, [ + (chain = "chain", FieldDefault::Required, true, true), + (assertions = "assertions", FieldDefault::Required, true, true), + (model_names = "model_names", FieldDefault::Required, true, true), + (warnings = "warnings", FieldDefault::Required, true, true), + (error_messages = "error_messages", FieldDefault::Required, true, true), + (sql = "sql", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// One SQL test's plan entry and warnings, or the fixture diagnostics preventing it. + SqlTestPlanResult, false, [ + (entry = "entry", FieldDefault::Required, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + (fixture_diagnostics = "fixture_diagnostics", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Inferred graph slice and fixture boundaries for one SQL scenario. + ScenarioGraphPlan, false, [ + (key = "key", FieldDefault::Required, true, true), + (name = "name", FieldDefault::Required, true, true), + (target_model_names = "target_model_names", FieldDefault::EmptyTuple, true, true), + (assertion_target_model_names = "assertion_target_model_names", FieldDefault::EmptyTuple, true, true), + (model_names = "model_names", FieldDefault::EmptyTuple, true, true), + (source_fixture_names = "source_fixture_names", FieldDefault::EmptyTuple, true, true), + (ref_fixture_names = "ref_fixture_names", FieldDefault::EmptyTuple, true, true), + (seed_names = "seed_names", FieldDefault::EmptyTuple, true, true), + (seed_fixture_names = "seed_fixture_names", FieldDefault::EmptyTuple, true, true), + (dbt_ref_fixture_names = "dbt_ref_fixture_names", FieldDefault::EmptyTuple, true, true), + (function_deps = "function_deps", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Logical identity for one scenario-owned physical artifact. + ScenarioArtifactIdentity, false, [ + (kind = "kind", FieldDefault::Required, true, true), + (logical_name = "logical_name", FieldDefault::Required, true, true), + ], + convert = scenario_artifact_kind +} + +planner_record! { + /// Resolved physical relation name for one scenario artifact. + ScenarioArtifactName, false, [ + (identity = "identity", FieldDefault::Required, true, true), + (physical_name = "physical_name", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Provider usage metadata scoped to planned work. + PlanProviderUsage, false, [ + (provider_name = "provider_name", FieldDefault::Required, true, true), + (consumer_kind = "consumer_kind", FieldDefault::Required, true, true), + (consumer_name = "consumer_name", FieldDefault::Required, true, true), + (parameter_name = "parameter_name", FieldDefault::Required, true, true), + (annotation_class_name = "annotation_class_name", FieldDefault::NoneValue, true, true), + (annotation_module = "annotation_module", FieldDefault::NoneValue, true, true), + ] +} + +planner_record! { + /// Resolved execution environment shared by planner phases. + PlannerRuntime, false, [ + (project = "project", FieldDefault::Required, true, true), + (adapter = "adapter", FieldDefault::Required, true, true), + (connection = "connection", FieldDefault::Required, true, true), + (project_config = "project_config", FieldDefault::NoneValue, true, true), + (local_config = "local_config", FieldDefault::NoneValue, true, true), + (on_progress = "on_progress", FieldDefault::NoneValue, true, true), + (invocation_time = "invocation_time", FieldDefault::Factory(utc_now), true, true), + ] +} + +planner_record! { + /// Resolved planner scopes for one plan build. + PlannerScopeResolution, false, [ + (selected_scope = "selected_scope", FieldDefault::Required, true, true), + (stale_warning_scope = "stale_warning_scope", FieldDefault::Required, true, true), + (inspection_scope = "inspection_scope", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Warehouse snapshot and inspection relations gathered once per plan. + PlannerWarehouseState, false, [ + (snapshot = "snapshot", FieldDefault::Required, true, true), + (inspection_relations = "inspection_relations", FieldDefault::Required, true, true), + (migration_entries = "migration_entries", FieldDefault::EmptyTuple, true, true), + (migration_warnings = "migration_warnings", FieldDefault::EmptyTuple, true, true), + (column_migration_entries = "column_migration_entries", FieldDefault::EmptyTuple, true, true), + (column_rename_hints = "column_rename_hints", FieldDefault::EmptyTuple, true, true), + (old_name_view_entries = "old_name_view_entries", FieldDefault::EmptyTuple, true, true), + (old_name_views = "old_name_views", FieldDefault::EmptyTuple, true, true), + (migration_fingerprints = "migration_fingerprints", FieldDefault::Class(("sqlbuild._native", "MigrationFingerprintCache")), true, true), + ] +} + +planner_record! { + /// One destination model whose data should come from an earlier relation. + ModelMigrationRequest, false, [ + (model = "model", FieldDefault::Required, true, true), + (discovery = "discovery", FieldDefault::Required, true, true), + (raw_origin = "raw_origin", FieldDefault::NoneValue, true, true), + (origin_location = "origin_location", FieldDefault::NoneValue, true, true), + (origin_model = "origin_model", FieldDefault::NoneValue, true, true), + (force = "force", FieldDefault::Bool(false), true, true), + (identity_only = "identity_only", FieldDefault::Bool(false), true, true), + ] +} + +planner_record! { + /// One project model's explicit migrate_from, resolved to a relation. + ModelMigrationDeclaration, false, [ + (model_name = "model_name", FieldDefault::Required, true, true), + (origin_location = "origin_location", FieldDefault::Required, true, true), + (origin_model = "origin_model", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Manual and automatically discovered migration requests for one plan. + ModelMigrationDiscovery, false, [ + (requests = "requests", FieldDefault::EmptyTuple, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + (destination_fingerprints = "destination_fingerprints", FieldDefault::EmptyDict, true, true), + ] +} + +planner_record! { + /// Compatibility status plus human-readable blocking findings. + MigrationCompatibilityResult, false, [ + (status = "status", FieldDefault::Required, true, true), + (findings = "findings", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Migration decisions plus the warehouse snapshot as it will look after moves. + ModelMigrationPlanning, false, [ + (snapshot = "snapshot", FieldDefault::Required, true, true), + (entries = "entries", FieldDefault::EmptyTuple, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Old-name steps for this build plus every compatibility view still recorded. + OldNameViewPlanning, false, [ + (entries = "entries", FieldDefault::EmptyTuple, true, true), + (views = "views", FieldDefault::EmptyTuple, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Column rename decisions plus the warehouse snapshot as it will look after renames. + ColumnMigrationPlanning, false, [ + (snapshot = "snapshot", FieldDefault::Required, true, true), + (entries = "entries", FieldDefault::EmptyTuple, true, true), + (hints = "hints", FieldDefault::EmptyTuple, true, true), + (warnings = "warnings", FieldDefault::EmptyTuple, true, true), + ] +} + +planner_record! { + /// Expected version identities for inspection and stale-warning scopes. + PlannerIdentityContext, false, [ + (version_identities = "version_identities", FieldDefault::Required, true, true), + (stale_warning_identities = "stale_warning_identities", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Scope pruning outcome with pruned scopes, actions, and staleness state. + PlannerScopePruningResult, false, [ + (inspection_scope = "inspection_scope", FieldDefault::Required, true, true), + (execution_scope = "execution_scope", FieldDefault::Required, true, true), + (resolved_actions = "resolved_actions", FieldDefault::Required, true, true), + (pruned_direct_model_names = "pruned_direct_model_names", FieldDefault::Required, true, true), + (direct_identity_stale_model_names = "direct_identity_stale_model_names", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Write-hash-honest changes merged back into resolved actions. + PlannerChangeReconciliation, false, [ + (changes = "changes", FieldDefault::Required, true, true), + (resolved_actions = "resolved_actions", FieldDefault::Required, true, true), + ] +} + +planner_record! { + /// Model plan entries for one plan build. + PlannerEntryResults, false, [ + (model_entry_results = "model_entry_results", FieldDefault::Required, true, true), + ] +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs new file mode 100644 index 0000000000..53ca49e8bc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs @@ -0,0 +1,79 @@ +//! Registers the planner value records. + +use crate::bindings::_helpers::planner_models::records::{ + BackfillResult, ChangeDetectionResult, ColumnMigrationPlanning, ColumnRenameHint, + CursorOverridePair, CursorSnapshotScope, DirectModelVersionIdentities, FixtureColumnMetadata, + FixtureRelationMetadata, FunctionChangeResult, MigrationCompatibilityResult, MissingUpstream, + ModelMigrationDeclaration, ModelMigrationDiscovery, ModelMigrationPlanning, + ModelMigrationRequest, ModelPlanContext, NativeSqlTestArtifact, NativeSqlTestPlan, + OldNameViewPlanning, ParsedScenarioArtifactName, PlanEntryBuildInputs, PlanOutputExtras, + PlanProviderUsage, PlanWarning, PlannedSqlTests, PlannerChangeReconciliation, + PlannerChangeResults, PlannerEntryResults, PlannerIdentityContext, PlannerModelEntryResults, + PlannerResolvedActions, PlannerRuntime, PlannerScopePruningResult, PlannerScopeResolution, + PlannerWarehouseState, QueryProjection, QueryShape, ReferenceRename, RelationFixtureCompletion, + RelationFixtureDiagnostic, RelationFixturePlanningContext, ResolvedModelAction, + ScenarioArtifactIdentity, ScenarioArtifactName, ScenarioGraphPlan, SchemaAction, + SelectionStalenessGraph, SelectionStalenessNodeKey, SelectionStalenessWarning, + SelectorExpansion, SqlTestPlanResult, UnitTestSelectorSplit, WarehouseFingerprints, + WarehouseSnapshot, +}; +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + NativeSqlTestArtifact::register_class(module)?; + RelationFixtureDiagnostic::register_class(module)?; + RelationFixtureCompletion::register_class(module)?; + FixtureColumnMetadata::register_class(module)?; + FixtureRelationMetadata::register_class(module)?; + RelationFixturePlanningContext::register_class(module)?; + SelectorExpansion::register_class(module)?; + ParsedScenarioArtifactName::register_class(module)?; + MissingUpstream::register_class(module)?; + WarehouseFingerprints::register_class(module)?; + CursorSnapshotScope::register_class(module)?; + ReferenceRename::register_class(module)?; + WarehouseSnapshot::register_class(module)?; + SelectionStalenessNodeKey::register_class(module)?; + SelectionStalenessGraph::register_class(module)?; + SelectionStalenessWarning::register_class(module)?; + BackfillResult::register_class(module)?; + ChangeDetectionResult::register_class(module)?; + UnitTestSelectorSplit::register_class(module)?; + ModelPlanContext::register_class(module)?; + CursorOverridePair::register_class(module)?; + PlanEntryBuildInputs::register_class(module)?; + FunctionChangeResult::register_class(module)?; + PlannerChangeResults::register_class(module)?; + ResolvedModelAction::register_class(module)?; + PlannerResolvedActions::register_class(module)?; + DirectModelVersionIdentities::register_class(module)?; + PlannerModelEntryResults::register_class(module)?; + SchemaAction::register_class(module)?; + PlanWarning::register_class(module)?; + ColumnRenameHint::register_class(module)?; + QueryProjection::register_class(module)?; + QueryShape::register_class(module)?; + PlanOutputExtras::register_class(module)?; + PlannedSqlTests::register_class(module)?; + NativeSqlTestPlan::register_class(module)?; + SqlTestPlanResult::register_class(module)?; + ScenarioGraphPlan::register_class(module)?; + ScenarioArtifactIdentity::register_class(module)?; + ScenarioArtifactName::register_class(module)?; + PlanProviderUsage::register_class(module)?; + PlannerRuntime::register_class(module)?; + PlannerScopeResolution::register_class(module)?; + PlannerWarehouseState::register_class(module)?; + ModelMigrationRequest::register_class(module)?; + ModelMigrationDeclaration::register_class(module)?; + ModelMigrationDiscovery::register_class(module)?; + MigrationCompatibilityResult::register_class(module)?; + ModelMigrationPlanning::register_class(module)?; + OldNameViewPlanning::register_class(module)?; + ColumnMigrationPlanning::register_class(module)?; + PlannerIdentityContext::register_class(module)?; + PlannerScopePruningResult::register_class(module)?; + PlannerChangeReconciliation::register_class(module)?; + PlannerEntryResults::register_class(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index e2f7805a21..df59a1c784 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; -use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; +use crate::bindings::_helpers::planner_phases::objects::native; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -18,7 +18,7 @@ pub(crate) fn prune_planner_execution_scope<'py>( ) -> PyResult> { let py = scopes.py(); let inspection = scopes.getattr("inspection_scope")?; - planner_model( + native( py, "PlannerScopePruningResult", &[ @@ -64,7 +64,7 @@ pub(crate) fn reconcile_execution_changes<'py>( ("changes", honest.clone()), ], )?; - planner_model( + native( py, "PlannerChangeReconciliation", &[("changes", honest), ("resolved_actions", resolved)], @@ -158,14 +158,14 @@ pub(crate) fn build_planner_entry_results<'py>( max_override.clone() }; let limit_action = if config.is_none() { - planner_model(py, "PlanEntryBuildInputs", &[])?.getattr("microbatch_limit_action")? + native(py, "PlanEntryBuildInputs", &[])?.getattr("microbatch_limit_action")? } else { config .getattr("microbatches")? .getattr("limits")? .getattr("action")? }; - let build_inputs = planner_model( + let build_inputs = native( py, "PlanEntryBuildInputs", &[ @@ -218,7 +218,7 @@ pub(crate) fn build_planner_entry_results<'py>( ("build_inputs", build_inputs), ], )?; - planner_model( + native( py, "PlannerEntryResults", &[("model_entry_results", entries)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 4df1960b1a..1eaffa57e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,9 +1,7 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::planner_phases::objects::{ - frozenset, native, planner_model, replace, -}; +use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -41,7 +39,7 @@ pub(crate) fn build_planner_identity_context<'py>( } else { identities.clone() }; - planner_model( + native( project.py(), "PlannerIdentityContext", &[ @@ -168,7 +166,7 @@ pub(crate) fn detect_stale_warning_changes<'py>( for (name, change) in remaining_changes.downcast::()?.iter() { models.set_item(name, change)?; } - planner_model( + native( py, "PlannerChangeResults", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index ab65288a43..38700ac347 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -4,7 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::objects::{ - model_key_names, native, planner_model, replace, sorted_names, strings, text, tuple, + model_key_names, native, replace, sorted_names, strings, text, tuple, }; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -76,7 +76,7 @@ pub(crate) fn assemble_base_plan_output<'py>( let py = runtime.py(); let version = identities.getattr("version_identities")?; - let extras = planner_model( + let extras = native( py, "PlanOutputExtras", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs index 1d80a22359..5e4f8c23e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/scopes.rs @@ -1,6 +1,6 @@ //! Resolve the selected, stale-warning and inspection scopes of one plan. -use crate::bindings::_helpers::planner_phases::objects::{native, planner_model, replace, tuple}; +use crate::bindings::_helpers::planner_phases::objects::{native, replace, tuple}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; @@ -89,7 +89,7 @@ pub(crate) fn resolve_planner_scopes<'py>( ), ], )?; - planner_model( + native( py, "PlannerScopeResolution", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index c21f4b728a..632375212c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_phases::constants::NATIVE_MODULE; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; -use crate::bindings::_helpers::planner_phases::objects::{native, planner_model}; +use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; @@ -40,7 +40,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let selected_scope = scopes.getattr("selected_scope")?; let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; let config = runtime.getattr("project_config")?; - let cursor_scope = planner_model( + let cursor_scope = native( py, "CursorSnapshotScope", &[ @@ -176,7 +176,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( warnings.push(warning?); } } - planner_model( + native( py, "PlannerWarehouseState", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs index ab366575e1..ad4968c18c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_registration/registration.rs @@ -3,12 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::{ - planner_changes, planner_entries, planner_identity, planner_migrations, planner_phases, - planner_resolution, planner_scenario, planner_scope, planner_selection, planner_strategy, - planner_warehouse, + planner_changes, planner_entries, planner_identity, planner_migrations, planner_models, + planner_phases, planner_resolution, planner_scenario, planner_scope, planner_selection, + planner_strategy, planner_warehouse, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + planner_models::registration::register(module)?; planner_changes::registration::register(module)?; planner_migrations::registration::register(module)?; planner_phases::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs index 07ba97a0c5..bde0ad03bc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs @@ -7,7 +7,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -const MODELS: &str = "sqlbuild.compiler.planner.models"; const TYPES: &str = "sqlbuild.compiler.planner.types"; const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; @@ -80,7 +79,7 @@ pub(crate) fn with_cursor_safety_warnings<'py>( ] { let Some(message) = message else { continue }; all.push(call_with( - &imported(py, MODELS, "PlanWarning")?, + &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", model_name.clone()), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index f6ec9df817..600cfd0673 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -9,7 +9,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; -const MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const HOOK_MODULE: &str = "sqlbuild.compiler.discovery.models"; const ERROR_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -348,7 +347,10 @@ fn resolve_function_sql<'py>( } let resolution = Resolution { adapter: args.value("adapter")?, - context: call_with(&imported(py, MODELS_MODULE, "ModelPlanContext")?, &context)?, + context: call_with( + &imported(py, "sqlbuild._native", "ModelPlanContext")?, + &context, + )?, external: py.None().into_bound(py), }; let function = args.value("function")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs index 0b77cfebcd..4aa6985c0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/artifacts.rs @@ -173,7 +173,7 @@ pub(crate) fn parse_scenario_artifact_physical_name<'py>( return Ok(py.None().into_bound(py)); }; call_with( - &imported(py, MODELS, "ParsedScenarioArtifactName")?, + &imported(py, "sqlbuild._native", "ParsedScenarioArtifactName")?, &[ ("hash_prefix", string(py, hash)), ("kind", string(py, kind)), @@ -249,7 +249,7 @@ pub(crate) fn build_scenario_relation_map<'py>( } seen.set_item(normalized, &artifact)?; artifacts.push(call_with( - &imported(py, MODELS, "ScenarioArtifactName")?, + &imported(py, "sqlbuild._native", "ScenarioArtifactName")?, &[ ("identity", artifact), ("physical_name", string(py, &physical)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 9448426a6c..e03de51799 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -6,7 +6,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; -use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, string}; +use crate::bindings::_helpers::planner_scenario::values::{TYPES, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -59,7 +59,7 @@ pub(crate) fn build_cli_scenario_plan<'py>( ] { for name in graph.getattr(field)?.try_iter()? { identities.push(call_with( - &imported(py, MODELS, "ScenarioArtifactIdentity")?, + &imported(py, "sqlbuild._native", "ScenarioArtifactIdentity")?, &[ ( "kind", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index cf00568184..f87b466239 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -54,7 +54,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( let snapshot = if snapshot.is_truthy()? { snapshot } else { - imported(py, MODELS, "WarehouseSnapshot")?.call0()? + imported(py, "sqlbuild._native", "WarehouseSnapshot")?.call0()? }; let columns = args.value("source_warehouse_columns")?; let columns = if columns.is_truthy()? { @@ -145,7 +145,7 @@ pub(crate) fn build_scenario_execution_plan<'py>( ("full_refresh", PyBool::new(py, true).to_owned().into_any()), ( "cursor_overrides", - imported(py, MODELS, "CursorOverridePair")?.call0()?, + imported(py, "sqlbuild._native", "CursorOverridePair")?.call0()?, ), ], )?; @@ -255,7 +255,7 @@ fn model_context<'py>( let (models, locations, columns) = inputs; let py = project.py(); call_with( - &imported(py, MODELS, "ModelPlanContext")?, + &imported(py, "sqlbuild._native", "ModelPlanContext")?, &[ ( "model_locations", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index d9f2d68a51..9b1d14c7a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; -use crate::bindings::_helpers::planner_scenario::values::{MODELS, TYPES, string}; +use crate::bindings::_helpers::planner_scenario::values::{TYPES, string}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyTuple; use sqlbuild_analysis::planner_scenario::main::graph::graph; @@ -55,7 +55,7 @@ pub(crate) fn plan_scenario_graph<'py>( input.targets.sort(); input.assertions.sort(); let plan = call_with( - &imported(py, MODELS, "ScenarioGraphPlan")?, + &imported(py, "sqlbuild._native", "ScenarioGraphPlan")?, &[ ("key", scenario.getattr("key")?), ("name", scenario.getattr("name")?), @@ -104,7 +104,10 @@ pub(crate) fn plan_scenario_graph<'py>( if coded { values.push(("code", string(py, SCENARIO_PLAN_GRAPH_VALIDATION))); } - warnings.push(call_with(&imported(py, MODELS, "PlanWarning")?, &values)?); + warnings.push(call_with( + &imported(py, "sqlbuild._native", "PlanWarning")?, + &values, + )?); } Ok((plan, PyTuple::new(py, warnings)?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs index def3598381..eb4b6d65ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/model_selectors.rs @@ -15,7 +15,6 @@ use sqlbuild_analysis::planner_selection::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_selection::carrier::Carrier; -const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; fn shared_input_error(carrier: &Carrier<'_>, message: &str) -> PyErr { @@ -35,7 +34,7 @@ fn split_selector_expansion<'py>(raw: &Bound<'py, PyString>) -> PyResult( ], )?; call_with( - &imported(py, PLANNER_MODELS_MODULE, "UnitTestSelectorSplit")?, + &imported(py, "sqlbuild._native", "UnitTestSelectorSplit")?, &[ ("select", restored(&carrier, &split.select)?.into_any()), ("exclude", restored(&carrier, &split.exclude)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs index 0bc5231dda..f22fa7a1ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_values.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::planner_fixtures::models::FixtureColumn; -pub(crate) const MODELS: &str = "sqlbuild.compiler.planner.models"; +pub(crate) const MODELS: &str = "sqlbuild._native"; pub(crate) fn folded(value: &Bound<'_, PyAny>) -> PyResult { value.call_method0("casefold")?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs index a5a0018752..bb3633947b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -14,9 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::compiled_project::records::replace; use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; use crate::bindings::_helpers::planner_changes::results::planner_enum; -use crate::bindings::_helpers::planner_strategy::constants::{ - PLAN_MODELS_MODULE, PLANNER_TYPES_MODULE, -}; +use crate::bindings::_helpers::planner_strategy::constants::PLANNER_TYPES_MODULE; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, CONTRACT_TYPES_MODULE, capped_microbatch_bounds, count_microbatches, }; @@ -159,7 +157,7 @@ fn limited<'py>( )); } let plan_warning: Bound<'py, PyAny> = call_with( - &imported(py, PLAN_MODELS_MODULE, "PlanWarning")?, + &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", name), ("severity", planner_enum(py, "WarningSeverity", "warning")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index 26418b9f29..1c24ee9c0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -15,7 +15,7 @@ use sqlbuild_analysis::planner_strategy::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_changes::results::planner_enum; use crate::bindings::_helpers::planner_strategy::values::{ PLANNER_EXCEPTIONS_MODULE, change_facts, config_value, finding_rows, input_error, materialization_value, model_materialization, optional_string, plan_warning, text, @@ -112,7 +112,7 @@ fn resolve_schema_actions<'py>( on_schema_change: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = schema_findings.py(); - let class: Bound<'py, PyAny> = imported(py, MODELS_MODULE, "SchemaAction")?; + let class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "SchemaAction")?; let policy: Option = schema_policy(on_schema_change)?; let mut actions: Vec> = Vec::new(); for row in schema_actions(&finding_rows(schema_findings)?, policy.as_deref()) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs index 5101af6fd6..7754f870c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -13,7 +13,7 @@ use sqlbuild_analysis::planner_strategy::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_changes::results::{MODELS_MODULE, planner_enum}; +use crate::bindings::_helpers::planner_changes::results::planner_enum; pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -137,7 +137,7 @@ pub(crate) fn plan_warning<'py>( WarningSeverity::Info => "info", }; call_with( - &imported(py, MODELS_MODULE, "PlanWarning")?, + &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", model_name.clone()), ("severity", planner_enum(py, "WarningSeverity", severity)?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs index fbfeebf1e9..f46a99e78a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs @@ -4,7 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::{ - by_name, frozenset, model_key, native, planner_model, text, tuple, + by_name, frozenset, model_key, native, text, tuple, }; use crate::bindings::_helpers::planner_warehouse::constants::{ ACTIVE_CATALOG, FINGERPRINT_TABLE_NAME, FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, @@ -482,7 +482,7 @@ fn gather_fingerprints<'py>( ) -> PyResult> { let py = request.project.py(); let Some(schemas) = schemas else { - return planner_model(py, "WarehouseFingerprints", &[]); + return native(py, "WarehouseFingerprints", &[]); }; let node_names: Option> = if relevant.is_none() { None @@ -599,7 +599,7 @@ fn gather_fingerprints<'py>( } } } - planner_model( + native( py, "WarehouseFingerprints", &[ @@ -632,7 +632,7 @@ pub(crate) fn warehouse_snapshot<'py>( let schemas = target_schemas(project, &relevant)?; let metadata_names = metadata_name_filter(project, &relevant)?; if schemas.is_empty() && metadata_names.is_none() { - return planner_model(py, "WarehouseSnapshot", &[]); + return native(py, "WarehouseSnapshot", &[]); } let query_schemas = (!schemas.is_empty()).then_some(&schemas); prefetch_schemas(request, &database, &schemas, &relevant)?; @@ -686,7 +686,7 @@ pub(crate) fn warehouse_snapshot<'py>( names_set(py, &lowered)? } }; - planner_model( + native( py, "WarehouseSnapshot", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 703a42e003..71808d3954 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -6,7 +6,7 @@ use crate::bindings::_helpers::freshness::planning::{ build_direct_source_freshness_planning_result, build_direct_source_freshness_propagation_result, }; use crate::bindings::_helpers::planner_phases::objects::{ - model_key_names, planner_model, replace, text, tuple, + model_key_names, native, replace, text, tuple, }; use crate::bindings::_helpers::planner_warehouse::constants::RUN_SCOPE; use crate::bindings::_helpers::planner_warehouse::snapshot::{destination_nodes, lower}; @@ -45,7 +45,7 @@ pub(crate) fn source_freshness_unknown_warnings<'py>( let mut warnings: Vec<_> = Vec::new(); for (reason, names) in reasons { if let Some(message) = freshness_unknown_message(&reason, &names) { - warnings.push(planner_model( + warnings.push(native( py, "PlanWarning", &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index 6ecfaaac8d..0075483932 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -6,7 +6,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::sql_test_glue::constants::{NATIVE_PLANNING, PLAN_MODELS}; +use crate::bindings::_helpers::sql_test_glue::constants::NATIVE_PLANNING; #[pyfunction(name = "plan_sql_tests")] #[pyo3(signature = (**kwargs))] @@ -77,7 +77,7 @@ pub(crate) fn assemble_sql_test_plans<'py>( let model = model?; models.set_item(model.getattr("name")?, model)?; } - let result_class = imported(py, PLAN_MODELS, "SqlTestPlanResult")?; + let result_class = imported(py, "sqlbuild._native", "SqlTestPlanResult")?; let mut results: Vec> = Vec::new(); for (index, test) in tests.iter().enumerate() { if let Some(diagnostic) = diagnostics.get(&index) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index deb6b75c96..da46442b4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -3,7 +3,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -use crate::bindings::_helpers::sql_test_glue::constants::PLAN_MODELS; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -184,7 +183,7 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult(row: &Bound<'py, PyAny>) -> PyResult> for value in row.get_item(4)?.try_iter()? { let value = value?; warnings.push(call_with( - &imported(py, PLAN_MODELS, "PlanWarning")?, + &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", value.get_item(0)?), ("severity", severity.call1((value.get_item(1)?,))?), @@ -71,7 +71,7 @@ pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> )?); } call_with( - &imported(py, PLAN_MODELS, "NativeSqlTestPlan")?, + &imported(py, "sqlbuild._native", "NativeSqlTestPlan")?, &[ ("chain", PyTuple::new(py, chain)?.into_any()), ("assertions", PyTuple::new(py, assertions)?.into_any()), diff --git a/python-boundary.toml b/python-boundary.toml index 8623b5cd59..0b52452126 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 11981 +remaining_lines = 11383 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 7651f99654..0eeed686d4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -10,6 +10,7 @@ from decimal import Decimal from pathlib import Path, PurePath from threading import Event from typing import Any, Literal, Protocol, TextIO, TypedDict +from typing import Any, ClassVar, Literal, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -189,7 +190,9 @@ from sqlbuild.compiler.migrations.models import ( OldNameViewHistory, ) from sqlbuild.compiler.migrations.types import ( + MigrationCompatibility, MigrationDecision, + MigrationDiscovery, MigrationPromotion, OldNameViewEventType, ) @@ -205,101 +208,61 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) +from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, - BackfillResult, - ChangeDetectionResult, + ChainStep, CloneSourcePlanEntry, ColumnMigrationPlanEntry, - ColumnMigrationPlanning, - ColumnRenameHint, CursorBounds, CursorInputEvidence, CursorInputRelation, - CursorOverridePair, CursorOverrides, - CursorSnapshotScope, DeferralInputs, - DirectModelVersionIdentities, EmptyCursorInputDecision, - FixtureRelationMetadata, FunctionPlanEntry, FutureCursorSafetyEvidence, MaximumStartPolicyInputs, MaximumStartSafetyEvidence, - MigrationCompatibilityResult, - MissingUpstream, ModelCursorSnapshot, - ModelMigrationDeclaration, - ModelMigrationDiscovery, ModelMigrationPlanEntry, - ModelMigrationPlanning, - ModelMigrationRequest, - ModelPlanContext, ModelPlanEntry, - NativeSqlTestArtifact, - NativeSqlTestPlan, OldNameView, OldNameViewPlanEntry, - OldNameViewPlanning, - ParsedScenarioArtifactName, ParsedSelector, PathSelector, - PlanEntryBuildInputs, - PlannedSqlTests, - PlannerChangeReconciliation, - PlannerChangeResults, - PlannerEntryResults, - PlannerIdentityContext, - PlannerModelEntryResults, PlannerOverrides, PlannerPolicies, PlannerRelationsContext, - PlannerResolvedActions, - PlannerRuntime, PlannerScope, - PlannerScopePruningResult, - PlannerScopeResolution, PlannerSelection, - PlannerWarehouseState, PlanOutput, - PlanOutputExtras, - PlanWarning, - QueryShape, - ReferenceRename, - RelationFixtureCompletion, - RelationFixtureDiagnostic, - RelationFixturePlanningContext, RetentionPlanEntry, - ScenarioArtifactIdentity, ScenarioExecutionPlan, ScenarioFixturePlan, - ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, - SchemaAction, SchemaFinding, SeedPlanEntry, - SelectionStalenessGraph, - SelectionStalenessWarning, - SelectorExpansion, SourceLoadPlanEntry, + SqlTestAssertionStep, SqlTestPlanEntry, - SqlTestPlanResult, SqlTestSelection, TableTypePlanEntry, - UnitTestSelectorSplit, - WarehouseFingerprints, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( + BackfillAction, + ChangeKind, CursorGrain, CursorType, MaterializationType, OnSchemaChange, PlanAction, PlanReason, + ScenarioArtifactKind, + SchemaActionKind, + WarningSeverity, ) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, @@ -762,6 +725,945 @@ type _GraphIndexes = tuple[ type _SelectorFailure = tuple[str, str, str | None] type _SelectorKeys = tuple[list[_GraphKey] | None, _SelectorFailure | None] +class NativeSqlTestArtifact: + """One native-planned SQL-test artifact and its distinct planning error messages.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + sql: str + model_names: tuple[str, ...] + error_messages: tuple[str, ...] + def __init__( + self, sql: str, model_names: tuple[str, ...], error_messages: tuple[str, ...] = ... + ) -> None: ... + def __replace__(self, **changes: Any) -> NativeSqlTestArtifact: ... + +class RelationFixtureDiagnostic: + """One statically provable invalid fixture condition.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + key: FixtureKey + message: str + def __init__(self, key: FixtureKey, message: str) -> None: ... + def __replace__(self, **changes: Any) -> RelationFixtureDiagnostic: ... + +class RelationFixtureCompletion: + """Completed fixture SQL plus metadata reused by SQL-test validation.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + fixture_sql_by_key: dict[FixtureKey, str] + inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] + expected_types: Mapping[FixtureKey, dict[str, str]] + diagnostics: tuple[RelationFixtureDiagnostic, ...] + def __init__( + self, + fixture_sql_by_key: dict[FixtureKey, str], + inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]], + expected_types: Mapping[FixtureKey, dict[str, str]], + diagnostics: tuple[RelationFixtureDiagnostic, ...], + ) -> None: ... + def __replace__(self, **changes: Any) -> RelationFixtureCompletion: ... + +class FixtureColumnMetadata: + """Known type and nullability for one relation fixture column.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + name: str + type: str | None + nullable: bool | None + def __init__(self, name: str, type: str | None, nullable: bool | None) -> None: ... + def __replace__(self, **changes: Any) -> FixtureColumnMetadata: ... + +class FixtureRelationMetadata: + """Known columns and whether they form an authoritative relation shape.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + columns: tuple[FixtureColumnMetadata, ...] + authoritative_names: bool + def __init__( + self, columns: tuple[FixtureColumnMetadata, ...], authoritative_names: bool + ) -> None: ... + def __replace__(self, **changes: Any) -> FixtureRelationMetadata: ... + +class RelationFixturePlanningContext: + """Project-wide fixture metadata shared across test and scenario planning.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + models_by_name: dict[str, CompiledModelView] + relations: dict[FixtureKey, FixtureRelationMetadata] + authoritative_columns: dict[FixtureKey, frozenset[str]] + expected_types: dict[FixtureKey, dict[str, str]] + fixture_inferences: FixtureColumnInferences + def __init__( + self, + models_by_name: dict[str, CompiledModelView], + relations: dict[FixtureKey, FixtureRelationMetadata], + authoritative_columns: dict[FixtureKey, frozenset[str]], + expected_types: dict[FixtureKey, dict[str, str]], + fixture_inferences: FixtureColumnInferences = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> RelationFixturePlanningContext: ... + +class SelectorExpansion: + """One selector split into graph expansion flags and core text.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + core: str + upstream: bool + downstream: bool + def __init__(self, core: str, upstream: bool = ..., downstream: bool = ...) -> None: ... + def __replace__(self, **changes: Any) -> SelectorExpansion: ... + +class ParsedScenarioArtifactName: + """Parsed physical name for one planner-owned scenario artifact.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + hash_prefix: str + kind: str + logical_name: str + def __init__(self, hash_prefix: str, kind: str, logical_name: str) -> None: ... + def __replace__(self, **changes: Any) -> ParsedScenarioArtifactName: ... + +class MissingUpstream: + """One upstream dependency missing from both scope and warehouse.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + key: CompiledObjectKey + required_by: tuple[CompiledObjectKey, ...] + def __init__( + self, key: CompiledObjectKey, required_by: tuple[CompiledObjectKey, ...] + ) -> None: ... + def __replace__(self, **changes: Any) -> MissingUpstream: ... + +class WarehouseFingerprints: + """Latest direct fingerprints grouped by node type.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + models: dict[str, Fingerprint] + functions: dict[str, Fingerprint] + seeds: dict[str, Fingerprint] + python_nodes: dict[tuple[str, str], Fingerprint] + unfiltered_schemas: dict[str, tuple[Fingerprint, ...]] + unfiltered_database: str | None + def __init__( + self, + models: dict[str, Fingerprint] = ..., + functions: dict[str, Fingerprint] = ..., + seeds: dict[str, Fingerprint] = ..., + python_nodes: dict[tuple[str, str], Fingerprint] = ..., + unfiltered_schemas: dict[str, tuple[Fingerprint, ...]] = ..., + unfiltered_database: str | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> WarehouseFingerprints: ... + +class CursorSnapshotScope: + """Execution selection used for cursor validation and runtime producer resolution.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_keys: frozenset[CompiledObjectKey] + runtime_producer_keys: frozenset[CompiledObjectKey] + invocation_time: datetime | None + start_cursor_config: StartCursorsConfig | None + cursor_overrides: CursorOverrides | None + def __init__( + self, + model_keys: frozenset[CompiledObjectKey], + runtime_producer_keys: frozenset[CompiledObjectKey], + invocation_time: datetime | None = ..., + start_cursor_config: StartCursorsConfig | None = ..., + cursor_overrides: CursorOverrides | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> CursorSnapshotScope: ... + +class ReferenceRename: + """One model rename that downstream references may have been rewritten for.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + new_name: str + origin_name: str + recorded_at: datetime | None + def __init__( + self, new_name: str, origin_name: str, recorded_at: datetime | None = ... + ) -> None: ... + def __replace__(self, **changes: Any) -> ReferenceRename: ... + +class WarehouseSnapshot: + """Frozen point-in-time picture of warehouse state for planning.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + existing_relations: dict[str, RelationInfo] + existing_columns: dict[str, tuple[ColumnInfo, ...]] + fingerprints: WarehouseFingerprints + cursor_snapshots: dict[str, ModelCursorSnapshot] + source_freshness_state_schemas: frozenset[str] + column_dialect: str | None + renamed_models: frozenset[str] + reference_renames: tuple[ReferenceRename, ...] + migration_state_schemas: frozenset[str] + old_name_view_state_schemas: frozenset[str] + listed_state_schemas: frozenset[str] | None + def __init__( + self, + existing_relations: dict[str, RelationInfo] = ..., + existing_columns: dict[str, tuple[ColumnInfo, ...]] = ..., + fingerprints: WarehouseFingerprints = ..., + cursor_snapshots: dict[str, ModelCursorSnapshot] = ..., + source_freshness_state_schemas: frozenset[str] = ..., + column_dialect: str | None = ..., + renamed_models: frozenset[str] = ..., + reference_renames: tuple[ReferenceRename, ...] = ..., + migration_state_schemas: frozenset[str] = ..., + old_name_view_state_schemas: frozenset[str] = ..., + listed_state_schemas: frozenset[str] | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> WarehouseSnapshot: ... + +class SelectionStalenessNodeKey: + """Neutral node identity for shared selection-aware staleness classification.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + resource_type: str + name: str + def __init__(self, resource_type: str, name: str) -> None: ... + def __replace__(self, **changes: Any) -> SelectionStalenessNodeKey: ... + +class SelectionStalenessGraph: + """Neutral graph inputs for selected-model stale warning classification.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + upstream_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]] + selected_model_names: frozenset[str] + run_model_names: frozenset[str] + run_seed_names: frozenset[str] + run_source_names: frozenset[str] + changed_model_names: frozenset[str] + changed_seed_names: frozenset[str] + changed_source_names: frozenset[str] + def __init__( + self, + upstream_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]], + selected_model_names: frozenset[str], + run_model_names: frozenset[str], + run_seed_names: frozenset[str], + run_source_names: frozenset[str], + changed_model_names: frozenset[str], + changed_seed_names: frozenset[str], + changed_source_names: frozenset[str], + ) -> None: ... + def __replace__(self, **changes: Any) -> SelectionStalenessGraph: ... + +class SelectionStalenessWarning: + """Neutral stale warning classification for one selected model.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_name: str + trigger_names: tuple[str, ...] + def __init__(self, model_name: str, trigger_names: tuple[str, ...]) -> None: ... + def __replace__(self, **changes: Any) -> SelectionStalenessWarning: ... + +class BackfillResult: + """Resolved backfill action from a change detection policy.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + action: BackfillAction + duration: str | None + def __init__(self, action: BackfillAction, duration: str | None = ...) -> None: ... + def __replace__(self, **changes: Any) -> BackfillResult: ... + +class ChangeDetectionResult: + """Per-model output from change detection and policy resolution.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_name: str + change_kind: ChangeKind + query_changed: bool + config_changed: bool + fingerprint_metadata_json: str | None + previous_metadata_json: str | None + fingerprint_version_hash: str | None + previous_version_hash: str | None + schema_findings: tuple[SchemaFinding, ...] + backfill: BackfillResult + recorded_build_relation_missing: bool + changed_functions: tuple[str, ...] + def __init__( + self, + model_name: str, + change_kind: ChangeKind, + query_changed: bool = ..., + config_changed: bool = ..., + fingerprint_metadata_json: str | None = ..., + previous_metadata_json: str | None = ..., + fingerprint_version_hash: str | None = ..., + previous_version_hash: str | None = ..., + schema_findings: tuple[SchemaFinding, ...] = ..., + backfill: BackfillResult = ..., + recorded_build_relation_missing: bool = ..., + changed_functions: tuple[str, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ChangeDetectionResult: ... + +class UnitTestSelectorSplit: + """Resource selectors left after unit-test selectors moved into a test selection.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + select: tuple[str, ...] + exclude: tuple[str, ...] + sql_test_selection: SqlTestSelection + def __init__( + self, + select: tuple[str, ...], + exclude: tuple[str, ...], + sql_test_selection: SqlTestSelection, + ) -> None: ... + def __replace__(self, **changes: Any) -> UnitTestSelectorSplit: ... + +class ModelPlanContext: + """Relation lookups and source columns for building one model plan entry.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_locations: dict[str, CompiledRelationLocation] + models_by_name: dict[str, CompiledModelView] + seed_locations: dict[str, CompiledRelationLocation] + function_locations: dict[str, CompiledRelationLocation] + source_map: dict[str, SourceEntry] + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] + star_exclude_keyword: str + functions_by_name: dict[str, CompiledFunctionView] + runtime_cursor_producer_names: frozenset[str] + future_cursor_config: FutureCursorsConfig | None + start_cursor_config: StartCursorsConfig | None + invocation_time: datetime | None + def __init__( + self, + model_locations: dict[str, CompiledRelationLocation], + models_by_name: dict[str, CompiledModelView], + seed_locations: dict[str, CompiledRelationLocation], + function_locations: dict[str, CompiledRelationLocation], + source_map: dict[str, SourceEntry], + source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], + star_exclude_keyword: str, + functions_by_name: dict[str, CompiledFunctionView] = ..., + runtime_cursor_producer_names: frozenset[str] = ..., + future_cursor_config: FutureCursorsConfig | None = ..., + start_cursor_config: StartCursorsConfig | None = ..., + invocation_time: datetime | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ModelPlanContext: ... + +class CursorOverridePair: + """Resolved per-model cursor start/end overrides for plan entry construction.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + start_cursor_override: str | None + end_cursor_override: str | None + def __init__( + self, start_cursor_override: str | None = ..., end_cursor_override: str | None = ... + ) -> None: ... + def __replace__(self, **changes: Any) -> CursorOverridePair: ... + +class PlanEntryBuildInputs: + """Blocked models and cursor overrides for plan entry building.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + source_freshness_blocked_model_names: frozenset[str] + external_blocked_model_names: frozenset[str] + start_cursor_override: str | None + end_cursor_override: str | None + future_cursor_config: FutureCursorsConfig | None + start_cursor_config: StartCursorsConfig | None + invocation_time: datetime | None + max_microbatches: int | None + max_microbatches_is_override: bool + microbatch_limit_action: MicrobatchLimitAction + def __init__( + self, + source_freshness_blocked_model_names: frozenset[str] = ..., + external_blocked_model_names: frozenset[str] = ..., + start_cursor_override: str | None = ..., + end_cursor_override: str | None = ..., + future_cursor_config: FutureCursorsConfig | None = ..., + start_cursor_config: StartCursorsConfig | None = ..., + invocation_time: datetime | None = ..., + max_microbatches: int | None = ..., + max_microbatches_is_override: bool = ..., + microbatch_limit_action: MicrobatchLimitAction = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlanEntryBuildInputs: ... + +class FunctionChangeResult: + """Per-function output from change detection.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + fingerprint_sql: str + reason: PlanReason + def __init__(self, fingerprint_sql: str, reason: PlanReason = ...) -> None: ... + def __replace__(self, **changes: Any) -> FunctionChangeResult: ... + +class PlannerChangeResults: + """Change detection output for selected planner resources.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + models: dict[str, ChangeDetectionResult] + functions: dict[str, FunctionChangeResult] + def __init__( + self, models: dict[str, ChangeDetectionResult], functions: dict[str, FunctionChangeResult] + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerChangeResults: ... + +class ResolvedModelAction: + """Effective model change and backfill decided from the model's own change.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + change: ChangeDetectionResult + backfill: BackfillResult + def __init__(self, change: ChangeDetectionResult, backfill: BackfillResult) -> None: ... + def __replace__(self, **changes: Any) -> ResolvedModelAction: ... + +class PlannerResolvedActions: + """Per-model planning decisions keyed by model name.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + models: dict[str, ResolvedModelAction] + def __init__(self, models: dict[str, ResolvedModelAction]) -> None: ... + def __replace__(self, **changes: Any) -> PlannerResolvedActions: ... + +class DirectModelVersionIdentities: + """Current direct model version identity values by model name.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + function_local_hashes: dict[str, str] + seed_version_hashes: dict[str, str] + seed_metadata_jsons: dict[str, str] + model_metadata_jsons: dict[str, str] + model_local_hashes: dict[str, str] + model_version_hashes: dict[str, str] + def __init__( + self, + function_local_hashes: dict[str, str], + seed_version_hashes: dict[str, str], + seed_metadata_jsons: dict[str, str], + model_metadata_jsons: dict[str, str], + model_local_hashes: dict[str, str], + model_version_hashes: dict[str, str], + ) -> None: ... + def __replace__(self, **changes: Any) -> DirectModelVersionIdentities: ... + +class PlannerModelEntryResults: + """Model plan-entry phase output.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + entries: tuple[ModelPlanEntry, ...] + warnings: tuple[PlanWarning, ...] + def __init__( + self, entries: tuple[ModelPlanEntry, ...], warnings: tuple[PlanWarning, ...] = ... + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerModelEntryResults: ... + +class SchemaAction: + """One concrete schema change action to apply to a target table.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + kind: SchemaActionKind + column_name: str + column_type: str | None + def __init__( + self, kind: SchemaActionKind, column_name: str, column_type: str | None = ... + ) -> None: ... + def __replace__(self, **changes: Any) -> SchemaAction: ... + +class PlanWarning: + """One warning or error produced during plan resolution.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_name: str | None + severity: WarningSeverity + message: str + code: str | None + def __init__( + self, + model_name: str | None, + severity: WarningSeverity, + message: str, + code: str | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlanWarning: ... + +class ColumnRenameHint: + """A removed column that an added column may have been renamed from.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_name: str + added_column: str + candidate_columns: tuple[str, ...] + identical: bool + def __init__( + self, + model_name: str, + added_column: str, + candidate_columns: tuple[str, ...], + identical: bool = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ColumnRenameHint: ... + @property + def message(self) -> str: + """The one-line hint shown beside the added column.""" + +class QueryProjection: + """One named top-level output column of a query and its canonical expression.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + name: str + expression: str + references: frozenset[str] + passthrough: bool + def __init__( + self, name: str, expression: str, references: frozenset[str], passthrough: bool = ... + ) -> None: ... + def __replace__(self, **changes: Any) -> QueryProjection: ... + @property + def key(self) -> str: + """The case-insensitive output name.""" + +class QueryShape: + """Canonical top-level projections and the rest of one query, ignoring formatting.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + projections: tuple[QueryProjection, ...] + body: str + alias_clauses: str + positional: bool + input_relations: tuple[tuple[str, str], ...] | None + local_input_columns: frozenset[str] + clause_references: frozenset[str] + def __init__( + self, + projections: tuple[QueryProjection, ...], + body: str, + alias_clauses: str = ..., + positional: bool = ..., + input_relations: tuple[tuple[str, str], ...] | None = ..., + local_input_columns: frozenset[str] = ..., + clause_references: frozenset[str] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> QueryShape: ... + +class PlanOutputExtras: + """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + seed_version_hashes: dict[str, str] | None + seed_metadata_jsons: dict[str, str] | None + seed_plan_reasons: dict[str, PlanReason] | None + planned_sql_tests: PlannedSqlTests | None + def __init__( + self, + seed_version_hashes: dict[str, str] | None = ..., + seed_metadata_jsons: dict[str, str] | None = ..., + seed_plan_reasons: dict[str, PlanReason] | None = ..., + planned_sql_tests: PlannedSqlTests | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlanOutputExtras: ... + +class PlannedSqlTests: + """SQL test entries planned for one selection; reusable only for the same selection.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + selected_keys: frozenset[CompiledObjectKey] + sql_test_selection: SqlTestSelection + entries: tuple[SqlTestPlanEntry, ...] + warnings: tuple[PlanWarning, ...] + def __init__( + self, + selected_keys: frozenset[CompiledObjectKey], + sql_test_selection: SqlTestSelection = ..., + entries: tuple[SqlTestPlanEntry, ...] = ..., + warnings: tuple[PlanWarning, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannedSqlTests: ... + +class NativeSqlTestPlan: + """Native chain and assertion steps for one SQL test, with SQL when rendered.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + chain: tuple[ChainStep, ...] + assertions: tuple[SqlTestAssertionStep, ...] + model_names: tuple[str, ...] + warnings: tuple[PlanWarning, ...] + error_messages: tuple[str, ...] + sql: str | None + def __init__( + self, + chain: tuple[ChainStep, ...], + assertions: tuple[SqlTestAssertionStep, ...], + model_names: tuple[str, ...], + warnings: tuple[PlanWarning, ...], + error_messages: tuple[str, ...], + sql: str | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> NativeSqlTestPlan: ... + +class SqlTestPlanResult: + """One SQL test's plan entry and warnings, or the fixture diagnostics preventing it.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + entry: SqlTestPlanEntry | None + warnings: tuple[PlanWarning, ...] + fixture_diagnostics: tuple[str, ...] + def __init__( + self, + entry: SqlTestPlanEntry | None, + warnings: tuple[PlanWarning, ...] = ..., + fixture_diagnostics: tuple[str, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> SqlTestPlanResult: ... + +class ScenarioGraphPlan: + """Inferred graph slice and fixture boundaries for one SQL scenario.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + key: CompiledObjectKey + name: str + target_model_names: tuple[str, ...] + assertion_target_model_names: tuple[str, ...] + model_names: tuple[str, ...] + source_fixture_names: tuple[str, ...] + ref_fixture_names: tuple[str, ...] + seed_names: tuple[str, ...] + seed_fixture_names: tuple[str, ...] + dbt_ref_fixture_names: tuple[str, ...] + function_deps: tuple[CompiledObjectKey, ...] + def __init__( + self, + key: CompiledObjectKey, + name: str, + target_model_names: tuple[str, ...] = ..., + assertion_target_model_names: tuple[str, ...] = ..., + model_names: tuple[str, ...] = ..., + source_fixture_names: tuple[str, ...] = ..., + ref_fixture_names: tuple[str, ...] = ..., + seed_names: tuple[str, ...] = ..., + seed_fixture_names: tuple[str, ...] = ..., + dbt_ref_fixture_names: tuple[str, ...] = ..., + function_deps: tuple[CompiledObjectKey, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ScenarioGraphPlan: ... + +class ScenarioArtifactIdentity: + """Logical identity for one scenario-owned physical artifact.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + kind: ScenarioArtifactKind | str + logical_name: str + def __init__(self, kind: ScenarioArtifactKind | str, logical_name: str) -> None: ... + def __replace__(self, **changes: Any) -> ScenarioArtifactIdentity: ... + +class ScenarioArtifactName: + """Resolved physical relation name for one scenario artifact.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + identity: ScenarioArtifactIdentity + physical_name: str + def __init__(self, identity: ScenarioArtifactIdentity, physical_name: str) -> None: ... + def __replace__(self, **changes: Any) -> ScenarioArtifactName: ... + +class PlanProviderUsage: + """Provider usage metadata scoped to planned work.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + provider_name: str + consumer_kind: str + consumer_name: str + parameter_name: str + annotation_class_name: str | None + annotation_module: str | None + def __init__( + self, + provider_name: str, + consumer_kind: str, + consumer_name: str, + parameter_name: str, + annotation_class_name: str | None = ..., + annotation_module: str | None = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlanProviderUsage: ... + +class PlannerRuntime: + """Resolved execution environment shared by planner phases.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + project: CompiledProjectView + adapter: BaseAdapter + connection: Any + project_config: ProjectConfig | None + local_config: LocalConfig | None + on_progress: Callable[[str], None] | None + invocation_time: datetime + def __init__( + self, + project: CompiledProjectView, + adapter: BaseAdapter, + connection: Any, + project_config: ProjectConfig | None = ..., + local_config: LocalConfig | None = ..., + on_progress: Callable[[str], None] | None = ..., + invocation_time: datetime = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerRuntime: ... + +class PlannerScopeResolution: + """Resolved planner scopes for one plan build.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + selected_scope: PlannerScope + stale_warning_scope: PlannerScope + inspection_scope: PlannerScope + def __init__( + self, + selected_scope: PlannerScope, + stale_warning_scope: PlannerScope, + inspection_scope: PlannerScope, + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... + +class PlannerWarehouseState: + """Warehouse snapshot and inspection relations gathered once per plan.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + snapshot: WarehouseSnapshot + inspection_relations: PlannerRelationsContext + migration_entries: tuple[ModelMigrationPlanEntry, ...] + migration_warnings: tuple[PlanWarning, ...] + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] + column_rename_hints: tuple[ColumnRenameHint, ...] + old_name_view_entries: tuple[OldNameViewPlanEntry, ...] + old_name_views: tuple[OldNameView, ...] + migration_fingerprints: MigrationFingerprintCache + def __init__( + self, + snapshot: WarehouseSnapshot, + inspection_relations: PlannerRelationsContext, + migration_entries: tuple[ModelMigrationPlanEntry, ...] = ..., + migration_warnings: tuple[PlanWarning, ...] = ..., + column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = ..., + column_rename_hints: tuple[ColumnRenameHint, ...] = ..., + old_name_view_entries: tuple[OldNameViewPlanEntry, ...] = ..., + old_name_views: tuple[OldNameView, ...] = ..., + migration_fingerprints: MigrationFingerprintCache = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerWarehouseState: ... + +class ModelMigrationRequest: + """One destination model whose data should come from an earlier relation.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model: CompiledModelView + discovery: MigrationDiscovery + raw_origin: str | None + origin_location: CompiledRelationLocation | None + origin_model: str | None + force: bool + identity_only: bool + def __init__( + self, + model: CompiledModelView, + discovery: MigrationDiscovery, + raw_origin: str | None = ..., + origin_location: CompiledRelationLocation | None = ..., + origin_model: str | None = ..., + force: bool = ..., + identity_only: bool = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ModelMigrationRequest: ... + +class ModelMigrationDeclaration: + """One project model's explicit migrate_from, resolved to a relation.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_name: str + origin_location: CompiledRelationLocation + origin_model: str | None + def __init__( + self, model_name: str, origin_location: CompiledRelationLocation, origin_model: str | None + ) -> None: ... + def __replace__(self, **changes: Any) -> ModelMigrationDeclaration: ... + +class ModelMigrationDiscovery: + """Manual and automatically discovered migration requests for one plan.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + requests: tuple[ModelMigrationRequest, ...] + warnings: tuple[PlanWarning, ...] + destination_fingerprints: dict[str, tuple[str, ...]] + def __init__( + self, + requests: tuple[ModelMigrationRequest, ...] = ..., + warnings: tuple[PlanWarning, ...] = ..., + destination_fingerprints: dict[str, tuple[str, ...]] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ModelMigrationDiscovery: ... + +class MigrationCompatibilityResult: + """Compatibility status plus human-readable blocking findings.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + status: MigrationCompatibility + findings: tuple[str, ...] + def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... + def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... + +class ModelMigrationPlanning: + """Migration decisions plus the warehouse snapshot as it will look after moves.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + snapshot: WarehouseSnapshot + entries: tuple[ModelMigrationPlanEntry, ...] + warnings: tuple[PlanWarning, ...] + def __init__( + self, + snapshot: WarehouseSnapshot, + entries: tuple[ModelMigrationPlanEntry, ...] = ..., + warnings: tuple[PlanWarning, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ModelMigrationPlanning: ... + +class OldNameViewPlanning: + """Old-name steps for this build plus every compatibility view still recorded.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + entries: tuple[OldNameViewPlanEntry, ...] + views: tuple[OldNameView, ...] + warnings: tuple[PlanWarning, ...] + def __init__( + self, + entries: tuple[OldNameViewPlanEntry, ...] = ..., + views: tuple[OldNameView, ...] = ..., + warnings: tuple[PlanWarning, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> OldNameViewPlanning: ... + +class ColumnMigrationPlanning: + """Column rename decisions plus the warehouse snapshot as it will look after renames.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + snapshot: WarehouseSnapshot + entries: tuple[ColumnMigrationPlanEntry, ...] + hints: tuple[ColumnRenameHint, ...] + warnings: tuple[PlanWarning, ...] + def __init__( + self, + snapshot: WarehouseSnapshot, + entries: tuple[ColumnMigrationPlanEntry, ...] = ..., + hints: tuple[ColumnRenameHint, ...] = ..., + warnings: tuple[PlanWarning, ...] = ..., + ) -> None: ... + def __replace__(self, **changes: Any) -> ColumnMigrationPlanning: ... + +class PlannerIdentityContext: + """Expected version identities for inspection and stale-warning scopes.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + version_identities: DirectModelVersionIdentities + stale_warning_identities: DirectModelVersionIdentities + def __init__( + self, + version_identities: DirectModelVersionIdentities, + stale_warning_identities: DirectModelVersionIdentities, + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... + +class PlannerScopePruningResult: + """Scope pruning outcome with pruned scopes, actions, and staleness state.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + inspection_scope: PlannerScope + execution_scope: PlannerScope + resolved_actions: PlannerResolvedActions + pruned_direct_model_names: tuple[str, ...] + direct_identity_stale_model_names: frozenset[str] + def __init__( + self, + inspection_scope: PlannerScope, + execution_scope: PlannerScope, + resolved_actions: PlannerResolvedActions, + pruned_direct_model_names: tuple[str, ...], + direct_identity_stale_model_names: frozenset[str], + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerScopePruningResult: ... + +class PlannerChangeReconciliation: + """Write-hash-honest changes merged back into resolved actions.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + changes: PlannerChangeResults + resolved_actions: PlannerResolvedActions + def __init__( + self, changes: PlannerChangeResults, resolved_actions: PlannerResolvedActions + ) -> None: ... + def __replace__(self, **changes: Any) -> PlannerChangeReconciliation: ... + +class PlannerEntryResults: + """Model plan entries for one plan build.""" + + __dataclass_fields__: ClassVar[dict[str, Any]] + + model_entry_results: PlannerModelEntryResults + def __init__(self, model_entry_results: PlannerModelEntryResults) -> None: ... + def __replace__(self, **changes: Any) -> PlannerEntryResults: ... + class NativeModelInput: @property def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 424088c573..5deb9bb80a 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -2,16 +2,22 @@ from __future__ import annotations -from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import datetime from decimal import Decimal, InvalidOperation from pathlib import Path -from typing import TYPE_CHECKING, Any - -from sqlbuild._native import MigrationFingerprintCache +from typing import Any + +from sqlbuild._native import ( + BackfillResult, + ColumnRenameHint, + PlanProviderUsage, + PlanWarning, + ScenarioArtifactName, + ScenarioGraphPlan, + SchemaAction, +) from sqlbuild._native import cursor_parse as parse -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest from sqlbuild.adapter.contract.types import MigrationTransfer from sqlbuild.compiler.auditing.models import MeasurementThresholds @@ -25,13 +31,10 @@ CompiledObjectKey, CompiledRelationLocation, FunctionReturnColumn, - InferredColumn, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, - CompiledFunctionView, CompiledModelView, - CompiledProjectView, FunctionLanguage, ) from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration @@ -44,11 +47,9 @@ MigrationPromotion, OldNameViewAction, ) -from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.types import ( BackfillAction, - ChangeKind, CursorGrain, CursorType, CursorWatermarkMode, @@ -60,11 +61,9 @@ RetentionDirection, RetentionPlanPhase, ScenarioArtifactKind, - SchemaActionKind, SchemaChangeKind, SchemaColumnSource, SelectorKind, - WarningSeverity, ) from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, @@ -82,8 +81,6 @@ from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.models import ( FutureCursorsConfig, - LocalConfig, - ProjectConfig, SchemaDynamicColumnFamily, SeedCsvSettings, SourceEntry, @@ -97,84 +94,6 @@ ) from sqlbuild.sql_values.models import SqlValue -if TYPE_CHECKING: - from sqlbuild._native import FixtureKey - - -@dataclass(frozen=True) -class NativeSqlTestArtifact: - """One native-planned SQL-test artifact and its distinct planning error messages.""" - - sql: str - model_names: tuple[str, ...] - error_messages: tuple[str, ...] = () - - -@dataclass(frozen=True) -class RelationFixtureDiagnostic: - """One statically provable invalid fixture condition.""" - - key: FixtureKey - message: str - - -@dataclass(frozen=True) -class RelationFixtureCompletion: - """Completed fixture SQL plus metadata reused by SQL-test validation.""" - - fixture_sql_by_key: dict[FixtureKey, str] - inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] - expected_types: Mapping[FixtureKey, dict[str, str]] - diagnostics: tuple[RelationFixtureDiagnostic, ...] - - -@dataclass(frozen=True) -class FixtureColumnMetadata: - """Known type and nullability for one relation fixture column.""" - - name: str - type: str | None - nullable: bool | None - - -@dataclass(frozen=True) -class FixtureRelationMetadata: - """Known columns and whether they form an authoritative relation shape.""" - - columns: tuple[FixtureColumnMetadata, ...] - authoritative_names: bool - - -@dataclass(frozen=True) -class RelationFixturePlanningContext: - """Project-wide fixture metadata shared across test and scenario planning.""" - - models_by_name: dict[str, CompiledModelView] - relations: dict[FixtureKey, FixtureRelationMetadata] - authoritative_columns: dict[FixtureKey, frozenset[str]] - expected_types: dict[FixtureKey, dict[str, str]] - fixture_inferences: FixtureColumnInferences = field( - default_factory=FixtureColumnInferences, compare=False - ) - - -@dataclass(frozen=True) -class SelectorExpansion: - """One selector split into graph expansion flags and core text.""" - - core: str - upstream: bool = False - downstream: bool = False - - -@dataclass(frozen=True) -class ParsedScenarioArtifactName: - """Parsed physical name for one planner-owned scenario artifact.""" - - hash_prefix: str - kind: str - logical_name: str - @dataclass(frozen=True, kw_only=True, init=False) class CursorOverrides: @@ -276,14 +195,6 @@ def _parse_integer(*, field_name: str, value: str | CursorScalar | None) -> Curs raise PlannerInputError(f"{field_name} value '{value}' is not a valid integer") -@dataclass(frozen=True) -class MissingUpstream: - """One upstream dependency missing from both scope and warehouse.""" - - key: CompiledObjectKey - required_by: tuple[CompiledObjectKey, ...] - - @dataclass(frozen=True) class ParsedSelector: """One parsed selector token before graph resolution.""" @@ -631,31 +542,6 @@ def is_runtime_owned(self) -> bool: return self.is_runtime_produced -@dataclass(frozen=True) -class WarehouseFingerprints: - """Latest direct fingerprints grouped by node type.""" - - models: dict[str, Fingerprint] = field(default_factory=dict) - functions: dict[str, Fingerprint] = field(default_factory=dict) - seeds: dict[str, Fingerprint] = field(default_factory=dict) - python_nodes: dict[tuple[str, str], Fingerprint] = field(default_factory=dict) - unfiltered_schemas: dict[str, tuple[Fingerprint, ...]] = field( - default_factory=dict, compare=False, repr=False - ) - unfiltered_database: str | None = field(default=None, compare=False, repr=False) - - -@dataclass(frozen=True) -class CursorSnapshotScope: - """Execution selection used for cursor validation and runtime producer resolution.""" - - model_keys: frozenset[CompiledObjectKey] - runtime_producer_keys: frozenset[CompiledObjectKey] - invocation_time: datetime | None = None - start_cursor_config: StartCursorsConfig | None = None - cursor_overrides: CursorOverrides | None = None - - @dataclass(frozen=True) class MaximumStartPolicyInputs: """Effective policy and materialization safety inputs for automatic starts.""" @@ -666,62 +552,6 @@ class MaximumStartPolicyInputs: incremental_mode: str | None = None -@dataclass(frozen=True) -class ReferenceRename: - """One model rename that downstream references may have been rewritten for.""" - - new_name: str - origin_name: str - recorded_at: datetime | None = None - - -@dataclass(frozen=True) -class WarehouseSnapshot: - """Frozen point-in-time picture of warehouse state for planning.""" - - existing_relations: dict[str, RelationInfo] = field(default_factory=dict) - existing_columns: dict[str, tuple[ColumnInfo, ...]] = field(default_factory=dict) - fingerprints: WarehouseFingerprints = field(default_factory=WarehouseFingerprints) - cursor_snapshots: dict[str, ModelCursorSnapshot] = field(default_factory=dict) - source_freshness_state_schemas: frozenset[str] = field(default_factory=frozenset) - column_dialect: str | None = None - renamed_models: frozenset[str] = field(default_factory=frozenset) - reference_renames: tuple[ReferenceRename, ...] = () - migration_state_schemas: frozenset[str] = field(default_factory=frozenset) - old_name_view_state_schemas: frozenset[str] = field(default_factory=frozenset) - listed_state_schemas: frozenset[str] | None = field(default_factory=frozenset) - - -@dataclass(frozen=True) -class SelectionStalenessNodeKey: - """Neutral node identity for shared selection-aware staleness classification.""" - - resource_type: str - name: str - - -@dataclass(frozen=True) -class SelectionStalenessGraph: - """Neutral graph inputs for selected-model stale warning classification.""" - - upstream_deps: dict[SelectionStalenessNodeKey, tuple[SelectionStalenessNodeKey, ...]] - selected_model_names: frozenset[str] - run_model_names: frozenset[str] - run_seed_names: frozenset[str] - run_source_names: frozenset[str] - changed_model_names: frozenset[str] - changed_seed_names: frozenset[str] - changed_source_names: frozenset[str] - - -@dataclass(frozen=True) -class SelectionStalenessWarning: - """Neutral stale warning classification for one selected model.""" - - model_name: str - trigger_names: tuple[str, ...] - - @dataclass(frozen=True) class SchemaFinding: """One detected schema difference between expected and warehouse columns.""" @@ -733,34 +563,6 @@ class SchemaFinding: actual_type: str | None = None -@dataclass(frozen=True) -class BackfillResult: - """Resolved backfill action from a change detection policy.""" - - action: BackfillAction - duration: str | None = None - - -@dataclass(frozen=True) -class ChangeDetectionResult: - """Per-model output from change detection and policy resolution.""" - - model_name: str - change_kind: ChangeKind - query_changed: bool = False - config_changed: bool = False - fingerprint_metadata_json: str | None = None - previous_metadata_json: str | None = None - fingerprint_version_hash: str | None = None - previous_version_hash: str | None = None - schema_findings: tuple[SchemaFinding, ...] = field(default_factory=tuple) - backfill: BackfillResult = field( - default_factory=lambda: BackfillResult(action=BackfillAction.FORWARD_ONLY) - ) - recorded_build_relation_missing: bool = False - changed_functions: tuple[str, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class SqlTestSelection: """Unit tests selected or excluded by name, beside the tests of the selected resources.""" @@ -770,15 +572,6 @@ class SqlTestSelection: models_selected: bool = True -@dataclass(frozen=True) -class UnitTestSelectorSplit: - """Resource selectors left after unit-test selectors moved into a test selection.""" - - select: tuple[str, ...] - exclude: tuple[str, ...] - sql_test_selection: SqlTestSelection - - @dataclass(frozen=True) class PlannerScope: """Resolved graph scope for one planner invocation.""" @@ -809,118 +602,6 @@ class PlannerRelationsContext: listed_source_names: frozenset[str] = frozenset() -@dataclass(frozen=True) -class ModelPlanContext: - """Relation lookups and source columns for building one model plan entry.""" - - model_locations: dict[str, CompiledRelationLocation] - models_by_name: dict[str, CompiledModelView] - seed_locations: dict[str, CompiledRelationLocation] - function_locations: dict[str, CompiledRelationLocation] - source_map: dict[str, SourceEntry] - source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] - star_exclude_keyword: str - functions_by_name: dict[str, CompiledFunctionView] = field(default_factory=dict) - runtime_cursor_producer_names: frozenset[str] = frozenset() - future_cursor_config: FutureCursorsConfig | None = None - start_cursor_config: StartCursorsConfig | None = None - invocation_time: datetime | None = None - - -@dataclass(frozen=True) -class CursorOverridePair: - """Resolved per-model cursor start/end overrides for plan entry construction.""" - - start_cursor_override: str | None = None - end_cursor_override: str | None = None - - -@dataclass(frozen=True) -class PlanEntryBuildInputs: - """Blocked models and cursor overrides for plan entry building.""" - - source_freshness_blocked_model_names: frozenset[str] = frozenset() - external_blocked_model_names: frozenset[str] = frozenset() - start_cursor_override: str | None = None - end_cursor_override: str | None = None - future_cursor_config: FutureCursorsConfig | None = None - start_cursor_config: StartCursorsConfig | None = None - invocation_time: datetime | None = None - max_microbatches: int | None = None - max_microbatches_is_override: bool = False - microbatch_limit_action: MicrobatchLimitAction = MicrobatchLimitAction.ERROR - - -@dataclass(frozen=True) -class FunctionChangeResult: - """Per-function output from change detection.""" - - fingerprint_sql: str - reason: PlanReason = PlanReason.NO_CHANGE - - -@dataclass(frozen=True) -class PlannerChangeResults: - """Change detection output for selected planner resources.""" - - models: dict[str, ChangeDetectionResult] - functions: dict[str, FunctionChangeResult] - - -@dataclass(frozen=True) -class ResolvedModelAction: - """Effective model change and backfill decided from the model's own change.""" - - change: ChangeDetectionResult - backfill: BackfillResult - - -@dataclass(frozen=True) -class PlannerResolvedActions: - """Per-model planning decisions keyed by model name.""" - - models: dict[str, ResolvedModelAction] - - -@dataclass(frozen=True) -class DirectModelVersionIdentities: - """Current direct model version identity values by model name.""" - - function_local_hashes: dict[str, str] - seed_version_hashes: dict[str, str] - seed_metadata_jsons: dict[str, str] - model_metadata_jsons: dict[str, str] - model_local_hashes: dict[str, str] - model_version_hashes: dict[str, str] - - -@dataclass(frozen=True) -class PlannerModelEntryResults: - """Model plan-entry phase output.""" - - entries: tuple[ModelPlanEntry, ...] - warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SchemaAction: - """One concrete schema change action to apply to a target table.""" - - kind: SchemaActionKind - column_name: str - column_type: str | None = None - - -@dataclass(frozen=True) -class PlanWarning: - """One warning or error produced during plan resolution.""" - - model_name: str | None - severity: WarningSeverity - message: str - code: str | None = None - - @dataclass(frozen=True) class RetentionPlanEntry: """Warehouse retention work independent of model identity actions.""" @@ -1094,76 +775,6 @@ def blocks_build(self) -> bool: return self.decision.blocks_build -@dataclass(frozen=True) -class ColumnRenameHint: - """A removed column that an added column may have been renamed from.""" - - model_name: str - added_column: str - candidate_columns: tuple[str, ...] - identical: bool = False - - @property - def message(self) -> str: - """Return the one-line hint shown beside the added column.""" - - relation: str = "same expression as" if self.identical else "similar to" - origin: str = self.candidate_columns[0] if len(self.candidate_columns) == 1 else "" - return ( - f"{relation} {', '.join(self.candidate_columns)}; if this is a rename, add " - f"{self.added_column} (migrate_from {origin})" - ) - - -@dataclass(frozen=True) -class QueryProjection: - """One named top-level output column of a query and its canonical expression.""" - - name: str - expression: str - references: frozenset[str] - passthrough: bool = False - - @property - def key(self) -> str: - """Return the case-insensitive output name.""" - - return self.name.lower() - - -@dataclass(frozen=True) -class QueryShape: - """Canonical top-level projections and the rest of one query, ignoring formatting.""" - - projections: tuple[QueryProjection, ...] - body: str - alias_clauses: str = "{}" - positional: bool = False - input_relations: tuple[tuple[str, str], ...] | None = None - local_input_columns: frozenset[str] = frozenset() - clause_references: frozenset[str] = frozenset() - - -@dataclass(frozen=True) -class PlanOutputExtras: - """Optional supplemental seed fingerprints and precomputed SQL tests for plan assembly.""" - - seed_version_hashes: dict[str, str] | None = None - seed_metadata_jsons: dict[str, str] | None = None - seed_plan_reasons: dict[str, PlanReason] | None = None - planned_sql_tests: PlannedSqlTests | None = None - - -@dataclass(frozen=True) -class PlannedSqlTests: - """SQL test entries planned for one selection; reusable only for the same selection.""" - - selected_keys: frozenset[CompiledObjectKey] - sql_test_selection: SqlTestSelection = field(default_factory=SqlTestSelection) - entries: tuple[SqlTestPlanEntry, ...] = () - warnings: tuple[PlanWarning, ...] = () - - @dataclass(frozen=True) class ModelPlanEntry: """Per-model execution plan entry with action, reason, and resolved artifacts.""" @@ -1395,63 +1006,6 @@ class SqlTestPlanEntry: sql_analysis_enabled: bool = True -@dataclass(frozen=True) -class NativeSqlTestPlan: - """Native chain and assertion steps for one SQL test, with SQL when rendered.""" - - chain: tuple[ChainStep, ...] - assertions: tuple[SqlTestAssertionStep, ...] - model_names: tuple[str, ...] - warnings: tuple[PlanWarning, ...] - error_messages: tuple[str, ...] - sql: str | None = None - - -@dataclass(frozen=True) -class SqlTestPlanResult: - """One SQL test's plan entry and warnings, or the fixture diagnostics preventing it.""" - - entry: SqlTestPlanEntry | None - warnings: tuple[PlanWarning, ...] = field(default_factory=tuple) - fixture_diagnostics: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScenarioGraphPlan: - """Inferred graph slice and fixture boundaries for one SQL scenario.""" - - key: CompiledObjectKey - name: str - target_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) - model_names: tuple[str, ...] = field(default_factory=tuple) - source_fixture_names: tuple[str, ...] = field(default_factory=tuple) - ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - seed_names: tuple[str, ...] = field(default_factory=tuple) - seed_fixture_names: tuple[str, ...] = field(default_factory=tuple) - dbt_ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - function_deps: tuple[CompiledObjectKey, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ScenarioArtifactIdentity: - """Logical identity for one scenario-owned physical artifact.""" - - kind: ScenarioArtifactKind | str - logical_name: str - - def __post_init__(self) -> None: - object.__setattr__(self, "kind", ScenarioArtifactKind(self.kind)) - - -@dataclass(frozen=True) -class ScenarioArtifactName: - """Resolved physical relation name for one scenario artifact.""" - - identity: ScenarioArtifactIdentity - physical_name: str - - @dataclass(frozen=True) class ScenarioRelationMap: """Resolved scenario hash prefix and scenario-owned relation names.""" @@ -1525,18 +1079,6 @@ class ScenarioExecutionPlan: ) -@dataclass(frozen=True) -class PlanProviderUsage: - """Provider usage metadata scoped to planned work.""" - - provider_name: str - consumer_kind: str - consumer_name: str - parameter_name: str - annotation_class_name: str | None = None - annotation_module: str | None = None - - @dataclass(frozen=True) class PlanOutput: """Complete execution plan produced by the planner.""" @@ -1625,143 +1167,3 @@ class PlannerPolicies: selection_diagnostics: bool = False plan_sql_tests: bool = True record_migration_fingerprints: bool = True - - -@dataclass(frozen=True) -class PlannerRuntime: - """Resolved execution environment shared by planner phases.""" - - project: CompiledProjectView - adapter: BaseAdapter - connection: Any - project_config: ProjectConfig | None = None - local_config: LocalConfig | None = None - on_progress: Callable[[str], None] | None = None - invocation_time: datetime = field(default_factory=lambda: datetime.now(UTC)) - - -@dataclass(frozen=True) -class PlannerScopeResolution: - """Resolved planner scopes for one plan build.""" - - selected_scope: PlannerScope - stale_warning_scope: PlannerScope - inspection_scope: PlannerScope - - -@dataclass(frozen=True) -class PlannerWarehouseState: - """Warehouse snapshot and inspection relations gathered once per plan.""" - - snapshot: WarehouseSnapshot - inspection_relations: PlannerRelationsContext - migration_entries: tuple[ModelMigrationPlanEntry, ...] = () - migration_warnings: tuple[PlanWarning, ...] = () - column_migration_entries: tuple[ColumnMigrationPlanEntry, ...] = () - column_rename_hints: tuple[ColumnRenameHint, ...] = () - old_name_view_entries: tuple[OldNameViewPlanEntry, ...] = () - old_name_views: tuple[OldNameView, ...] = () - migration_fingerprints: MigrationFingerprintCache = field( - default_factory=MigrationFingerprintCache - ) - - -@dataclass(frozen=True) -class ModelMigrationRequest: - """One destination model whose data should come from an earlier relation.""" - - model: CompiledModelView - discovery: MigrationDiscovery - raw_origin: str | None = None - origin_location: CompiledRelationLocation | None = None - origin_model: str | None = None - force: bool = False - identity_only: bool = False - - -@dataclass(frozen=True) -class ModelMigrationDeclaration: - """One project model's explicit migrate_from, resolved to a relation.""" - - model_name: str - origin_location: CompiledRelationLocation - origin_model: str | None - - -@dataclass(frozen=True) -class ModelMigrationDiscovery: - """Manual and automatically discovered migration requests for one plan.""" - - requests: tuple[ModelMigrationRequest, ...] = () - warnings: tuple[PlanWarning, ...] = () - destination_fingerprints: dict[str, tuple[str, ...]] = field(default_factory=dict) - - -@dataclass(frozen=True) -class MigrationCompatibilityResult: - """Compatibility status plus human-readable blocking findings.""" - - status: MigrationCompatibility - findings: tuple[str, ...] = () - - -@dataclass(frozen=True) -class ModelMigrationPlanning: - """Migration decisions plus the warehouse snapshot as it will look after moves.""" - - snapshot: WarehouseSnapshot - entries: tuple[ModelMigrationPlanEntry, ...] = () - warnings: tuple[PlanWarning, ...] = () - - -@dataclass(frozen=True) -class OldNameViewPlanning: - """Old-name steps for this build plus every compatibility view still recorded.""" - - entries: tuple[OldNameViewPlanEntry, ...] = () - views: tuple[OldNameView, ...] = () - warnings: tuple[PlanWarning, ...] = () - - -@dataclass(frozen=True) -class ColumnMigrationPlanning: - """Column rename decisions plus the warehouse snapshot as it will look after renames.""" - - snapshot: WarehouseSnapshot - entries: tuple[ColumnMigrationPlanEntry, ...] = () - hints: tuple[ColumnRenameHint, ...] = () - warnings: tuple[PlanWarning, ...] = () - - -@dataclass(frozen=True) -class PlannerIdentityContext: - """Expected version identities for inspection and stale-warning scopes.""" - - version_identities: DirectModelVersionIdentities - stale_warning_identities: DirectModelVersionIdentities - - -@dataclass(frozen=True) -class PlannerScopePruningResult: - """Scope pruning outcome with pruned scopes, actions, and staleness state.""" - - inspection_scope: PlannerScope - execution_scope: PlannerScope - resolved_actions: PlannerResolvedActions - pruned_direct_model_names: tuple[str, ...] - direct_identity_stale_model_names: frozenset[str] - - -@dataclass(frozen=True) -class PlannerChangeReconciliation: - """Write-hash-honest changes merged back into resolved actions.""" - - changes: PlannerChangeResults - resolved_actions: PlannerResolvedActions - - -@dataclass(frozen=True) -class PlannerEntryResults: - """Model plan entries for one plan build.""" - - model_entry_results: PlannerModelEntryResults diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index e614d69543..f1870dff72 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,10 +8,9 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import NativeProjectResources, ProjectColumnLineage +from sqlbuild._native import NativeProjectResources, NativeSqlTestArtifact, ProjectColumnLineage from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from sqlbuild.compiler.sql_test_glue.models import NativeSqlTestAssembly from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 49860e9287..b071a4373a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -15,7 +15,11 @@ import sqlbuild._native as native_module import sqlbuild._native as plan_artifacts_module -from sqlbuild._native import PlannedStaticSqlTests, PreparedCompileArtifacts +from sqlbuild._native import ( + NativeSqlTestArtifact, + PlannedStaticSqlTests, + PreparedCompileArtifacts, +) from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( @@ -28,7 +32,6 @@ from sqlbuild.compiler.analysis_session.models import NativeQueryColumns from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError -from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 8173500d6b..fb54d3e2c9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -1,8 +1,13 @@ from dataclasses import dataclass, field +from sqlbuild._native import ( + CursorSnapshotScope, +) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import CursorSnapshotScope, ModelCursorSnapshot +from sqlbuild.compiler.planner.models import ( + ModelCursorSnapshot, +) from sqlbuild.cursor_algebra.types import CursorScalar from sqlbuild.spec.contracts.models import SourceEntry diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py index 4627afef02..601f40a9c9 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -6,6 +6,8 @@ import pytest from sqlbuild._native import ( + RelationFixtureCompletion, + RelationFixturePlanningContext, build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, @@ -13,10 +15,6 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.planner.models import ( - RelationFixtureCompletion, - RelationFixturePlanningContext, -) from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures._test_types import ( FixtureTypeScopeTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 775ecf7c84..ddaaa19ad2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -10,6 +10,7 @@ from typing import Any, ClassVar from sqlbuild._native import ( + WarehouseSnapshot, compiled_model_record, gather_warehouse_snapshot, project_with, @@ -39,7 +40,6 @@ DiscoveredSeedFile, DiscoveredSourceFile, ) -from sqlbuild.compiler.planner.models import WarehouseSnapshot from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ( SchemaColumn, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index e5e227e68a..da3017754f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -7,6 +7,10 @@ from typing import Any, cast from sqlbuild._native import ( + BackfillResult, + CursorOverridePair, + ModelPlanContext, + WarehouseSnapshot, assemble_project, build_compile_inputs, discover_project_inputs, @@ -28,11 +32,7 @@ CompiledResourceType, ) from sqlbuild.compiler.planner.models import ( - BackfillResult, - CursorOverridePair, ModelCursorSnapshot, - ModelPlanContext, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import BackfillAction from sqlbuild.compiler.references.types import SqlReferenceKind diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py index 1042f9d7aa..b55a6cc093 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_orchestration.py @@ -7,6 +7,10 @@ import pytest from sqlbuild._native import ( + BackfillResult, + CursorOverridePair, + ModelPlanContext, + WarehouseSnapshot, build_function_locations, build_model_locations, build_seed_locations, @@ -16,12 +20,6 @@ ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CursorOverridePair, - ModelPlanContext, - WarehouseSnapshot, -) from sqlbuild.compiler.planner.types import BackfillAction from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( OrchestrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index 8c6dab8361..361406c3c8 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -6,13 +6,18 @@ import pytest +from sqlbuild._native import ( + WarehouseSnapshot, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( CompiledRelationLocation, ) from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ModelCursorSnapshot, WarehouseSnapshot +from sqlbuild.compiler.planner.models import ( + ModelCursorSnapshot, +) from sqlbuild.compiler.planner.types import CursorType from sqlbuild.spec.contracts.models import SourceColumnEntry, SourceEntry from tests.integration.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index ea7d115776..e719bfb69a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -6,6 +6,8 @@ import pytest from sqlbuild._native import ( + RelationFixtureCompletion, + RelationFixturePlanningContext, build_compiled_project, build_relation_fixture_completion, build_relation_fixture_context, @@ -16,8 +18,6 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import ( - RelationFixtureCompletion, - RelationFixturePlanningContext, SqlTestPlanEntry, SqlTestSelection, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index 27bf62fe45..e92bdd06ff 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -5,16 +5,18 @@ import pytest -from sqlbuild._native import gather_warehouse_snapshot +from sqlbuild._native import ( + CursorSnapshotScope, + WarehouseSnapshot, + gather_warehouse_snapshot, +) from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( - CursorSnapshotScope, ModelCursorSnapshot, - WarehouseSnapshot, ) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ConcurrentCursorReadsTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 7b5d5fbdec..e9d59dc840 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + CursorSnapshotScope, + WarehouseSnapshot, compiled_model_record, gather_source_columns, gather_warehouse_snapshot, @@ -28,9 +30,7 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorOverrides, - CursorSnapshotScope, ModelCursorSnapshot, - WarehouseSnapshot, ) from sqlbuild.cursor_algebra.models import TimestampValue from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index ce7415f003..a2803ba4f3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -9,6 +9,7 @@ import sqlbuild._native as identity_planning from sqlbuild._native import ( + PlanWarning, compiled_model_record, format_plan, project_with, @@ -21,7 +22,6 @@ CursorBounds, ModelPlanEntry, PlanOutput, - PlanWarning, ) from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index b587e45dff..4383d4ec61 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -7,12 +7,14 @@ import pytest -from sqlbuild._native import compute_cursor_bounds +from sqlbuild._native import ( + BackfillResult, + compute_cursor_bounds, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.planner.models import ( - BackfillResult, CursorBounds, ModelCursorSnapshot, ModelPlanEntry, diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index 23899932b8..b7d9284811 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -3,6 +3,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + ScenarioGraphPlan, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -15,7 +18,6 @@ ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, - ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, ) diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index 3c99950166..72c07d7589 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -4,6 +4,12 @@ from pathlib import Path +from sqlbuild._native import ( + BackfillResult, + ColumnRenameHint, + PlanProviderUsage, + PlanWarning, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -11,15 +17,11 @@ from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage from sqlbuild.compiler.planner.models import ( - BackfillResult, - ColumnRenameHint, CursorBounds, CursorInputRelation, FunctionPlanEntry, ModelPlanEntry, PlanOutput, - PlanProviderUsage, - PlanWarning, SchemaFinding, SeedPlanEntry, SourceLoadPlanEntry, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 988650b137..9376a9c36b 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -6,11 +6,13 @@ import pytest -from sqlbuild._native import format_plan +from sqlbuild._native import ( + ColumnRenameHint, + format_plan, +) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( - ColumnRenameHint, CursorBounds, CursorInputRelation, PlanOutput, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 7144fe00cb..46b3075d3c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -2,6 +2,13 @@ from dataclasses import dataclass, field from pathlib import Path +from sqlbuild._native import ( + MissingUpstream, + PlanWarning, + ScenarioArtifactIdentity, + ScenarioGraphPlan, + SchemaAction, +) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -15,14 +22,9 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( CursorBounds, - MissingUpstream, ParsedSelector, PathSelector, - PlanWarning, - ScenarioArtifactIdentity, - ScenarioGraphPlan, ScenarioRelationMap, - SchemaAction, SchemaFinding, SourceLoadPlanEntry, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index cece72c0bf..7274d5f1ec 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -7,7 +7,12 @@ from pathlib import Path from typing import cast -from sqlbuild._native import build_version_identity_metadata_json, compute_query_hash +from sqlbuild._native import ( + WarehouseFingerprints, + WarehouseSnapshot, + build_version_identity_metadata_json, + compute_query_hash, +) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.compiler.compile.models import ( CompiledFunction, @@ -26,7 +31,9 @@ FunctionLanguage, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import PlannerScope, WarehouseFingerprints, WarehouseSnapshot +from sqlbuild.compiler.planner.models import ( + PlannerScope, +) from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index c6f621657f..8c6ea178bf 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -1,8 +1,14 @@ from dataclasses import dataclass, field +from sqlbuild._native import ( + BackfillResult, + ReferenceRename, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn -from sqlbuild.compiler.planner.models import BackfillResult, ReferenceRename, SchemaFinding +from sqlbuild.compiler.planner.models import ( + SchemaFinding, +) from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, PlanAction, PlanReason from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index ee1c1effcb..3d8cd1f269 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -6,6 +6,12 @@ import pytest from sqlbuild._native import ( + BackfillResult, + ChangeDetectionResult, + PlannerChangeResults, + PlanWarning, + WarehouseFingerprints, + WarehouseSnapshot, build_model_warnings, build_version_identity_metadata_json, compute_query_hash, @@ -19,13 +25,7 @@ from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - PlannerChangeResults, PlannerScope, - PlanWarning, - WarehouseFingerprints, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py index 6e9cae5263..955b73c549 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_reference_renames.py @@ -6,9 +6,12 @@ import pytest -from sqlbuild._native import compute_query_hash, origin_reference_names +from sqlbuild._native import ( + ReferenceRename, + compute_query_hash, + origin_reference_names, +) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.planner.models import ReferenceRename from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( OriginReferenceNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index bf57006e39..47e15bb13f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -13,6 +13,11 @@ import sqlbuild._native as _native from sqlbuild._native import ( + BackfillResult, + ChangeDetectionResult, + PlannerChangeResults, + ScenarioArtifactIdentity, + WarehouseSnapshot, build_downstream_deps, build_scenario_relation_map, compiled_model_record, @@ -68,13 +73,8 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - PlannerChangeResults, PlanOutput, - ScenarioArtifactIdentity, ScenarioRelationMap, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ( BackfillAction, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py index 17fa0c5cfe..799d182f48 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, PlannerResolvedActions, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index 0e9b1029fe..fe70911fea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild._native import merge_recomputed_model_changes -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, PlannerChangeResults, PlannerResolvedActions, ResolvedModelAction, + merge_recomputed_model_changes, ) from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py index 1c613cdf19..709c37fed0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/helpers.py @@ -6,10 +6,16 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING -from sqlbuild._native import build_migration_fingerprint, parse_query_shape +from sqlbuild._native import ( + QueryShape, + build_migration_fingerprint, + parse_query_shape, +) from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.migrations.types import OldNameViewAction -from sqlbuild.compiler.planner.models import OldNameViewPlanEntry, QueryShape +from sqlbuild.compiler.planner.models import ( + OldNameViewPlanEntry, +) if TYPE_CHECKING: from sqlbuild._native import InputColumns diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index f302136b45..bcd1c93b32 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -8,7 +8,12 @@ import pytest -from sqlbuild._native import plan_column_migrations +from sqlbuild._native import ( + PlannerRuntime, + WarehouseFingerprints, + WarehouseSnapshot, + plan_column_migrations, +) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -23,10 +28,7 @@ from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, - PlannerRuntime, PlannerScope, - WarehouseFingerprints, - WarehouseSnapshot, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py index e5682b411a..fd43ad9082 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_renames.py @@ -5,12 +5,12 @@ import pytest from sqlbuild._native import ( + ColumnRenameHint, identical_renames, parse_query_shape, rename_hints, renames_explain_change, ) -from sqlbuild.compiler.planner.models import ColumnRenameHint from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( IdenticalRenameTestCase, RenameExplainsChangeTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py index 678171429d..dc9ba3a2f6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_query_shape.py @@ -6,8 +6,11 @@ import pytest -from sqlbuild._native import build_migration_fingerprint, parse_query_shape -from sqlbuild.compiler.planner.models import QueryShape +from sqlbuild._native import ( + QueryShape, + build_migration_fingerprint, + parse_query_shape, +) from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( StoredProjectionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py index 28651b6839..5b420844ba 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/_test_types.py @@ -2,8 +2,10 @@ from dataclasses import dataclass +from sqlbuild._native import ( + ChangeDetectionResult, +) from sqlbuild.adapter.contract.models import RetentionState -from sqlbuild.compiler.planner.models import ChangeDetectionResult from sqlbuild.compiler.planner.types import RetentionDirection, RetentionPlanPhase from sqlbuild.spec.contracts.types import TableType, TableTypeDowngradePolicy diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 98414f42c7..e8194607f9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -4,6 +4,11 @@ from typing import Any from unittest.mock import Mock +from sqlbuild._native import ( + PlannerRuntime, + PlannerWarehouseState, + WarehouseSnapshot, +) from sqlbuild.adapter.contract.classes.retention_adapter import RetentionAdapterMixin from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( @@ -20,10 +25,7 @@ ) from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, - PlannerRuntime, PlannerScope, - PlannerWarehouseState, - WarehouseSnapshot, ) from sqlbuild.spec.contracts.models import ( LocalConfig, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py index 672a3357a0..f4f162d30a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py @@ -4,9 +4,13 @@ import pytest -from sqlbuild._native import check_full_rebuild_allowed, describe_full_rebuild_cause +from sqlbuild._native import ( + BackfillResult, + ChangeDetectionResult, + check_full_rebuild_allowed, + describe_full_rebuild_cause, +) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import BackfillResult, ChangeDetectionResult from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( FullRebuildCheckTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py index 62e07557bb..626b6e49c1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py @@ -5,15 +5,17 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import detect_stale_warning_changes -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( ChangeDetectionResult, DirectModelVersionIdentities, PlannerChangeResults, PlannerIdentityContext, - PlannerScope, PlannerScopeResolution, WarehouseSnapshot, + detect_stale_warning_changes, +) +from sqlbuild.compiler.planner.models import ( + PlannerScope, ) from sqlbuild.compiler.planner.types import ChangeKind from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index f6efbc2626..7935807286 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -6,7 +6,13 @@ import pytest -from sqlbuild._native import plan_retention, plan_table_types +from sqlbuild._native import ( + PlannerRuntime, + PlannerWarehouseState, + WarehouseFingerprints, + plan_retention, + plan_table_types, +) from sqlbuild.adapter.contract.models import ( RelationInfo, RenderedRetentionChange, @@ -21,12 +27,9 @@ from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( - PlannerRuntime, PlannerScope, - PlannerWarehouseState, RetentionPlanEntry, TableTypePlanEntry, - WarehouseFingerprints, ) from sqlbuild.compiler.planner.types import ( MaterializationType, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py index f92acf0a35..17cf987f13 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py @@ -2,11 +2,13 @@ import pytest -from sqlbuild._native import resolve_planner_scopes +from sqlbuild._native import ( + PlannerScopeResolution, + resolve_planner_scopes, +) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import ( PlannerPolicies, - PlannerScopeResolution, PlannerSelection, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_test_project diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 500efbbcb9..75bdb881a9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -4,6 +4,9 @@ from pathlib import Path +from sqlbuild._native import ( + ModelPlanContext, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -12,7 +15,6 @@ CompileModelConfig, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanContext def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> CompiledModelView: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index 4a6a6ee82c..30324d6023 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -2,15 +2,17 @@ import pytest -from sqlbuild._native import resolve_model_sql +from sqlbuild._native import ( + BackfillResult, + CursorOverridePair, + WarehouseSnapshot, + resolve_model_sql, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( - BackfillResult, - CursorOverridePair, ModelCursorSnapshot, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index 14311bc6e5..142a004783 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -6,11 +6,13 @@ import pytest -from sqlbuild._native import resolve_model_hook_entries +from sqlbuild._native import ( + ModelPlanContext, + resolve_model_hook_entries, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry -from sqlbuild.compiler.planner.models import ModelPlanContext from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( HookSqlResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py index 0dfa091810..8ff43b33f1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/selection/test_unit_test_selectors.py @@ -4,10 +4,15 @@ import pytest -from sqlbuild._native import split_project_unit_test_selectors +from sqlbuild._native import ( + UnitTestSelectorSplit, + split_project_unit_test_selectors, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import SqlTestSelection, UnitTestSelectorSplit +from sqlbuild.compiler.planner.models import ( + SqlTestSelection, +) from tests.unit.src.sqlbuild.compiler.planner._helpers.selection._test_types import ( UnitTestSelectorErrorTestCase, UnitTestSelectorSplitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 53c4009afe..395b2862df 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -8,6 +8,9 @@ from typing import Any from sqlbuild._native import ( + NativeSqlTestPlan, + PlanWarning, + SqlTestPlanResult, expand_sql_macros, extract_sql_references, plan_compiled_sql_test_artifacts, @@ -42,10 +45,7 @@ DiscoveredSqlTestFile, ) from sqlbuild.compiler.planner.models import ( - NativeSqlTestPlan, - PlanWarning, SqlTestPlanEntry, - SqlTestPlanResult, ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index b6c625c5b3..4e29c5ef81 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -6,6 +6,9 @@ import pytest from sqlbuild._native import ( + NativeSqlTestPlan, + PlanWarning, + SqlTestPlanResult, build_test_command_plan, plan_compiled_sql_test_artifacts, project_with, @@ -32,13 +35,10 @@ from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( ChainStep, - NativeSqlTestPlan, PlannerScope, PlannerSelection, PlanOutput, - PlanWarning, SqlTestPlanEntry, - SqlTestPlanResult, ) from sqlbuild.compiler.planner.types import WarningSeverity from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py index fc42a40697..7c38e4b974 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_buildability.py @@ -2,9 +2,13 @@ import pytest -from sqlbuild._native import check_buildability, missing_upstream_message +from sqlbuild._native import ( + MissingUpstream, + WarehouseSnapshot, + check_buildability, + missing_upstream_message, +) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.planner.models import MissingUpstream, WarehouseSnapshot from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CheckBuildabilityTestCase, MissingUpstreamMessageTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py index 8e9db31606..87577812a6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py @@ -4,9 +4,11 @@ import pytest -from sqlbuild._native import check_cursor_type_consistency +from sqlbuild._native import ( + PlanWarning, + check_cursor_type_consistency, +) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.models import PlanWarning from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorTypeCheckTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 832fda88be..37bb32fb5e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -8,6 +8,11 @@ import sqlbuild._native as sql_test_assembly from sqlbuild import _native as fixture_completion from sqlbuild._native import ( + FixtureColumnMetadata, + FixtureRelationMetadata, + PlanWarning, + RelationFixtureCompletion, + RelationFixturePlanningContext, build_selected_test_entries, compiled_model_record, project_with, @@ -24,11 +29,6 @@ ) from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences from sqlbuild.compiler.planner.models import ( - FixtureColumnMetadata, - FixtureRelationMetadata, - PlanWarning, - RelationFixtureCompletion, - RelationFixturePlanningContext, SqlTestPlanEntry, SqlTestSelection, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index 437ceae8ce..0fd1466f3b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -5,6 +5,8 @@ import pytest from sqlbuild._native import ( + PlanEntryBuildInputs, + PlanWarning, apply_configured_microbatch_limits, apply_microbatch_limit, ) @@ -19,8 +21,6 @@ CursorBounds, CursorInputRelation, ModelPlanEntry, - PlanEntryBuildInputs, - PlanWarning, ) from sqlbuild.compiler.planner.types import ( IncrementalMode, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index 1616717d3f..2d2412f27a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -7,7 +7,12 @@ import pytest -from sqlbuild._native import compute_microbatch_range, compute_plan_cursor_bounds +from sqlbuild._native import ( + BackfillResult, + WarehouseSnapshot, + compute_microbatch_range, + compute_plan_cursor_bounds, +) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -18,11 +23,9 @@ from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.constants import MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL from sqlbuild.compiler.planner.models import ( - BackfillResult, CursorBounds, CursorInputRelation, ModelCursorSnapshot, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index c04e12e040..f874eeed7d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -3,6 +3,9 @@ import pytest from sqlbuild._native import ( + ParsedScenarioArtifactName, + ScenarioArtifactIdentity, + ScenarioArtifactName, build_scenario_artifact_name, build_scenario_hash_index, build_scenario_relation_map, @@ -12,9 +15,6 @@ ) from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( - ParsedScenarioArtifactName, - ScenarioArtifactIdentity, - ScenarioArtifactName, ScenarioRelationMap, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py index 94e928157e..8b9ef461f2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_cli.py @@ -5,10 +5,15 @@ import pytest -from sqlbuild._native import build_cli_scenario_plan +from sqlbuild._native import ( + ScenarioArtifactName, + build_cli_scenario_plan, +) from sqlbuild.compiler.compile.models import CompiledSqlScenario from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioArtifactName, ScenarioExecutionPlan +from sqlbuild.compiler.planner.models import ( + ScenarioExecutionPlan, +) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ScenarioCliPlanIdentifierLimitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index 0491153fb4..4c1e956a8a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -4,13 +4,15 @@ import pytest -from sqlbuild._native import plan_scenario_graph +from sqlbuild._native import ( + ScenarioGraphPlan, + plan_scenario_graph, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlScenario, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ScenarioGraphPlan from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanScenarioGraphErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index b1e7474eb1..a0639c392c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + ScenarioGraphPlan, build_scenario_execution_plan, build_scenario_fixture_plans, build_scenario_relation_plan, @@ -29,7 +30,6 @@ from sqlbuild.compiler.planner.models import ( ScenarioExecutionPlan, ScenarioFixturePlan, - ScenarioGraphPlan, ScenarioRelationPlan, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 689b7fb73f..1d0f717bea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -6,6 +6,13 @@ import pytest from sqlbuild._native import ( + ChangeDetectionResult, + DirectModelVersionIdentities, + PlannerChangeResults, + PlannerScopeResolution, + PlanWarning, + WarehouseFingerprints, + WarehouseSnapshot, build_stale_out_of_selection_warnings, compiled_model_record, resolve_planner_scopes, @@ -19,16 +26,9 @@ from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DirectModelVersionIdentities, - PlannerChangeResults, PlannerPolicies, PlannerScope, - PlannerScopeResolution, PlannerSelection, - PlanWarning, - WarehouseFingerprints, - WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.compiler.source_freshness.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index 02c08d1056..a9890a19da 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -2,9 +2,14 @@ import pytest -from sqlbuild._native import resolve_model_plan_action +from sqlbuild._native import ( + ChangeDetectionResult, + resolve_model_plan_action, +) from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.planner.models import ChangeDetectionResult, SchemaFinding +from sqlbuild.compiler.planner.models import ( + SchemaFinding, +) from sqlbuild.compiler.planner.types import ( BackfillAction, ChangeKind, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py index 24d986187f..3585b5f548 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py @@ -2,8 +2,13 @@ import pytest -from sqlbuild._native import resolve_schema_actions -from sqlbuild.compiler.planner.models import SchemaAction, SchemaFinding +from sqlbuild._native import ( + SchemaAction, + resolve_schema_actions, +) +from sqlbuild.compiler.planner.models import ( + SchemaFinding, +) from sqlbuild.compiler.planner.types import ( OnSchemaChange, SchemaActionKind, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index 34a4154bea..e9038b4f4e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -2,11 +2,13 @@ import pytest -from sqlbuild._native import build_model_warnings -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( ChangeDetectionResult, PlanWarning, SchemaAction, + build_model_warnings, +) +from sqlbuild.compiler.planner.models import ( SchemaFinding, ) from sqlbuild.compiler.planner.types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 0b8f24f86e..a0877b163d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -6,11 +6,17 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import start_sql_test_planning +from sqlbuild._native import ( + PlannedSqlTests, + PlanWarning, + start_sql_test_planning, +) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import PlannedSqlTests, PlanWarning, SqlTestSelection +from sqlbuild.compiler.planner.models import ( + SqlTestSelection, +) from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py index a67263cbbc..9aebae157b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/_test_types.py @@ -1,11 +1,13 @@ from dataclasses import dataclass from datetime import datetime -from sqlbuild.compiler.planner.models import ( - CursorBounds, +from sqlbuild._native import ( SelectionStalenessGraph, SelectionStalenessWarning, ) +from sqlbuild.compiler.planner.models import ( + CursorBounds, +) from sqlbuild.spec.contracts.models import FutureCursorsConfig diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index b4dd031f7c..a7cf81c185 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -6,6 +6,8 @@ from typing import Any from sqlbuild._native import ( + SelectionStalenessGraph, + SelectionStalenessNodeKey, build_execution_plan, build_version_identity_metadata_json, compute_query_hash, @@ -31,8 +33,6 @@ PlannerPolicies, PlannerSelection, PlanOutput, - SelectionStalenessGraph, - SelectionStalenessNodeKey, ) from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 8f7c4ad4ae..2320037d58 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from sqlbuild._native import ( + DirectModelVersionIdentities, build_direct_model_version_identities, build_downstream_deps, build_model_version_identity_metadata_json, @@ -24,7 +25,9 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.planner.models import DirectModelVersionIdentities, PlannerScope +from sqlbuild.compiler.planner.models import ( + PlannerScope, +) from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py index 95faa636e2..b52fea1e16 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_deep_graph_identity.py @@ -2,7 +2,9 @@ import pytest -from sqlbuild.compiler.planner.models import DirectModelVersionIdentities +from sqlbuild._native import ( + DirectModelVersionIdentities, +) from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( DeepChainIdentityTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index 1062a23f69..6e892c3296 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -4,14 +4,16 @@ import pytest -from sqlbuild._native import build_function_local_hashes +from sqlbuild._native import ( + DirectModelVersionIdentities, + build_function_local_hashes, +) from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledObjectKey, FunctionReturnColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import DirectModelVersionIdentities from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( FunctionReturnContractIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py index e58e05e028..b440e90918 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selection_staleness.py @@ -2,11 +2,11 @@ import pytest -from sqlbuild._native import classify_selection_staleness_warnings -from sqlbuild.compiler.planner.models import ( +from sqlbuild._native import ( SelectionStalenessGraph, SelectionStalenessNodeKey, SelectionStalenessWarning, + classify_selection_staleness_warnings, ) from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( DeepChainStalenessTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py index 54984ac676..e3385efc70 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_selector_expansion.py @@ -2,8 +2,10 @@ import pytest -from sqlbuild._native import split_selector_expansion -from sqlbuild.compiler.planner.models import SelectorExpansion +from sqlbuild._native import ( + SelectorExpansion, + split_selector_expansion, +) from tests.unit.src.sqlbuild.compiler.planner.main._test_types import ( SelectorExpansionErrorTestCase, SelectorExpansionTestCase, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 461c921660..a680ad3f23 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -8,6 +8,9 @@ from typing import Any, ClassVar from unittest.mock import MagicMock +from sqlbuild._native import ( + ScenarioGraphPlan, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo @@ -30,7 +33,6 @@ AuditPlanEntry, PlanOutput, ScenarioExecutionPlan, - ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, SeedPlanEntry, diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py index c053e78852..c9ef94e5bb 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py @@ -4,6 +4,9 @@ from pathlib import Path from typing import cast +from sqlbuild._native import ( + ScenarioGraphPlan, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledRelationLocation, @@ -15,7 +18,6 @@ ScenarioExecutionPlan, ScenarioExpectedExpectationPlan, ScenarioFixturePlan, - ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, SeedPlanEntry, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 3b6d8cf6bf..8001e4cf88 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -4,6 +4,9 @@ from pathlib import Path from typing import Any, ClassVar +from sqlbuild._native import ( + ScenarioGraphPlan, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult, RelationInfo @@ -16,7 +19,6 @@ ModelPlanEntry, ScenarioExecutionPlan, ScenarioFixturePlan, - ScenarioGraphPlan, ScenarioRelationMap, ScenarioRelationPlan, SeedPlanEntry, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index b5c3000671..dbe8f4735e 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -8,6 +8,7 @@ from typing import cast from sqlbuild._native import ( + BackfillResult, DbtCombinedGraph, DbtEventStream, DbtEventStreamUpdate, @@ -30,7 +31,9 @@ DiscoveredProjectInputs, DiscoveredSqlModelFile, ) -from sqlbuild.compiler.planner.models import BackfillResult, ModelPlanEntry +from sqlbuild.compiler.planner.models import ( + ModelPlanEntry, +) from sqlbuild.compiler.planner.types import ( BackfillAction, MaterializationType, From fbd0172837052c61d444170e9ac9734ed2768579 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:29:52 +0100 Subject: [PATCH 07/23] refactor: declare planner enums and the fixture inference cache natively --- .../compile_driver/python_plan_entries.rs | 3 +- .../_helpers/planner_changes/detect.rs | 3 +- .../_helpers/planner_changes/results.rs | 15 +++- .../_helpers/planner_entries/model_entries.rs | 5 +- .../_helpers/planner_identity/staleness.rs | 4 +- .../planner_migrations/column_warnings.rs | 7 +- .../planner_migrations/discovery_requests.rs | 2 +- .../planner_migrations/old_name_references.rs | 3 +- .../_helpers/planner_migrations/old_names.rs | 3 +- .../_helpers/planner_models/constants.rs | 51 ++++++++++++ .../bindings/_helpers/planner_models/enums.rs | 19 +++++ .../planner_models/fixture_inferences.rs | 79 +++++++++++++++++++ .../bindings/_helpers/planner_models/mod.rs | 5 +- .../_helpers/planner_models/records.rs | 2 +- .../_helpers/planner_models/registration.rs | 6 +- .../planner_resolution/cursor_warnings.rs | 3 +- .../bindings/_helpers/planner_scenario/cli.rs | 2 +- .../_helpers/planner_scenario/graph.rs | 4 +- .../planner_strategy/cursor_inputs.rs | 4 +- .../planner_strategy/microbatch_limits.rs | 4 +- .../_helpers/planner_strategy/strategy.rs | 3 +- .../_helpers/planner_strategy/values.rs | 4 +- .../_helpers/planner_warehouse/cursors.rs | 2 +- .../planner_warehouse/source_freshness.rs | 3 +- .../_helpers/sql_test_glue/plan_rows.rs | 2 +- fensu.toml | 2 +- src/sqlbuild/_native.pyi | 55 +++++++++++-- .../classes/fixture_column_inferences.py | 46 ----------- src/sqlbuild/compiler/planner/models.py | 8 +- src/sqlbuild/compiler/planner/types.py | 47 ----------- .../compiler/planner/main/_test_types.py | 4 +- .../planner/main/test_build_execution_plan.py | 2 +- .../test_microbatch_reconciliation.py | 5 +- .../sqlbuild/cli/output/main/plan/helpers.py | 6 +- .../cli/output/main/plan/test_plan_text.py | 6 +- .../output/main/plan_json/test_plan_json.py | 6 +- .../compiler/planner/_helpers/_test_types.py | 4 +- .../planner/_helpers/changes/_test_helpers.py | 2 +- .../planner/_helpers/changes/_test_types.py | 7 +- .../planner/_helpers/changes/test_detect.py | 4 +- .../planner/_helpers/changes/test_schema.py | 7 +- .../planner/_helpers/identity/_test_types.py | 7 +- .../planner/_helpers/identity/helpers.py | 5 +- .../test_merge_recomputed_model_changes.py | 5 +- .../_helpers/planning/test_full_refresh.py | 5 +- .../_helpers/planning/test_identities.py | 2 +- .../planner/_helpers/resolve/test_cursor.py | 7 +- .../_helpers/sql_test_assembly/_test_types.py | 4 +- .../sql_test_assembly/test_plan_test.py | 2 +- .../_helpers/test_cursor_type_check.py | 2 +- .../_helpers/test_fixture_planning_work.py | 5 +- .../planner/_helpers/test_scenario_graph.py | 2 +- .../_helpers/test_selection_staleness.py | 2 +- .../planner/_helpers/test_strategy_action.py | 6 +- .../_helpers/test_strategy_schema_actions.py | 6 +- .../_helpers/test_strategy_warnings.py | 10 +-- .../compiler/planner/classes/helpers.py | 3 +- .../classes/test_fixture_column_inferences.py | 6 +- .../planner/classes/test_sql_test_planning.py | 2 +- .../run/_helpers/test_cursor_bounds.py | 6 +- 60 files changed, 345 insertions(+), 191 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs delete mode 100644 src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 9cf7e18501..403eda0b72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; -const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const NATIVE_MODULE: &str = "sqlbuild._native"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; @@ -352,7 +351,7 @@ fn skipped_ingress_warnings<'py>( let upstream: HashMap> = upstream_names(graph)?; let warning_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "PlanWarning")?; let severity: Bound<'py, PyAny> = - imported(py, PLANNER_TYPES_MODULE, "WarningSeverity")?.getattr("WARNING")?; + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("WARNING")?; let mut warnings: Vec> = Vec::new(); for loader in loaders { if !nodes.contains(loader.as_str())? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 0ee8cbe412..88011119fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -1,5 +1,6 @@ //! Detect how each selected model and function changed since its recorded build. +use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -742,7 +743,7 @@ impl<'py> ChangeResult<'py> { } fn finish(self, backfill: Backfill, kind: &str) -> PyResult> { - let kind: Bound<'py, PyAny> = planner_enum(self.py, "ChangeKind", kind)?; + let kind: Bound<'py, PyAny> = native_enum(self.py, "ChangeKind", kind)?; self.finish_kind(backfill, kind) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs index 727df386ed..8fd29dcaf2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/results.rs @@ -34,6 +34,15 @@ pub(crate) fn planner_enum<'py>( imported(py, TYPES_MODULE, name)?.call1((value,)) } +/// A member of a natively declared planner enumeration, from its value. +pub(crate) fn native_enum<'py>( + py: Python<'py>, + name: &str, + value: &str, +) -> PyResult> { + imported(py, "sqlbuild._native", name)?.call1((value,)) +} + /// `BackfillResult(action=..., duration=...)`. pub(crate) fn backfill_result<'py>( py: Python<'py>, @@ -65,7 +74,7 @@ pub(crate) fn change_kind<'py>(py: Python<'py>, kind: ChangeKind) -> PyResult "config_changed", ChangeKind::NoChange => "no_change", }; - planner_enum(py, "ChangeKind", value) + native_enum(py, "ChangeKind", value) } /// A tuple of `SchemaFinding` values. @@ -88,12 +97,12 @@ pub(crate) fn schema_findings<'py>( items.push(call_with( &class, &[ - ("kind", planner_enum(py, "SchemaChangeKind", kind)?), + ("kind", native_enum(py, "SchemaChangeKind", kind)?), ( "column_name", finding.column_name.clone().into_pyobject(py)?.into_any(), ), - ("source", planner_enum(py, "SchemaColumnSource", source)?), + ("source", native_enum(py, "SchemaColumnSource", source)?), ( "expected_type", finding.expected_type.clone().into_pyobject(py)?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index b9a0afdb50..2b7025516e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -376,7 +376,10 @@ fn check_microbatch_concurrency<'py>( "PlanWarning", &[ ("model_name", none(py)), - ("severity", planner_type(py, "WarningSeverity", "WARNING")?), + ( + "severity", + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("WARNING")?, + ), ("code", string(py, MICROBATCH_COMPLEXITY_CODE)), ( "message", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs index f3041d9f4c..f01cb48fc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/staleness.rs @@ -1,5 +1,6 @@ //! Warnings for selected models stale through changed upstreams outside the run set. +use crate::bindings::_helpers::planner_changes::results::native_enum; use std::collections::{HashMap, HashSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -19,7 +20,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_changes::detect::{ModelPass, model_change}; -use crate::bindings::_helpers::planner_changes::results::planner_enum; use crate::bindings::_helpers::planner_identity::hashing::graph_key; /// Change kinds that mean the model's own identity changed. @@ -144,7 +144,7 @@ fn plan_warnings<'py>( warnings: Vec, ) -> PyResult> { let warning_class: Bound<'py, PyAny> = imported(py, "sqlbuild._native", "PlanWarning")?; - let severity: Bound<'py, PyAny> = planner_enum(py, "WarningSeverity", "warning")?; + let severity: Bound<'py, PyAny> = native_enum(py, "WarningSeverity", "warning")?; let mut rendered: Vec> = Vec::with_capacity(warnings.len()); for warning in warnings { let message: String = stale_upstream_warning_message( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs index a205870f8e..352ed81e8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_warnings.rs @@ -26,8 +26,11 @@ pub(super) fn warning<'py>( ("model_name", entry.getattr("model_name")?), ( "severity", - imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? - .getattr(if error { "ERROR" } else { "WARNING" })?, + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr(if error { + "ERROR" + } else { + "WARNING" + })?, ), ("message", text(py, message)), ("code", text(py, code)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs index ed9e4b42d4..01a9e160b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_requests.rs @@ -186,7 +186,7 @@ pub(super) fn warnings<'py>( if selected.contains(&name)? { warnings.push(native(py, "PlanWarning", &[ ("model_name", PyString::new(py, &name).into_any()), - ("severity", imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")?.getattr("WARNING")?), + ("severity", imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("WARNING")?), ("message", PyString::new(py, &format!("'{name}' matches {detail}; no automatic migration was inferred. Add migrate_from to choose one.")).into_any()), ("code", PyString::new(py, "M107").into_any()), ])?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs index c2fd95fd9a..ee9fe726d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_references.rs @@ -128,8 +128,7 @@ pub(super) fn errors<'py>( ("model_name", py.None().into_bound(py)), ( "severity", - imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? - .getattr("ERROR")?, + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("ERROR")?, ), ( "message", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs index 0cc97d4143..44af340e9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_names.rs @@ -142,8 +142,7 @@ fn old_name_claim_errors<'py>( ("model_name", name.clone()), ( "severity", - imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? - .getattr("ERROR")?, + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("ERROR")?, ), ("message", message.into_pyobject(py)?.into_any()), ("code", "M114".into_pyobject(py)?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs new file mode 100644 index 0000000000..7680642569 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/constants.rs @@ -0,0 +1,51 @@ +//! Planner enumerations declared natively. + +/// Each planner `StrEnum`: its name and its members with their values, in declaration order. +pub(crate) const PLANNER_STR_ENUMS: &[(&str, &[(&str, &str)])] = &[ + ( + "GraphResourceKind", + &[ + ("MODEL", "model"), + ("SEED", "seed"), + ("SOURCE", "source"), + ("FUNCTION", "function"), + ], + ), + ( + "ChangeKind", + &[ + ("FIRST_RUN", "first_run"), + ("RENAMED", "renamed"), + ("QUERY_CHANGED", "query_changed"), + ("FUNCTION_CHANGED", "function_changed"), + ("CONFIG_CHANGED", "config_changed"), + ("SCHEMA_CHANGED", "schema_changed"), + ("NO_CHANGE", "no_change"), + ], + ), + ( + "SchemaChangeKind", + &[ + ("COLUMN_ADDED", "column_added"), + ("COLUMN_REMOVED", "column_removed"), + ("COLUMN_TYPE_CHANGED", "column_type_changed"), + ], + ), + ( + "SchemaColumnSource", + &[("YML", "yml"), ("SQL_ANALYSIS", "sql_analysis")], + ), + ("CursorWatermarkMode", &[("ALL", "all"), ("ANY", "any")]), + ( + "SchemaActionKind", + &[ + ("ADD_COLUMN", "add_column"), + ("DROP_COLUMN", "drop_column"), + ("ALTER_COLUMN_TYPE", "alter_column_type"), + ], + ), + ( + "WarningSeverity", + &[("ERROR", "error"), ("WARNING", "warning"), ("INFO", "info")], + ), +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs new file mode 100644 index 0000000000..cca4508a64 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/enums.rs @@ -0,0 +1,19 @@ +//! Creates the planner `StrEnum` classes in the native module. + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::planner_models::constants::PLANNER_STR_ENUMS; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList}; + +pub(crate) fn register_enums(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py = module.py(); + let str_enum = imported(py, "enum", "StrEnum")?; + for (name, members) in PLANNER_STR_ENUMS { + let options = PyDict::new(py); + options.set_item("module", "sqlbuild._native")?; + options.set_item("qualname", *name)?; + let class = str_enum.call((*name, PyList::new(py, members.iter())?), Some(&options))?; + module.add(*name, class)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs new file mode 100644 index 0000000000..0db81c84a5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/fixture_inferences.rs @@ -0,0 +1,79 @@ +//! Fixture column inference computed once per fixture body and inference profile. + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use std::sync::Mutex; + +/// Reuse inference for identical fixture SQL under an equal, catalog-free profile. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct FixtureColumnInferences { + by_profile: Mutex, Py)>>, +} + +#[pymethods] +impl FixtureColumnInferences { + #[new] + fn new() -> Self { + Self { + by_profile: Mutex::new(Vec::new()), + } + } + + /// The inference for one fixture query, computing each distinct query once. + #[pyo3(signature = (*, query_sql, inference_profile))] + fn infer<'py>( + &self, + query_sql: &Bound<'py, PyAny>, + inference_profile: &Bound<'py, PyAny>, + ) -> PyResult> { + let py = query_sql.py(); + let compute = || { + call_with( + &imported(py, "sqlbuild._native", "infer_fixture_column_facts")?, + &[ + ("query_sql", query_sql.clone()), + ("inference_profile", inference_profile.clone()), + ], + ) + }; + if !inference_profile.getattr("binding_catalog")?.is_none() { + return compute(); + } + let inferred = self.profile_results(py, inference_profile)?; + if let Some(found) = inferred.get_item(query_sql)? { + return Ok(found); + } + let computed = compute()?; + inferred.set_item(query_sql, &computed)?; + Ok(computed) + } +} + +impl FixtureColumnInferences { + fn profile_results<'py>( + &self, + py: Python<'py>, + inference_profile: &Bound<'py, PyAny>, + ) -> PyResult> { + let known: Vec<(Py, Py)> = self + .by_profile + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? + .iter() + .map(|(profile, results)| (profile.clone_ref(py), results.clone_ref(py))) + .collect(); + for (profile, results) in known { + if profile.bind(py).eq(inference_profile)? { + return Ok(results.into_bound(py)); + } + } + let results = PyDict::new(py); + self.by_profile + .lock() + .map_err(|_| pyo3::exceptions::PyRuntimeError::new_err("fixture inference cache"))? + .push((inference_profile.clone().unbind(), results.clone().unbind())); + Ok(results) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs index ab99095c4b..0be40906ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/mod.rs @@ -1,5 +1,8 @@ -//! Planner value records hosted natively. +//! Planner value records, enumerations and caches hosted natively. +pub(crate) mod constants; +pub(crate) mod enums; +pub(crate) mod fixture_inferences; pub(crate) mod record; pub(crate) mod records; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs index ae17c63bdd..41beda161f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/records.rs @@ -242,7 +242,7 @@ planner_record! { (relations = "relations", FieldDefault::Required, true, true), (authoritative_columns = "authoritative_columns", FieldDefault::Required, true, true), (expected_types = "expected_types", FieldDefault::Required, true, true), - (fixture_inferences = "fixture_inferences", FieldDefault::Class(("sqlbuild.compiler.planner.classes.fixture_column_inferences", "FixtureColumnInferences")), false, true), + (fixture_inferences = "fixture_inferences", FieldDefault::Class(("sqlbuild._native", "FixtureColumnInferences")), false, true), ] } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs index 53ca49e8bc..242379387b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_models/registration.rs @@ -1,5 +1,7 @@ //! Registers the planner value records. +use crate::bindings::_helpers::planner_models::enums::register_enums; +use crate::bindings::_helpers::planner_models::fixture_inferences::FixtureColumnInferences; use crate::bindings::_helpers::planner_models::records::{ BackfillResult, ChangeDetectionResult, ColumnMigrationPlanning, ColumnRenameHint, CursorOverridePair, CursorSnapshotScope, DirectModelVersionIdentities, FixtureColumnMetadata, @@ -17,9 +19,11 @@ use crate::bindings::_helpers::planner_models::records::{ SelectorExpansion, SqlTestPlanResult, UnitTestSelectorSplit, WarehouseFingerprints, WarehouseSnapshot, }; -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + register_enums(module)?; + module.add_class::()?; NativeSqlTestArtifact::register_class(module)?; RelationFixtureDiagnostic::register_class(module)?; RelationFixtureCompletion::register_class(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs index bde0ad03bc..71d874c6bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_warnings.rs @@ -7,7 +7,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -const TYPES: &str = "sqlbuild.compiler.planner.types"; const FUTURE_CAPPED_CODE: &str = "FUTURE_CURSOR_CAPPED"; const START_CAPPED_CODE: &str = "AUTOMATIC_START_CAPPED"; @@ -84,7 +83,7 @@ pub(crate) fn with_cursor_safety_warnings<'py>( ("model_name", model_name.clone()), ( "severity", - imported(py, TYPES, "WarningSeverity")?.getattr("WARNING")?, + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("WARNING")?, ), ("code", string(py, code)), ("message", string(py, &message)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index e03de51799..04db2673d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -142,7 +142,7 @@ fn namespace<'py>( fn raise_for_warnings(warnings: &Bound<'_, PyAny>) -> PyResult<()> { let py = warnings.py(); - let severity = imported(py, TYPES, "WarningSeverity")?.getattr("ERROR")?; + let severity = imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("ERROR")?; let mut messages: Vec = Vec::new(); let mut code = None; for warning in warnings.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs index 9b1d14c7a0..d0ec52b3ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/graph.rs @@ -3,7 +3,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::graph::keys::{KeyCache, native_key}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; -use crate::bindings::_helpers::planner_scenario::values::{TYPES, string}; +use crate::bindings::_helpers::planner_scenario::values::string; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::PyTuple; use sqlbuild_analysis::planner_scenario::main::graph::graph; @@ -97,7 +97,7 @@ pub(crate) fn plan_scenario_graph<'py>( ("model_name", py.None().into_bound(py)), ( "severity", - imported(py, TYPES, "WarningSeverity")?.getattr("ERROR")?, + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("ERROR")?, ), ("message", string(py, &message)), ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 2a266b95c7..d877d50c3f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::model_config::config_values::{config_cursor_bound use crate::bindings::_helpers::planner_identity::metadata::or_empty; use crate::bindings::_helpers::planner_strategy::constants::{ ENFORCED_CONTRACT, INCREMENTAL_MATERIALIZATION, MODEL_REFERENCE, PLAN_MODELS_MODULE, - PLANNER_TYPES_MODULE, SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, + SEED_REFERENCE, SOURCE_REFERENCE, TIMESTAMP_CURSOR, }; use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; use crate::bindings::_helpers::planner_strategy::values::{ @@ -188,7 +188,7 @@ fn input_kind<'py>( SOURCE_REFERENCE => "source", _ => return Ok(py.None().into_bound(py)), }; - imported(py, PLANNER_TYPES_MODULE, "GraphResourceKind")?.call1((value,)) + imported(py, "sqlbuild._native", "GraphResourceKind")?.call1((value,)) } /// A producer model's terminal cursor bound parsed with the consumer's cursor type. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs index bb3633947b..f63f1dfafb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/microbatch_limits.rs @@ -1,5 +1,6 @@ //! Microbatch limits applied to planned model entries in CLI, project and model precedence. +use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction}; @@ -13,7 +14,6 @@ use sqlbuild_core::plan_display::models::TimestampGrainFacts; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::compiled_project::records::replace; use crate::bindings::_helpers::plan_output::display::resolve_effective_microbatch_batch_size; -use crate::bindings::_helpers::planner_changes::results::planner_enum; use crate::bindings::_helpers::planner_strategy::constants::PLANNER_TYPES_MODULE; use crate::bindings::_helpers::planner_strategy::microbatches::{ CAP_ACTIONS, CONTRACT_TYPES_MODULE, capped_microbatch_bounds, count_microbatches, @@ -160,7 +160,7 @@ fn limited<'py>( &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", name), - ("severity", planner_enum(py, "WarningSeverity", "warning")?), + ("severity", native_enum(py, "WarningSeverity", "warning")?), ( "code", PyString::new(py, MICROBATCH_LIMIT_WARNING_CODE).into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index 1c24ee9c0a..ba5ea4535a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -1,5 +1,6 @@ //! Per-model plan actions, schema actions and plan warnings. +use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; @@ -124,7 +125,7 @@ fn resolve_schema_actions<'py>( actions.push(call_with( &class, &[ - ("kind", planner_enum(py, "SchemaActionKind", kind)?), + ("kind", native_enum(py, "SchemaActionKind", kind)?), ( "column_name", PyString::new(py, &row.column_name).into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs index 7754f870c2..61936868fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/values.rs @@ -1,5 +1,6 @@ //! Conversions between the planner's Python models and the native strategy rows. +use crate::bindings::_helpers::planner_changes::results::native_enum; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString}; @@ -13,7 +14,6 @@ use sqlbuild_analysis::planner_strategy::models::{ }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::planner_changes::results::planner_enum; pub(crate) const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; pub(crate) const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; @@ -140,7 +140,7 @@ pub(crate) fn plan_warning<'py>( &imported(py, "sqlbuild._native", "PlanWarning")?, &[ ("model_name", model_name.clone()), - ("severity", planner_enum(py, "WarningSeverity", severity)?), + ("severity", native_enum(py, "WarningSeverity", severity)?), ("message", PyString::new(py, &row.message).into_any()), ("code", row.code.into_pyobject(py)?.into_any()), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 35a18d4420..5b7f203d9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -478,7 +478,7 @@ fn collect_models<'py>( let mode = config_str(&model, "cursor_watermark_mode")? .filter(|value| !value.is_empty()) .unwrap_or_else(|| "all".to_owned()); - let watermark_mode = member(py, (PLANNER_TYPES, "CursorWatermarkMode"), &mode)?; + let watermark_mode = member(py, ("sqlbuild._native", "CursorWatermarkMode"), &mode)?; models.push(CursorModel { target_tag, target_relation, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index 71808d3954..fcc5658781 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -52,8 +52,7 @@ pub(crate) fn source_freshness_unknown_warnings<'py>( ("model_name", py.None().into_bound(py)), ( "severity", - imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")? - .getattr("WARNING")?, + imported(py, "sqlbuild._native", "WarningSeverity")?.getattr("WARNING")?, ), ("message", text(py, &message)), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs index 24a21f6ce0..7972c17034 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -58,7 +58,7 @@ pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> assertions.push(step(&value?, false)?); } let mut warnings: Vec> = Vec::new(); - let severity = imported(py, "sqlbuild.compiler.planner.types", "WarningSeverity")?; + let severity = imported(py, "sqlbuild._native", "WarningSeverity")?; for value in row.get_item(4)?.try_iter()? { let value = value?; warnings.push(call_with( diff --git a/fensu.toml b/fensu.toml index 1738998284..c51d431272 100644 --- a/fensu.toml +++ b/fensu.toml @@ -61,7 +61,7 @@ reason = "Native bindings own configuration resolution; this package only declar [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/planner/__init__.py" -reason = "Native bindings own planning, plan entries, plan output and command plans; this package only declares the plan models, types, constants, errors and the fixture inference cache the native planner reads." +reason = "Native bindings own planning, plan entries, plan output, command plans and the planner value records; this package only declares the plan models, types, constants and errors the executor, integrations and pipeline import." [[targets.python.rule_exceptions]] rule = "FFR309" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0eeed686d4..1b8978598b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7,10 +7,10 @@ from contextlib import AbstractContextManager from contextvars import Token from datetime import date, datetime from decimal import Decimal +from enum import StrEnum from pathlib import Path, PurePath from threading import Event -from typing import Any, Literal, Protocol, TextIO, TypedDict -from typing import Any, ClassVar, Literal, TextIO, TypedDict +from typing import Any, ClassVar, Literal, Protocol, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -208,7 +208,6 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) -from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( AuditPlanEntry, @@ -253,7 +252,6 @@ from sqlbuild.compiler.planner.models import ( ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ChangeKind, CursorGrain, CursorType, MaterializationType, @@ -261,8 +259,6 @@ from sqlbuild.compiler.planner.types import ( PlanAction, PlanReason, ScenarioArtifactKind, - SchemaActionKind, - WarningSeverity, ) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, @@ -725,6 +721,53 @@ type _GraphIndexes = tuple[ type _SelectorFailure = tuple[str, str, str | None] type _SelectorKeys = tuple[list[_GraphKey] | None, _SelectorFailure | None] +class GraphResourceKind(StrEnum): + MODEL = "model" + SEED = "seed" + SOURCE = "source" + FUNCTION = "function" + +class ChangeKind(StrEnum): + FIRST_RUN = "first_run" + RENAMED = "renamed" + QUERY_CHANGED = "query_changed" + FUNCTION_CHANGED = "function_changed" + CONFIG_CHANGED = "config_changed" + SCHEMA_CHANGED = "schema_changed" + NO_CHANGE = "no_change" + +class SchemaChangeKind(StrEnum): + COLUMN_ADDED = "column_added" + COLUMN_REMOVED = "column_removed" + COLUMN_TYPE_CHANGED = "column_type_changed" + +class SchemaColumnSource(StrEnum): + YML = "yml" + SQL_ANALYSIS = "sql_analysis" + +class CursorWatermarkMode(StrEnum): + ALL = "all" + ANY = "any" + +class SchemaActionKind(StrEnum): + ADD_COLUMN = "add_column" + DROP_COLUMN = "drop_column" + ALTER_COLUMN_TYPE = "alter_column_type" + +class WarningSeverity(StrEnum): + ERROR = "error" + WARNING = "warning" + INFO = "info" + +class FixtureColumnInferences: + """Reuse inference for identical fixture SQL under an equal, catalog-free profile.""" + + def __init__(self) -> None: ... + def infer( + self, *, query_sql: str, inference_profile: ExpressionInferenceProfile + ) -> FixtureColumnInference | None: + """Return the inference for one fixture query, computing each distinct query once.""" + class NativeSqlTestArtifact: """One native-planned SQL-test artifact and its distinct planning error messages.""" diff --git a/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py b/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py deleted file mode 100644 index 16025222bf..0000000000 --- a/src/sqlbuild/compiler/planner/classes/fixture_column_inferences.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Fixture column inference computed once per fixture body and inference profile.""" - -from __future__ import annotations - -from sqlbuild._native import infer_fixture_column_facts -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import FixtureColumnInference - - -class FixtureColumnInferences: - """Reuse inference for identical fixture SQL under an equal, catalog-free profile.""" - - def __init__(self) -> None: - self._by_profile: list[ - tuple[ExpressionInferenceProfile, dict[str, FixtureColumnInference | None]] - ] = [] - - def infer( - self, *, query_sql: str, inference_profile: ExpressionInferenceProfile - ) -> FixtureColumnInference | None: - """Return the inference for one fixture query, computing each distinct query once.""" - - if inference_profile.binding_catalog is not None: - return infer_fixture_column_facts( - query_sql=query_sql, inference_profile=inference_profile - ) - inferred: dict[str, FixtureColumnInference | None] = self._profile_results( - inference_profile - ) - if query_sql not in inferred: - inferred[query_sql] = infer_fixture_column_facts( - query_sql=query_sql, inference_profile=inference_profile - ) - return inferred[query_sql] - - def _profile_results( - self, inference_profile: ExpressionInferenceProfile - ) -> dict[str, FixtureColumnInference | None]: - profile: ExpressionInferenceProfile - results: dict[str, FixtureColumnInference | None] - for profile, results in self._by_profile: - if profile == inference_profile: - return results - results = {} - self._by_profile.append((inference_profile, results)) - return results diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 5deb9bb80a..5fafa8a24c 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -11,11 +11,15 @@ from sqlbuild._native import ( BackfillResult, ColumnRenameHint, + CursorWatermarkMode, + GraphResourceKind, PlanProviderUsage, PlanWarning, ScenarioArtifactName, ScenarioGraphPlan, SchemaAction, + SchemaChangeKind, + SchemaColumnSource, ) from sqlbuild._native import cursor_parse as parse from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RetentionRequest @@ -52,8 +56,6 @@ BackfillAction, CursorGrain, CursorType, - CursorWatermarkMode, - GraphResourceKind, MaterializationType, OnSchemaChange, PlanAction, @@ -61,8 +63,6 @@ RetentionDirection, RetentionPlanPhase, ScenarioArtifactKind, - SchemaChangeKind, - SchemaColumnSource, SelectorKind, ) from sqlbuild.compiler.source_freshness.models import ( diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index 026265d449..c0a9ec7c69 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -18,23 +18,6 @@ class SelectorKind(StrEnum): TEST = "test" -class GraphResourceKind(StrEnum): - MODEL = "model" - SEED = "seed" - SOURCE = "source" - FUNCTION = "function" - - -class ChangeKind(StrEnum): - FIRST_RUN = "first_run" - RENAMED = "renamed" - QUERY_CHANGED = "query_changed" - FUNCTION_CHANGED = "function_changed" - CONFIG_CHANGED = "config_changed" - SCHEMA_CHANGED = "schema_changed" - NO_CHANGE = "no_change" - - class BackfillAction(StrEnum): FULL = "full" BOUNDED = "bounded" @@ -53,17 +36,6 @@ class ContractPolicy(StrEnum): ENFORCED = "enforced" -class SchemaChangeKind(StrEnum): - COLUMN_ADDED = "column_added" - COLUMN_REMOVED = "column_removed" - COLUMN_TYPE_CHANGED = "column_type_changed" - - -class SchemaColumnSource(StrEnum): - YML = "yml" - SQL_ANALYSIS = "sql_analysis" - - class RetentionDirection(StrEnum): INCREASE = "increase" DECREASE = "decrease" @@ -119,13 +91,6 @@ class MicrobatchStrategy(StrEnum): WATERMARK = "watermark" -class CursorWatermarkMode(StrEnum): - """How multiple watermark inputs contribute availability.""" - - ALL = "all" - ANY = "any" - - class IncrementalMode(StrEnum): FULL = "full" MICROBATCH = "microbatch" @@ -182,18 +147,6 @@ def is_table_backed(cls, *, materialized: object | None) -> bool: return materialized in {cls.TABLE, cls.INCREMENTAL, cls.SNAPSHOT} -class SchemaActionKind(StrEnum): - ADD_COLUMN = "add_column" - DROP_COLUMN = "drop_column" - ALTER_COLUMN_TYPE = "alter_column_type" - - -class WarningSeverity(StrEnum): - ERROR = "error" - WARNING = "warning" - INFO = "info" - - class ScenarioArtifactKind(StrEnum): SOURCE = "source" REF = "ref" diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 11294e2004..b6e7da3b27 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -1,11 +1,13 @@ from dataclasses import dataclass, field +from sqlbuild._native import ( + WarningSeverity, +) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.planner.types import ( BackfillAction, PlanAction, PlanReason, - WarningSeverity, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index a2803ba4f3..8697f8058f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -10,6 +10,7 @@ import sqlbuild._native as identity_planning from sqlbuild._native import ( PlanWarning, + WarningSeverity, compiled_model_record, format_plan, project_with, @@ -27,7 +28,6 @@ BackfillAction, PlanAction, PlanReason, - WarningSeverity, ) from sqlbuild.spec.contracts.models import ( CursorsConfig, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py index 4383d4ec61..e779e11b4a 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/test_microbatch_reconciliation.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( BackfillResult, + CursorWatermarkMode, compute_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token @@ -19,7 +20,9 @@ ModelCursorSnapshot, ModelPlanEntry, ) -from sqlbuild.compiler.planner.types import BackfillAction, CursorWatermarkMode +from sqlbuild.compiler.planner.types import ( + BackfillAction, +) from sqlbuild.cursor_algebra.models import IntegerValue from sqlbuild.executor.run._helpers.materializations.microbatch import ( _clamp_intervals_to_model_domain, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index 72c07d7589..11716681e2 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -9,6 +9,9 @@ ColumnRenameHint, PlanProviderUsage, PlanWarning, + SchemaChangeKind, + SchemaColumnSource, + WarningSeverity, ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -32,9 +35,6 @@ MaterializationType, PlanAction, PlanReason, - SchemaChangeKind, - SchemaColumnSource, - WarningSeverity, ) from sqlbuild.spec.contracts.models import SeedCsvSettings from sqlbuild.spec.contracts.types import MicrobatchLimitAction, SourceWriteStrategy diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 9376a9c36b..088c9ba527 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -8,6 +8,9 @@ from sqlbuild._native import ( ColumnRenameHint, + GraphResourceKind, + SchemaChangeKind, + WarningSeverity, format_plan, ) from sqlbuild.compiler.compile.types import FunctionLanguage @@ -19,12 +22,9 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - GraphResourceKind, MaterializationType, PlanAction, PlanReason, - SchemaChangeKind, - WarningSeverity, ) from sqlbuild.compiler.python_nodes.types import ( PythonIdentityStatus, diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index a26ec438f1..d60b7efda3 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -7,7 +7,10 @@ import pytest -from sqlbuild._native import format_plan_json +from sqlbuild._native import ( + WarningSeverity, + format_plan_json, +) from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( CursorBounds, @@ -22,7 +25,6 @@ MaterializationType, PlanAction, PlanReason, - WarningSeverity, ) from sqlbuild.compiler.python_nodes.types import ( PythonIdentityStatus, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 46b3075d3c..ebaad34eb9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -3,11 +3,13 @@ from pathlib import Path from sqlbuild._native import ( + ChangeKind, MissingUpstream, PlanWarning, ScenarioArtifactIdentity, ScenarioGraphPlan, SchemaAction, + WarningSeverity, ) from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, @@ -30,12 +32,10 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ChangeKind, MaterializationType, OnSchemaChange, PlanAction, PlanReason, - WarningSeverity, ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 7274d5f1ec..fb105642c1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -8,6 +8,7 @@ from typing import cast from sqlbuild._native import ( + ChangeKind, WarehouseFingerprints, WarehouseSnapshot, build_version_identity_metadata_json, @@ -34,7 +35,6 @@ from sqlbuild.compiler.planner.models import ( PlannerScope, ) -from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectModelChangesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 8c6ea178bf..f84a55b035 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -2,6 +2,7 @@ from sqlbuild._native import ( BackfillResult, + ChangeKind, ReferenceRename, ) from sqlbuild.adapter.contract.models import ColumnInfo @@ -9,7 +10,11 @@ from sqlbuild.compiler.planner.models import ( SchemaFinding, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind, PlanAction, PlanReason +from sqlbuild.compiler.planner.types import ( + BackfillAction, + PlanAction, + PlanReason, +) from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 3d8cd1f269..931a523c57 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -8,10 +8,12 @@ from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, + ChangeKind, PlannerChangeResults, PlanWarning, WarehouseFingerprints, WarehouseSnapshot, + WarningSeverity, build_model_warnings, build_version_identity_metadata_json, compute_query_hash, @@ -29,10 +31,8 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ChangeKind, PlanAction, PlanReason, - WarningSeverity, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_helpers import ( build_function_caller_metadata_case, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py index e99f28e530..4b9ad9fa4c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_schema.py @@ -2,11 +2,14 @@ import pytest -from sqlbuild._native import detect_schema_changes +from sqlbuild._native import ( + SchemaChangeKind, + SchemaColumnSource, + detect_schema_changes, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.planner.models import SchemaFinding -from sqlbuild.compiler.planner.types import SchemaChangeKind, SchemaColumnSource from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( DetectSchemaChangesTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py index 9d47f1817a..fbb009e40f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/_test_types.py @@ -2,7 +2,12 @@ from dataclasses import dataclass -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild._native import ( + ChangeKind, +) +from sqlbuild.compiler.planner.types import ( + BackfillAction, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py index 799d182f48..3ac9251cf6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/helpers.py @@ -5,10 +5,13 @@ from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, + ChangeKind, PlannerResolvedActions, ResolvedModelAction, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild.compiler.planner.types import ( + BackfillAction, +) MERGE_MODEL_NAME: str = "orders" MERGE_RESOLVED_HASH: str = "resolved-hash" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index fe70911fea..0918c63812 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -5,12 +5,15 @@ from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, + ChangeKind, PlannerChangeResults, PlannerResolvedActions, ResolvedModelAction, merge_recomputed_model_changes, ) -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild.compiler.planner.types import ( + BackfillAction, +) from tests.unit.src.sqlbuild.compiler.planner._helpers.identity._test_types import ( MergeRecomputedModelChangesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py index f4f162d30a..56d3145fc6 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_full_refresh.py @@ -7,11 +7,14 @@ from sqlbuild._native import ( BackfillResult, ChangeDetectionResult, + ChangeKind, check_full_rebuild_allowed, describe_full_rebuild_cause, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind +from sqlbuild.compiler.planner.types import ( + BackfillAction, +) from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( FullRebuildCheckTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py index 626b6e49c1..5d91b2ba62 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_identities.py @@ -7,6 +7,7 @@ import sqlbuild._native as native_module from sqlbuild._native import ( ChangeDetectionResult, + ChangeKind, DirectModelVersionIdentities, PlannerChangeResults, PlannerIdentityContext, @@ -17,7 +18,6 @@ from sqlbuild.compiler.planner.models import ( PlannerScope, ) -from sqlbuild.compiler.planner.types import ChangeKind from tests.unit.src.sqlbuild.compiler.planner._helpers.planning._test_types import ( ReusableExecutionModelNamesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py index 2aac608caf..1ad561525f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor.py @@ -7,7 +7,11 @@ import pytest -from sqlbuild._native import apply_typed_cursor_replay_policy, compute_cursor_bounds +from sqlbuild._native import ( + CursorWatermarkMode, + apply_typed_cursor_replay_policy, + compute_cursor_bounds, +) from sqlbuild._native import cursor_observed_partition as observed_partition from sqlbuild._native import cursor_parse as parse from sqlbuild._native import cursor_render as render @@ -19,7 +23,6 @@ from sqlbuild.compiler.planner.types import ( CursorGrain, CursorType, - CursorWatermarkMode, ) from sqlbuild.cursor_algebra.models import AlignedInterval, DateValue, TimestampValue from sqlbuild.cursor_algebra.types import CursorScalar diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py index 6c50339415..4a292223f1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py @@ -1,6 +1,8 @@ from dataclasses import dataclass, field -from sqlbuild.compiler.planner.types import WarningSeverity +from sqlbuild._native import ( + WarningSeverity, +) @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 4e29c5ef81..ae3cce2643 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -9,6 +9,7 @@ NativeSqlTestPlan, PlanWarning, SqlTestPlanResult, + WarningSeverity, build_test_command_plan, plan_compiled_sql_test_artifacts, project_with, @@ -40,7 +41,6 @@ PlanOutput, SqlTestPlanEntry, ) -from sqlbuild.compiler.planner.types import WarningSeverity from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py index 87577812a6..ceb8aeed49 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_type_check.py @@ -6,10 +6,10 @@ from sqlbuild._native import ( PlanWarning, + WarningSeverity, check_cursor_type_consistency, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorTypeCheckTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 37bb32fb5e..fe2cd6aab1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -27,7 +27,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.planner.classes import fixture_column_inferences as fixture_inferences from sqlbuild.compiler.planner.models import ( SqlTestPlanEntry, SqlTestSelection, @@ -233,9 +232,9 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ }, ) fixture_completion.fixture_read_cache_info(reset=True) - fixture_inference: Mock = Mock(wraps=fixture_inferences.infer_fixture_column_facts) + fixture_inference: Mock = Mock(wraps=sql_test_assembly.infer_fixture_column_facts) monkeypatch.setattr( - fixture_inferences, + sql_test_assembly, "infer_fixture_column_facts", fixture_inference, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index 4c1e956a8a..aa955bd78a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( ScenarioGraphPlan, + WarningSeverity, plan_scenario_graph, ) from sqlbuild.compiler.compile.models import ( @@ -13,7 +14,6 @@ CompiledSqlScenario, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanScenarioGraphErrorTestCase, PlanScenarioGraphTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 1d0f717bea..df1c67d96b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( ChangeDetectionResult, + ChangeKind, DirectModelVersionIdentities, PlannerChangeResults, PlannerScopeResolution, @@ -30,7 +31,6 @@ PlannerScope, PlannerSelection, ) -from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index a9890a19da..63b5df9306 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -4,6 +4,9 @@ from sqlbuild._native import ( ChangeDetectionResult, + ChangeKind, + SchemaChangeKind, + SchemaColumnSource, resolve_model_plan_action, ) from sqlbuild.compiler.compile.types import CompiledModelView @@ -12,11 +15,8 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ChangeKind, PlanAction, PlanReason, - SchemaChangeKind, - SchemaColumnSource, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( IncrementalStrategyErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py index 3585b5f548..296125ebf7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_schema_actions.py @@ -4,6 +4,9 @@ from sqlbuild._native import ( SchemaAction, + SchemaActionKind, + SchemaChangeKind, + SchemaColumnSource, resolve_schema_actions, ) from sqlbuild.compiler.planner.models import ( @@ -11,9 +14,6 @@ ) from sqlbuild.compiler.planner.types import ( OnSchemaChange, - SchemaActionKind, - SchemaChangeKind, - SchemaColumnSource, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ResolveSchemaActionsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py index e9038b4f4e..16f393b55d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_warnings.py @@ -4,8 +4,13 @@ from sqlbuild._native import ( ChangeDetectionResult, + ChangeKind, PlanWarning, SchemaAction, + SchemaActionKind, + SchemaChangeKind, + SchemaColumnSource, + WarningSeverity, build_model_warnings, ) from sqlbuild.compiler.planner.models import ( @@ -13,13 +18,8 @@ ) from sqlbuild.compiler.planner.types import ( BackfillAction, - ChangeKind, MaterializationType, OnSchemaChange, - SchemaActionKind, - SchemaChangeKind, - SchemaColumnSource, - WarningSeverity, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( BuildModelWarningsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index 12b3a8f69f..9c6088189d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -11,7 +11,6 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.planner.classes.fixture_column_inferences as fixture_inferences from sqlbuild._native import build_migration_fingerprint, infer_fixture_column_facts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import FixtureColumnInference @@ -125,5 +124,5 @@ def counting( inferences.append(1) return infer_fixture_column_facts(query_sql=query_sql, inference_profile=inference_profile) - monkeypatch.setattr(fixture_inferences, "infer_fixture_column_facts", counting) + monkeypatch.setattr(native_module, "infer_fixture_column_facts", counting) return inferences diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py index a731ac9d73..6f970978e9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_fixture_column_inferences.py @@ -4,9 +4,11 @@ import pytest -from sqlbuild._native import infer_fixture_column_facts +from sqlbuild._native import ( + FixtureColumnInferences, + infer_fixture_column_facts, +) from sqlbuild.compiler.compile.models import FixtureColumnInference -from sqlbuild.compiler.planner.classes.fixture_column_inferences import FixtureColumnInferences from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( FixtureColumnInferencesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index a0877b163d..f2c8c5e441 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( PlannedSqlTests, PlanWarning, + WarningSeverity, start_sql_test_planning, ) from sqlbuild.compiler.compile.models import CompiledObjectKey @@ -17,7 +18,6 @@ from sqlbuild.compiler.planner.models import ( SqlTestSelection, ) -from sqlbuild.compiler.planner.types import WarningSeverity from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 9c27ea3d76..2fd1c8da80 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -8,9 +8,12 @@ import duckdb import pytest +from sqlbuild._native import ( + CursorWatermarkMode, + normalize_cursor_snapshot_grain, +) from sqlbuild._native import cursor_render as render from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild._native import normalize_cursor_snapshot_grain from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation @@ -25,7 +28,6 @@ from sqlbuild.compiler.planner.types import ( CursorGrain, CursorType, - CursorWatermarkMode, MaterializationType, PlanAction, PlanReason, From b49211182fe3361e4012a2e7c96873f5ae35164d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:31:23 +0100 Subject: [PATCH 08/23] chore: reconcile boundary after planner data models --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 0b52452126..47a0d41149 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 11383 +remaining_lines = 11290 [[stays]] path = "executor" From f69f2f69b2f85cb55655fa02cf0948e54637ef10 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:41:01 +0100 Subject: [PATCH 09/23] refactor: build CLI command requests and invocation contexts as native records --- crates/sqlbuild-cli/src/dispatch/constants.rs | 46 +- .../_helpers/audit_command/command.rs | 4 +- .../_helpers/build_command/completion.rs | 4 +- .../_helpers/build_command/invocation.rs | 4 +- .../bindings/_helpers/check/check_command.rs | 4 +- .../_helpers/command_runtime/invocation.rs | 4 +- .../_helpers/dbt_interop/pipeline_phases.rs | 2 +- .../_helpers/dbt_interop/sqlbuild_work.rs | 4 +- .../bindings/_helpers/diff_report/query.rs | 2 +- .../_helpers/janitor/janitor_command.rs | 6 +- .../src/bindings/_helpers/mod.rs | 1 + .../bindings/_helpers/records/constants.rs | 677 +++++++++++ .../src/bindings/_helpers/records/fields.rs | 146 +++ .../src/bindings/_helpers/records/mod.rs | 4 + .../src/bindings/_helpers/records/models.rs | 146 +++ .../bindings/_helpers/records/registration.rs | 52 + .../bindings/_helpers/seed_command/command.rs | 4 +- .../_helpers/sql_test_command/test_command.rs | 6 +- .../src/bindings/main/register.rs | 3 +- fensu.toml | 5 - src/sqlbuild/_native.pyi | 1064 ++++++++++++++++- src/sqlbuild/cli/commands/models.py | 650 ---------- src/sqlbuild/cli/commands/types.py | 2 +- .../main/playground/test_output_encoding.py | 2 +- .../main/playground/test_playground.py | 2 +- .../dagster/test_sqlbuild_assets.py | 2 +- .../rivers/test_sqlbuild_assets.py | 2 +- .../_helpers/refactor/test_command.py | 3 +- .../main/test_compile_startup_imports.py | 50 - .../cli/commands/main/test_load_command.py | 14 +- .../main/test_startup_import_footprint.py | 1 - .../diff/test_query_artifact_cleanup.py | 3 +- .../_helpers/build_planning/helpers.py | 2 +- .../cli/commands/_helpers/cost/helpers.py | 2 +- .../commands/_helpers/cost/test_refresh.py | 8 +- .../commands/_helpers/scope/test_output.py | 17 +- .../commands/_helpers/scope/test_target.py | 3 +- .../compile/test_compile_import_footprint.py | 1 - .../cli/commands/main/dbt/test_init.py | 2 +- .../cli/commands/main/entry/test_main.py | 8 +- .../cli/commands/main/entry/test_scope.py | 9 +- .../execution/test_build_process_resources.py | 3 +- .../main/execution/test_scenario_command.py | 7 +- .../commands/main/inspection/cost/helpers.py | 2 +- .../main/playground/test_playground.py | 7 +- .../cli/commands/main/scope/test_scope.py | 3 +- .../commands/shared/_helpers/test_progress.py | 2 +- 47 files changed, 2165 insertions(+), 830 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs delete mode 100644 src/sqlbuild/cli/commands/models.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 47223516b4..3bd77ad282 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -11,7 +11,7 @@ pub(crate) const DBT_COMMAND_NAME: &str = "dbt"; pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init"; pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; @@ -54,7 +54,7 @@ const SNAPSHOT_LIMIT_FIELDS: [Field; 5] = [ const SNAPSHOT_LIMITS: Field = field( "limit_inputs", Source::Nested( - class(COMMAND_MODELS, "ScenarioSnapshotLimitInputs"), + class(NATIVE_RECORDS, "ScenarioSnapshotLimitInputs"), &SNAPSHOT_LIMIT_FIELDS, ), ); @@ -95,7 +95,7 @@ const SCOPE_FIELDS: [Field; 22] = [ NO_COLOR, ]; const SCOPE_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "ScopeCommandRequest"), + class(NATIVE_RECORDS, "ScopeCommandRequest"), &SCOPE_FIELDS, )]; pub(crate) const SCOPE_ROUTE: Route = routed( @@ -138,7 +138,7 @@ const PLAN_FIELDS: [Field; 19] = [ same("selection_diagnostics"), ]; const PLAN_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "PlanCommandRequest"), + class(NATIVE_RECORDS, "PlanCommandRequest"), &PLAN_FIELDS, )]; pub(crate) const PLAN_ROUTE: Route = routed( @@ -185,7 +185,7 @@ const BUILD_FIELDS: [Field; 34] = [ same("selection_diagnostics"), ]; const BUILD_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "BuildCommandRequest"), + class(NATIVE_RECORDS, "BuildCommandRequest"), &BUILD_FIELDS, )]; pub(crate) const BUILD_ROUTE: Route = routed( @@ -210,7 +210,7 @@ const FRESHNESS_FIELDS: [Field; 12] = [ same("fail_on_stale"), ]; const FRESHNESS_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "FreshnessCommandRequest"), + class(NATIVE_RECORDS, "FreshnessCommandRequest"), &FRESHNESS_FIELDS, )]; pub(crate) const FRESHNESS_ROUTE: Route = routed( @@ -235,7 +235,7 @@ const TEST_FIELDS: [Field; 12] = [ same("inspect"), ]; const TEST_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "TestCommandRequest"), + class(NATIVE_RECORDS, "TestCommandRequest"), &TEST_FIELDS, )]; pub(crate) const TEST_ROUTE: Route = routed( @@ -257,7 +257,7 @@ const CHECK_FIELDS: [Field; 9] = [ JSON_OUTPUT_PATH, ]; const CHECK_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "CheckCommandRequest"), + class(NATIVE_RECORDS, "CheckCommandRequest"), &CHECK_FIELDS, )]; pub(crate) const CHECK_ROUTE: Route = routed( @@ -281,7 +281,7 @@ const AUDIT_FIELDS: [Field; 11] = [ JSON_OUTPUT_PATH, ]; const AUDIT_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "AuditCommandRequest"), + class(NATIVE_RECORDS, "AuditCommandRequest"), &AUDIT_FIELDS, )]; pub(crate) const AUDIT_ROUTE: Route = routed( @@ -305,7 +305,7 @@ const LOAD_FIELDS: [Field; 11] = [ JSON_OUTPUT_PATH, ]; const LOAD_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "LoadCommandRequest"), + class(NATIVE_RECORDS, "LoadCommandRequest"), &LOAD_FIELDS, )]; pub(crate) const LOAD_ROUTE: Route = routed( @@ -327,7 +327,7 @@ const SEED_FIELDS: [Field; 9] = [ JSON_OUTPUT_PATH, ]; const SEED_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "SeedCommandRequest"), + class(NATIVE_RECORDS, "SeedCommandRequest"), &SEED_FIELDS, )]; pub(crate) const SEED_ROUTE: Route = routed( @@ -357,7 +357,7 @@ const LINEAGE_FIELDS: [Field; 11] = [ CLI_VARS, ]; const LINEAGE_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "LineageCommandRequest"), + class(NATIVE_RECORDS, "LineageCommandRequest"), &LINEAGE_FIELDS, )]; pub(crate) const LINEAGE_ROUTE: Route = routed( @@ -382,7 +382,7 @@ const CLONE_FIELDS: [Field; 12] = [ attr("event_output_path", "event_output"), ]; const CLONE_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "CloneCommandRequest"), + class(NATIVE_RECORDS, "CloneCommandRequest"), &CLONE_FIELDS, )]; pub(crate) const CLONE_ROUTE: Route = routed( @@ -433,7 +433,7 @@ const DIFF_FIELDS: [Field; 35] = [ same("full_example_values"), ]; const DIFF_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "DiffCommandRequest"), + class(NATIVE_RECORDS, "DiffCommandRequest"), &DIFF_FIELDS, )]; pub(crate) const DIFF_ROUTE: Route = routed( @@ -475,7 +475,7 @@ const COST_FIELDS: [Field; 11] = [ JSON_OUTPUT_PATH, ]; const COST_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "CostCommandRequest"), + class(NATIVE_RECORDS, "CostCommandRequest"), &COST_FIELDS, )]; pub(crate) const COST_ROUTE: Route = routed( @@ -516,7 +516,7 @@ const JANITOR_FIELDS: [Field; 8] = [ same("as_target"), ]; const JANITOR_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "JanitorCommandRequest"), + class(NATIVE_RECORDS, "JanitorCommandRequest"), &JANITOR_FIELDS, )]; pub(crate) const JANITOR_ROUTE: Route = routed( @@ -538,7 +538,7 @@ const REFACTOR_FIELDS: [Field; 9] = [ NO_COLOR, ]; const REFACTOR_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "RefactorCommandRequest"), + class(NATIVE_RECORDS, "RefactorCommandRequest"), &REFACTOR_FIELDS, )]; pub(crate) const REFACTOR_ROUTE: Route = Route { @@ -564,7 +564,7 @@ const CONTRACT_FIELDS: [Field; 10] = [ CLI_VARS, ]; const CONTRACT_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "ContractCommandRequest"), + class(NATIVE_RECORDS, "ContractCommandRequest"), &CONTRACT_FIELDS, )]; pub(crate) const CONTRACT_ROUTE: Route = Route { @@ -637,7 +637,7 @@ const PLAYGROUND_FIELDS: [Field; 3] = [ attr("template", "playground_template"), ]; const PLAYGROUND_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "PlaygroundCommandRequest"), + class(NATIVE_RECORDS, "PlaygroundCommandRequest"), &PLAYGROUND_FIELDS, )]; pub(crate) const PLAYGROUND_ROUTE: Route = routed( @@ -694,7 +694,7 @@ const RULES_FIELDS: [Field; 7] = [ EXCLUDE, ]; const RULES_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "RulesCommandRequest"), + class(NATIVE_RECORDS, "RulesCommandRequest"), &RULES_FIELDS, )]; pub(crate) const RULES_ROUTE: Route = routed( @@ -716,7 +716,7 @@ const DBT_INIT_FIELDS: [Field; 9] = [ same("skip_dbt_debug"), ]; const DBT_INIT_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "DbtInitCommandRequest"), + class(NATIVE_RECORDS, "DbtInitCommandRequest"), &DBT_INIT_FIELDS, )]; pub(crate) const DBT_INIT_ROUTE: Route = routed( @@ -769,7 +769,7 @@ const SCENARIO_TEST_FIELDS: [Field; 15] = [ JSON_OUTPUT_PATH, ]; const SCENARIO_TEST_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "ScenarioTestCommandRequest"), + class(NATIVE_RECORDS, "ScenarioTestCommandRequest"), &SCENARIO_TEST_FIELDS, )]; pub(crate) const SCENARIO_TEST_ROUTE: Route = routed( @@ -790,7 +790,7 @@ const SCENARIO_CAPTURE_FIELDS: [Field; 8] = [ SNAPSHOT_LIMITS, ]; const SCENARIO_CAPTURE_ARGUMENTS: [Field; 1] = [request( - class(COMMAND_MODELS, "ScenarioCaptureCommandRequest"), + class(NATIVE_RECORDS, "ScenarioCaptureCommandRequest"), &SCENARIO_CAPTURE_FIELDS, )]; pub(crate) const SCENARIO_CAPTURE_ROUTE: Route = routed( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index c7b6027704..a18e28ee47 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -484,7 +484,7 @@ fn complete( fn run_audit_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, - &imported(py, COMMAND_MODELS, "AuditInvocation")?, + &imported(py, NATIVE_RECORDS, "AuditInvocation")?, true, )?; let pipeline: Bound<'_, PyAny> = compile_audits(request, &invocation)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 78c78c97db..ba47db5aa5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::cost_report::degradation::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; @@ -76,7 +76,7 @@ pub(crate) fn finalize_build_cost<'py>( ]; fields.extend(cost.overrides); let finalization: Bound<'py, PyAny> = call_with( - &imported(py, COMMAND_MODELS, "BuildCostFinalization")?, + &imported(py, NATIVE_RECORDS, "BuildCostFinalization")?, &fields, )?; finalize_cost(&finalization) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index f50c97829f..ef46c37a05 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -196,7 +196,7 @@ pub(crate) fn resolve_build_context<'py>( load_sources.is_truthy()? }; let invocation: Bound<'py, PyAny> = call_with( - &imported(py, COMMAND_MODELS, "BuildInvocation")?, + &imported(py, NATIVE_RECORDS, "BuildInvocation")?, &[ ("effective_project_dir", project_dir), ("discovered_inputs", discovered), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index aec20540f2..838260c9c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; @@ -663,7 +663,7 @@ fn complete<'py>( fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, - &imported(py, COMMAND_MODELS, "CheckInvocation")?, + &imported(py, NATIVE_RECORDS, "CheckInvocation")?, true, )?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 83936d55a5..2aecfd0649 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -8,7 +8,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; /// The project directory, discovered inputs, adapter context, colour and progress stream. type InvocationContext<'py> = ( @@ -50,7 +50,7 @@ pub(crate) fn resolve_adapter_connection_context<'py>( ], )?; call_with( - &imported(py, COMMAND_MODELS, "AdapterConnectionContext")?, + &imported(py, NATIVE_RECORDS, "AdapterConnectionContext")?, &[ ("adapter_name", adapter_name), ("adapter", adapter), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 8a0d48b7df..6f5fa1bd1a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -264,7 +264,7 @@ fn run_dbt_sqlbuild_work<'py>( let project = compiled.getattr("project")?; let discovered = invocation.getattr("discovered_inputs")?; let context = call_with( - &imported(py, "sqlbuild.cli.commands.models", "DbtSqlbuildWorkContext")?, + &imported(py, "sqlbuild._native", "DbtSqlbuildWorkContext")?, &[ ("plan_output", args.value("plan_output")?), ("connection_config", args.value("connection_config")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 53267ead8b..5fe7f76bc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -27,13 +27,13 @@ const WORK_OPTIONS: (&[&str], &[&str]) = ( &[], ); const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; /// Interop has no allow flags; destructive work uses the ordinary interactive safety gates. #[pyfunction] fn dbt_interop_plan_safety_gates(py: Python<'_>) -> PyResult> { call_with( - &imported(py, COMMAND_MODELS, "PlanSafetyGates")?, + &imported(py, NATIVE_RECORDS, "PlanSafetyGates")?, &[ ("allow_missing_migration_origin", boolean(py, false)), ("allow_snapshot_full_refresh", boolean(py, false)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index fb2d82986a..420f59afef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -23,7 +23,7 @@ const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const NAMESPACE_MODULE: &str = "sqlbuild._native"; const RUNTIME_MODULE: &str = "sqlbuild._native"; const POLYGLOT_MODULE: &str = "sqlbuild.compiler.sql_analysis.main.import_polyglot_sql"; -const COMMAND_MODELS_MODULE: &str = "sqlbuild.cli.commands.models"; +const COMMAND_MODELS_MODULE: &str = "sqlbuild._native"; const LIFECYCLE_MODULE: &str = "sqlbuild.executor.diff.classes.query_artifact_lifecycle"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 578e2994a7..576dae6261 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; @@ -117,7 +117,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( ), ], )?; - imported(py, COMMAND_MODELS, "JanitorSettings")?.call((), Some(&options)) + imported(py, NATIVE_RECORDS, "JanitorSettings")?.call((), Some(&options)) } /// Ask for the exact confirmation phrase; an interrupt cancels. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 5b0a588488..ccc4eacfbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -63,6 +63,7 @@ pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod query_command; +pub(crate) mod records; pub(crate) mod refactor; pub(crate) mod references; pub(crate) mod render; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs new file mode 100644 index 0000000000..addd9c8db4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -0,0 +1,677 @@ +//! The layouts of every native record class. + +use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; + +pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The class attribute holding a record class's index into `RECORDS`. +pub(crate) const RECORD_SPEC_ATTRIBUTE: &str = "__native_record__"; +pub(crate) const FROZEN_INSTANCE_ERROR: (&str, &str) = ("dataclasses", "FrozenInstanceError"); + +/// Every native record group, in registration order. +pub(crate) const RECORDS: &[&[RecordSpec]] = &[COMMAND_RECORDS]; + +const NESTED_PROGRESS_CHILD_ROW: RecordSpec = RecordSpec { + name: "NestedProgressChildRow", + doc: "One child row rendered below a completed nested progress item.", + fields: &[ + FieldSpec::required("label"), + FieldSpec::required("name"), + FieldSpec::required("status_text"), + FieldSpec::new("detail", FieldDefault::Text("")), + ], +}; +const AUDIT_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "AuditCommandRequest", + doc: "CLI inputs for one audit command invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("defer_to", FieldDefault::None), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + ], +}; +const AUDIT_INVOCATION: RecordSpec = RecordSpec { + name: "AuditInvocation", + doc: "Resolved project, adapter, and reporter context for the audit command.", + fields: &[ + FieldSpec::required("effective_project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("use_color"), + FieldSpec::required("progress_stream"), + FieldSpec::required("connection_progress"), + FieldSpec::required("planning_progress"), + ], +}; +const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "BuildCommandRequest", + doc: "CLI inputs for one build command invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("defer_to", FieldDefault::None), + FieldSpec::new("defer_clone_from", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("fail_fast", FieldDefault::False), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("load_sources", FieldDefault::None), + FieldSpec::new("reload_sources", FieldDefault::False), + FieldSpec::new("include_python", FieldDefault::True), + FieldSpec::new("allow_snapshot_full_refresh", FieldDefault::False), + FieldSpec::new("allow_table_type_downgrade", FieldDefault::False), + FieldSpec::new("allow_retention_decrease", FieldDefault::False), + FieldSpec::new("allow_missing_migration_origin", FieldDefault::False), + FieldSpec::new("allow_snapshot_schema_change", FieldDefault::False), + FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("debug", FieldDefault::False), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("run_tests", FieldDefault::True), + FieldSpec::new("run_audits", FieldDefault::True), + FieldSpec::new("manifest", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + FieldSpec::new("event_output_path", FieldDefault::None), + FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("selection_diagnostics", FieldDefault::False), + FieldSpec::new("selector_files", FieldDefault::EmptyTuple), + ], +}; +const BUILD_INVOCATION: RecordSpec = RecordSpec { + name: "BuildInvocation", + doc: "Resolved project, adapter, and reporter context for the build command.", + fields: &[ + FieldSpec::required("effective_project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("effective_defer_clone_from"), + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("use_color"), + FieldSpec::required("progress_stream"), + FieldSpec::required("connection_progress"), + FieldSpec::required("planning_progress"), + FieldSpec::required("should_load_sources"), + FieldSpec::new("effective_target_name", FieldDefault::None), + FieldSpec::new( + "execution_limits", + FieldDefault::Call("sqlbuild.spec.contracts.models", "ExecutionLimitsConfig"), + ), + ], +}; +const CHECK_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "CheckCommandRequest", + doc: "CLI inputs for one check command invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + ], +}; +const CHECK_INVOCATION: RecordSpec = RecordSpec { + name: "CheckInvocation", + doc: "Resolved project, adapter, and reporter context for the check command.", + fields: &[ + FieldSpec::required("effective_project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("use_color"), + FieldSpec::required("progress_stream"), + FieldSpec::required("connection_progress"), + FieldSpec::required("planning_progress"), + ], +}; +const CLONE_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "CloneCommandRequest", + doc: "CLI inputs for one clone command invocation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("no_color"), + FieldSpec::required("no_sql_validation"), + FieldSpec::required("origin_target_name"), + FieldSpec::required("destination_target_name"), + FieldSpec::required("hard_copy"), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output_path", FieldDefault::None), + FieldSpec::new("event_output_path", FieldDefault::None), + ], +}; +const DBT_SQLBUILD_WORK_CONTEXT: RecordSpec = RecordSpec { + name: "DbtSqlbuildWorkContext", + doc: "Shared execution context for one dbt interop SQLBuild work run.", + fields: &[ + FieldSpec::required("plan_output"), + FieldSpec::required("connection_config"), + FieldSpec::required("adapter"), + FieldSpec::required("adapter_name"), + FieldSpec::required("output_stream"), + FieldSpec::required("use_color"), + FieldSpec::required("snapshots_config"), + FieldSpec::required("execution_limits"), + ], +}; +const PLAN_SAFETY_GATES: RecordSpec = RecordSpec { + name: "PlanSafetyGates", + doc: "Pre-confirmed plan safety gates and the guidance shown when one needs a terminal.", + fields: &[ + FieldSpec::required("allow_missing_migration_origin"), + FieldSpec::required("allow_snapshot_full_refresh"), + FieldSpec::required("allow_table_type_downgrade"), + FieldSpec::required("allow_retention_decrease"), + FieldSpec::required("missing_origin_help"), + FieldSpec::required("snapshot_full_refresh_help"), + FieldSpec::required("table_type_downgrade_help"), + FieldSpec::required("retention_decrease_help"), + FieldSpec::required("execution_limit_note"), + ], +}; +const DBT_INIT_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "DbtInitCommandRequest", + doc: "CLI inputs for one dbt init command invocation.", + fields: &[ + FieldSpec::required("cwd"), + FieldSpec::required("dbt_project_dir"), + FieldSpec::required("profiles_dir"), + FieldSpec::required("profile_name"), + FieldSpec::required("target_name"), + FieldSpec::required("sqb_output_dir"), + FieldSpec::required("dry_run"), + FieldSpec::required("overwrite"), + FieldSpec::required("skip_dbt_debug"), + ], +}; +const DIFF_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "DiffCommandRequest", + doc: "CLI inputs for one diff command invocation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("no_color"), + FieldSpec::required("no_sql_validation"), + FieldSpec::required("from_name"), + FieldSpec::required("to_name"), + FieldSpec::required("full"), + FieldSpec::required("schema_only"), + FieldSpec::required("bounded"), + FieldSpec::new("max_column_examples", FieldDefault::None), + FieldSpec::new("max_row_only_examples", FieldDefault::None), + FieldSpec::new("sample_rows", FieldDefault::None), + FieldSpec::new("sample_seed", FieldDefault::None), + FieldSpec::new("exhaustive", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + FieldSpec::new("max_models", FieldDefault::None), + FieldSpec::new("max_columns", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("left_query", FieldDefault::None), + FieldSpec::new("left_query_file", FieldDefault::None), + FieldSpec::new("left_label", FieldDefault::None), + FieldSpec::new("right_query", FieldDefault::None), + FieldSpec::new("right_query_file", FieldDefault::None), + FieldSpec::new("right_label", FieldDefault::None), + FieldSpec::new("unique_key_override", FieldDefault::EmptyTuple), + FieldSpec::new("unkeyed", FieldDefault::False), + FieldSpec::new("excluded_columns_override", FieldDefault::EmptyTuple), + FieldSpec::new("tolerance_overrides", FieldDefault::EmptyTuple), + FieldSpec::new("max_value_length", FieldDefault::None), + FieldSpec::new("suppress_example_values", FieldDefault::False), + FieldSpec::new("full_example_values", FieldDefault::False), + ], +}; +const QUERY_DIFF_PREPARATION: RecordSpec = RecordSpec { + name: "QueryDiffPreparation", + doc: "Resolved raw-query diff adapter, connection, SQL, and limits.", + fields: &[ + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("left_sql"), + FieldSpec::required("right_sql"), + FieldSpec::required("left_label"), + FieldSpec::required("right_label"), + FieldSpec::required("selected_target"), + FieldSpec::required("database"), + FieldSpec::required("schema"), + FieldSpec::required("run_id"), + FieldSpec::required("artifact_ttl"), + FieldSpec::required("effective_max_column_examples"), + FieldSpec::required("effective_max_row_only_examples"), + ], +}; +const FRESHNESS_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "FreshnessCommandRequest", + doc: "CLI inputs for one `sqb freshness` invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + FieldSpec::new("fail_on_error", FieldDefault::False), + FieldSpec::new("compare_state", FieldDefault::False), + FieldSpec::new("fail_on_stale", FieldDefault::False), + ], +}; +const JANITOR_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "JanitorCommandRequest", + doc: "CLI inputs for one janitor command invocation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("auto_approve", FieldDefault::False), + FieldSpec::new("retention_days", FieldDefault::None), + FieldSpec::new("direct_state_history_versions", FieldDefault::None), + FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("as_target", FieldDefault::None), + ], +}; +const JANITOR_INVOCATION: RecordSpec = RecordSpec { + name: "JanitorInvocation", + doc: "Resolved project discovery and output context for janitor.", + fields: &[ + FieldSpec::required("effective_project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("use_color"), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("active_target_name", FieldDefault::None), + FieldSpec::new("as_target", FieldDefault::None), + ], +}; +const JANITOR_SETTINGS: RecordSpec = RecordSpec { + name: "JanitorSettings", + doc: "Validated effective janitor settings.", + fields: &[ + FieldSpec::required("retention_days"), + FieldSpec::required("direct_state_history_versions"), + FieldSpec::new("archive_retention_days", FieldDefault::Int(14)), + FieldSpec::new("drop_old_name_views", FieldDefault::EmptyTuple), + ], +}; +const LOAD_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "LoadCommandRequest", + doc: "CLI inputs for one load command invocation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("reload", FieldDefault::False), + FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + ], +}; +const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "PlanCommandRequest", + doc: "CLI inputs for one plan command invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("defer_to", FieldDefault::None), + FieldSpec::new("defer_sources_to", FieldDefault::None), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("as_target", FieldDefault::None), + FieldSpec::new("cursor_overrides", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("load_sources", FieldDefault::None), + FieldSpec::new("include_python", FieldDefault::True), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("selection_diagnostics", FieldDefault::False), + ], +}; +const PLAYGROUND_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "PlaygroundCommandRequest", + doc: "CLI inputs for one playground command invocation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("target_path"), + FieldSpec::new("template", FieldDefault::Text("waffle_shop")), + ], +}; +const ADAPTER_CONNECTION_CONTEXT: RecordSpec = RecordSpec { + name: "AdapterConnectionContext", + doc: "Resolved adapter and connection configuration for one CLI command.", + fields: &[ + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + ], +}; +const SCENARIO_SNAPSHOT_LIMIT_INPUTS: RecordSpec = RecordSpec { + name: "ScenarioSnapshotLimitInputs", + doc: "CLI snapshot capture-limit overrides for one scenario run.", + fields: &[ + FieldSpec::new("max_snapshot_rows", FieldDefault::None), + FieldSpec::new("max_snapshot_total_rows", FieldDefault::None), + FieldSpec::new("max_snapshot_bytes", FieldDefault::None), + FieldSpec::new("max_snapshot_total_bytes", FieldDefault::None), + FieldSpec::new("force", FieldDefault::False), + ], +}; +const SCENARIO_TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "ScenarioTestCommandRequest", + doc: "CLI inputs for one `sqb scenario test` invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("scenario_namespace", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selectors", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("retain", FieldDefault::False), + FieldSpec::new("local", FieldDefault::False), + FieldSpec::new("strict", FieldDefault::False), + FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("sync_snapshots", FieldDefault::False), + FieldSpec::new("refresh", FieldDefault::False), + FieldSpec::new( + "limit_inputs", + FieldDefault::Call("sqlbuild._native", "ScenarioSnapshotLimitInputs"), + ), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + ], +}; +const SCENARIO_CAPTURE_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "ScenarioCaptureCommandRequest", + doc: "CLI inputs for one `sqb scenario capture` invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("scenario_namespace", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selectors", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("retain", FieldDefault::False), + FieldSpec::new( + "limit_inputs", + FieldDefault::Call("sqlbuild._native", "ScenarioSnapshotLimitInputs"), + ), + ], +}; +const SEED_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "SeedCommandRequest", + doc: "CLI inputs for one seed command invocation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + ], +}; +const SEED_INVOCATION: RecordSpec = RecordSpec { + name: "SeedInvocation", + doc: "Resolved project, adapter, connection, and output context for seed.", + fields: &[ + FieldSpec::required("effective_project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("use_color"), + FieldSpec::required("progress_stream"), + ], +}; +const TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "TestCommandRequest", + doc: "CLI inputs for one test command invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("json_output_path", FieldDefault::None), + FieldSpec::new("concurrency", FieldDefault::None), + FieldSpec::new("case_name", FieldDefault::None), + FieldSpec::new("inspect", FieldDefault::False), + ], +}; +const REFACTOR_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "RefactorCommandRequest", + doc: "CLI inputs for one `sqb rename` or `sqb mv` invocation.", + fields: &[ + FieldSpec::required("command"), + FieldSpec::required("target"), + FieldSpec::new("new_name", FieldDefault::None), + FieldSpec::new("destination", FieldDefault::None), + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("cascade", FieldDefault::False), + FieldSpec::new("dry_run", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + ], +}; +const SCOPE_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "ScopeCommandRequest", + doc: "CLI inputs for one offline declaration-scope inspection.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("at", FieldDefault::None), + FieldSpec::new("as_path", FieldDefault::None), + FieldSpec::new("browse", FieldDefault::None), + FieldSpec::new("list_path", FieldDefault::None), + FieldSpec::new("defined_under", FieldDefault::None), + FieldSpec::new("kinds", FieldDefault::EmptyTuple), + FieldSpec::new("match", FieldDefault::None), + FieldSpec::new("used_only", FieldDefault::False), + FieldSpec::new("include_nearby", FieldDefault::False), + FieldSpec::new("nearby_depth", FieldDefault::Int(1)), + FieldSpec::new("dependency_depth", FieldDefault::Int(0)), + FieldSpec::new("explain", FieldDefault::None), + FieldSpec::new("globals", FieldDefault::Text("summary")), + FieldSpec::new("page_size", FieldDefault::Int(100)), + FieldSpec::new("after", FieldDefault::None), + FieldSpec::new("paths", FieldDefault::Text("relative")), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + ], +}; +const TEST_INVOCATION: RecordSpec = RecordSpec { + name: "TestInvocation", + doc: "Resolved project, adapter, and reporter context for the test command.", + fields: &[ + FieldSpec::required("effective_project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("use_color"), + FieldSpec::required("progress_stream"), + FieldSpec::required("connection_progress"), + FieldSpec::required("planning_progress"), + ], +}; +const RULES_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "RulesCommandRequest", + doc: "Inputs for focused rule execution, inspection, or skill generation.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("json_output"), + FieldSpec::required("action"), + FieldSpec::required("rule_selector"), + FieldSpec::required("skills_check"), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + ], +}; +const COST_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "CostCommandRequest", + doc: "Inputs for local run-cost history and drill-down.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("selector"), + FieldSpec::required("no_color"), + FieldSpec::required("limit"), + FieldSpec::required("no_limit"), + FieldSpec::required("sort"), + FieldSpec::required("order"), + FieldSpec::required("since"), + FieldSpec::required("until"), + FieldSpec::required("json_output"), + FieldSpec::required("json_output_path"), + ], +}; +const BUILD_COST_FINALIZATION: RecordSpec = RecordSpec { + name: "BuildCostFinalization", + doc: "Inputs for nonfatal automatic build-cost finalization.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("connection_config"), + FieldSpec::required("target_name"), + FieldSpec::required("target_database"), + FieldSpec::required("run_id"), + FieldSpec::required("build_status"), + FieldSpec::required("started_at"), + FieldSpec::required("completed_at"), + FieldSpec::required("config"), + FieldSpec::required("output_stream"), + FieldSpec::required("use_color"), + FieldSpec::new("collect", FieldDefault::True), + FieldSpec::new("render", FieldDefault::True), + FieldSpec::new( + "cost_status", + FieldDefault::Member("sqlbuild.cost.types", "CostStatus", "COMPLETE"), + ), + FieldSpec::new( + "cost_message", + FieldDefault::Text("No executable warehouse work was planned for this build."), + ), + FieldSpec::new("had_executable_work", FieldDefault::True), + ], +}; +const CONTRACT_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "ContractCommandRequest", + doc: "CLI inputs for physical contract comparison and repository adoption.", + fields: &[ + FieldSpec::required("action"), + FieldSpec::required("from_target"), + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("write", FieldDefault::False), + FieldSpec::new("overwrite", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new("cli_vars", FieldDefault::None), + ], +}; +const LINEAGE_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "LineageCommandRequest", + doc: "CLI inputs for dependency and column lineage inspection.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("targets", FieldDefault::EmptyTuple), + FieldSpec::new("output_format", FieldDefault::Text("tree")), + FieldSpec::new("direction", FieldDefault::None), + FieldSpec::new("depth", FieldDefault::Text("all")), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new( + "lineage_mode", + FieldDefault::Member( + "sqlbuild.compiler.lineage.types", + "ColumnLineageMode", + "RICH", + ), + ), + FieldSpec::new("include_uses", FieldDefault::False), + FieldSpec::new("cli_vars", FieldDefault::None), + ], +}; + +/// `sqb` command requests and their resolved invocation contexts. +const COMMAND_RECORDS: &[RecordSpec] = &[ + NESTED_PROGRESS_CHILD_ROW, + AUDIT_COMMAND_REQUEST, + AUDIT_INVOCATION, + BUILD_COMMAND_REQUEST, + BUILD_INVOCATION, + CHECK_COMMAND_REQUEST, + CHECK_INVOCATION, + CLONE_COMMAND_REQUEST, + DBT_SQLBUILD_WORK_CONTEXT, + PLAN_SAFETY_GATES, + DBT_INIT_COMMAND_REQUEST, + DIFF_COMMAND_REQUEST, + QUERY_DIFF_PREPARATION, + FRESHNESS_COMMAND_REQUEST, + JANITOR_COMMAND_REQUEST, + JANITOR_INVOCATION, + JANITOR_SETTINGS, + LOAD_COMMAND_REQUEST, + PLAN_COMMAND_REQUEST, + PLAYGROUND_COMMAND_REQUEST, + ADAPTER_CONNECTION_CONTEXT, + SCENARIO_SNAPSHOT_LIMIT_INPUTS, + SCENARIO_TEST_COMMAND_REQUEST, + SCENARIO_CAPTURE_COMMAND_REQUEST, + SEED_COMMAND_REQUEST, + SEED_INVOCATION, + TEST_COMMAND_REQUEST, + REFACTOR_COMMAND_REQUEST, + SCOPE_COMMAND_REQUEST, + TEST_INVOCATION, + RULES_COMMAND_REQUEST, + COST_COMMAND_REQUEST, + BUILD_COST_FINALIZATION, + CONTRACT_COMMAND_REQUEST, + LINEAGE_COMMAND_REQUEST, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs new file mode 100644 index 0000000000..00b0d5f2fa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -0,0 +1,146 @@ +//! Construction, comparison and display of native record field values. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::{PyAttributeError, PyTypeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods, PyType, PyTypeMethods, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::constants::{ + FROZEN_INSTANCE_ERROR, RECORD_SPEC_ATTRIBUTE, RECORDS, +}; +use crate::bindings::_helpers::records::models::{FieldDefault, NativeRecord, RecordSpec}; + +/// Every registered record spec, in registration order. +pub(crate) fn all_specs() -> impl Iterator { + RECORDS.iter().flat_map(|group| group.iter()) +} + +/// The spec registered for `cls`, read from its class attribute. +pub(crate) fn record_spec(cls: &Bound<'_, PyType>) -> PyResult<&'static RecordSpec> { + let Ok(index) = cls.getattr(RECORD_SPEC_ATTRIBUTE) else { + return Err(PyTypeError::new_err(format!( + "{} cannot be instantiated directly", + cls.qualname()? + ))); + }; + let index: usize = index.extract()?; + all_specs() + .nth(index) + .ok_or_else(|| PyTypeError::new_err("unknown native record class")) +} + +/// Field values bound from `args` and `kwargs` as a dataclass `__init__` binds them. +pub(crate) fn build_values( + spec: &RecordSpec, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult>> { + let py: Python<'_> = args.py(); + let name: &str = spec.name; + if args.len() > spec.fields.len() { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() takes {} positional arguments but {} were given", + spec.fields.len() + 1, + args.len() + 1 + ))); + } + if let Some(kwargs) = kwargs { + for key in kwargs.keys() { + let key: String = key.extract()?; + if !spec.fields.iter().any(|field| field.name == key) { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() got an unexpected keyword argument '{key}'" + ))); + } + } + } + let mut values: Vec> = Vec::with_capacity(spec.fields.len()); + for (index, field) in spec.fields.iter().enumerate() { + let keyword: Option> = match kwargs { + Some(kwargs) => kwargs.get_item(field.name)?, + None => None, + }; + let positional: Option> = if index < args.len() { + Some(args.get_item(index)?) + } else { + None + }; + let value: Bound<'_, PyAny> = match (positional, keyword) { + (Some(_), Some(_)) => { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() got multiple values for argument '{}'", + field.name + ))); + } + (Some(value), None) | (None, Some(value)) => value, + (None, None) => default_value(py, name, field.name, field.default)?, + }; + values.push(value.unbind()); + } + Ok(values) +} + +fn default_value<'py>( + py: Python<'py>, + record: &str, + field: &str, + default: FieldDefault, +) -> PyResult> { + match default { + FieldDefault::Required => Err(PyTypeError::new_err(format!( + "{record}.__init__() missing 1 required positional argument: '{field}'" + ))), + FieldDefault::None => Ok(py.None().into_bound(py)), + FieldDefault::False => Ok(PyBool::new(py, false).to_owned().into_any()), + FieldDefault::True => Ok(PyBool::new(py, true).to_owned().into_any()), + FieldDefault::Int(value) => value.into_bound_py_any(py), + FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), + FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), + FieldDefault::Member(module, class, member) => imported(py, module, class)?.getattr(member), + FieldDefault::Call(module, name) => imported(py, module, name)?.call0(), + } +} + +/// The values of the fields that take part in equality, hashing and repr. +pub(crate) fn compared_values<'py>( + record: &Bound<'py, NativeRecord>, +) -> PyResult> { + let py: Python<'py> = record.py(); + let spec: &RecordSpec = record_spec(&record.get_type())?; + let values = spec + .fields + .iter() + .zip(record.get().values.iter()) + .filter(|(field, _)| field.compared) + .map(|(_, value)| value.clone_ref(py)) + .collect::>>(); + PyTuple::new(py, values) +} + +/// `Name(field=value!r, ...)` over the compared fields, as a dataclass repr reads. +pub(crate) fn repr_text(record: &Bound<'_, NativeRecord>) -> PyResult { + let py: Python<'_> = record.py(); + let cls: Bound<'_, PyType> = record.get_type(); + let spec: &RecordSpec = record_spec(&cls)?; + let mut parts: Vec = Vec::with_capacity(spec.fields.len()); + for (field, value) in spec.fields.iter().zip(record.get().values.iter()) { + if field.compared { + parts.push(format!("{}={}", field.name, value.bind(py).repr()?)); + } + } + Ok(format!("{}({})", cls.qualname()?, parts.join(", "))) +} + +/// The `dataclasses.FrozenInstanceError` raised when assigning to a record field. +pub(crate) fn frozen_error(py: Python<'_>, name: &str) -> PyErr { + let message: String = format!("cannot assign to field '{name}'"); + match imported(py, FROZEN_INSTANCE_ERROR.0, FROZEN_INSTANCE_ERROR.1) + .and_then(|error| error.call1((message.as_str(),))) + { + Ok(error) => PyErr::from_value(error), + Err(_) => PyAttributeError::new_err(message), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs new file mode 100644 index 0000000000..bd3477fe45 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod constants; +pub(crate) mod fields; +pub(crate) mod models; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs new file mode 100644 index 0000000000..fc8e397341 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -0,0 +1,146 @@ +//! Native frozen value records and the field layouts they are built from. + +use pyo3::exceptions::PyAttributeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple, PyType}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::records::fields::{ + build_values, compared_values, frozen_error, record_spec, repr_text, +}; + +/// How a record field is filled when the caller does not pass it. +#[derive(Debug, Clone, Copy)] +pub(crate) enum FieldDefault { + Required, + None, + False, + True, + Int(i64), + Text(&'static str), + EmptyTuple, + /// `getattr(getattr(import(module), class), member)`. + Member(&'static str, &'static str, &'static str), + /// `getattr(import(module), name)()`, evaluated for every instance. + Call(&'static str, &'static str), +} + +/// One record field: its name, default and whether it takes part in equality and repr. +#[derive(Debug, Clone, Copy)] +pub(crate) struct FieldSpec { + pub(crate) name: &'static str, + pub(crate) default: FieldDefault, + pub(crate) compared: bool, +} + +/// The layout of one native record class. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RecordSpec { + pub(crate) name: &'static str, + pub(crate) doc: &'static str, + pub(crate) fields: &'static [FieldSpec], +} + +/// Base of every native frozen record, built, compared, shown and pickled as a dataclass. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", subclass, frozen)] +pub(crate) struct NativeRecord { + pub(crate) values: Vec>, +} + +#[pymethods] +impl NativeRecord { + #[new] + #[classmethod] + #[pyo3(signature = (*args, **kwargs))] + fn new( + cls: &Bound<'_, PyType>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let spec: &RecordSpec = record_spec(cls)?; + Ok(Self { + values: build_values(spec, args, kwargs)?, + }) + } + + fn __eq__(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = slf.py(); + if !other.get_type().is(slf.get_type()) { + return Ok(py.NotImplemented()); + } + let other: &Bound<'_, Self> = other.cast::()?; + compared_values(slf)? + .as_any() + .eq(compared_values(other)?)? + .into_py_any(py) + } + + fn __hash__(slf: &Bound<'_, Self>) -> PyResult { + compared_values(slf)?.as_any().hash() + } + + fn __repr__(slf: &Bound<'_, Self>) -> PyResult { + repr_text(slf) + } + + fn __reduce__<'py>(slf: &Bound<'py, Self>) -> PyResult> { + let py: Python<'py> = slf.py(); + let values: Bound<'py, PyTuple> = PyTuple::new(py, slf.get().values.iter())?; + PyTuple::new(py, [slf.get_type().into_any(), values.into_any()]) + } +} + +/// Read-only access to one field of a native record. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct RecordAttribute { + pub(crate) index: usize, + pub(crate) name: &'static str, +} + +#[pymethods] +impl RecordAttribute { + fn __get__( + slf: &Bound<'_, Self>, + instance: &Bound<'_, PyAny>, + _owner: &Bound<'_, PyAny>, + ) -> PyResult> { + let py: Python<'_> = slf.py(); + if instance.is_none() { + return Ok(slf.clone().into_any().unbind()); + } + let record: &Bound<'_, NativeRecord> = instance.cast::()?; + let attribute: &Self = slf.get(); + record + .get() + .values + .get(attribute.index) + .map(|value| value.clone_ref(py)) + .ok_or_else(|| PyAttributeError::new_err(attribute.name)) + } + + fn __set__(&self, _instance: &Bound<'_, PyAny>, _value: &Bound<'_, PyAny>) -> PyResult<()> { + Err(frozen_error(_instance.py(), self.name)) + } + + fn __delete__(&self, _instance: &Bound<'_, PyAny>) -> PyResult<()> { + Err(frozen_error(_instance.py(), self.name)) + } +} + +impl FieldSpec { + /// A field every caller must pass. + pub(crate) const fn required(name: &'static str) -> Self { + Self::new(name, FieldDefault::Required) + } + + /// A field filled from `default` when it is not passed. + pub(crate) const fn new(name: &'static str, default: FieldDefault) -> Self { + Self { + name, + default, + compared: true, + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs new file mode 100644 index 0000000000..fcf81a0926 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -0,0 +1,52 @@ +//! Register the native record base, its field descriptor and every record class. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::records::constants::{NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE}; +use crate::bindings::_helpers::records::fields::all_specs; +use crate::bindings::_helpers::records::models::{NativeRecord, RecordAttribute, RecordSpec}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + let py: Python<'_> = module.py(); + module.add_class::()?; + module.add_class::()?; + let base: Bound<'_, PyAny> = py.get_type::().into_any(); + let metaclass: Bound<'_, PyAny> = base.get_type().into_any(); + for (index, spec) in all_specs().enumerate() { + module.add(spec.name, record_class(&metaclass, &base, (index, spec))?)?; + } + Ok(()) +} + +/// A `NativeRecord` subclass laid out as `spec`. +fn record_class<'py>( + metaclass: &Bound<'py, PyAny>, + base: &Bound<'py, PyAny>, + (index, spec): (usize, &'static RecordSpec), +) -> PyResult> { + let py: Python<'py> = base.py(); + let namespace: Bound<'py, PyDict> = PyDict::new(py); + namespace.set_item("__module__", NATIVE_MODULE)?; + namespace.set_item("__qualname__", spec.name)?; + namespace.set_item("__doc__", spec.doc)?; + namespace.set_item("__slots__", PyTuple::empty(py))?; + namespace.set_item(RECORD_SPEC_ATTRIBUTE, index)?; + let names: Vec> = spec + .fields + .iter() + .map(|field| PyString::new(py, field.name)) + .collect(); + namespace.set_item("__match_args__", PyTuple::new(py, &names)?)?; + for (position, field) in spec.fields.iter().enumerate() { + let attribute: Py = Py::new( + py, + RecordAttribute { + index: position, + name: field.name, + }, + )?; + namespace.set_item(field.name, attribute)?; + } + metaclass.call1((spec.name, (base,), namespace)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 2b26d2a393..d9727ea938 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -19,7 +19,7 @@ use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hook use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -416,7 +416,7 @@ fn load_seeds<'py>( #[pyo3(signature = (*, request))] fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let invocation: Bound<'_, PyAny> = - resolve_command_invocation(request, &imported(py, COMMAND_MODELS, "SeedInvocation")?)?; + resolve_command_invocation(request, &imported(py, NATIVE_RECORDS, "SeedInvocation")?)?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; let concurrency: i64 = if requested.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index fcc0b9a8e4..2bf64631a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -32,7 +32,7 @@ use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runti use crate::bindings::_helpers::result_output::integration_values::json_value; const NATIVE_MODULE: &str = "sqlbuild._native"; -const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -153,7 +153,7 @@ fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { /// Nested `expect` rows of a completed test, one per step. fn expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = result.py(); - let row_class: Bound<'py, PyAny> = imported(py, COMMAND_MODELS, "NestedProgressChildRow")?; + let row_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS, "NestedProgressChildRow")?; let name: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_name")?; let detail: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_detail")?; let mut rows: Vec> = Vec::new(); @@ -636,7 +636,7 @@ fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, - &imported(py, COMMAND_MODELS, "TestInvocation")?, + &imported(py, NATIVE_RECORDS, "TestInvocation")?, !inspect_only, )?; let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 1f4e40a899..4c229c0e09 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -8,13 +8,14 @@ use crate::bindings::_helpers::{ compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, cursor, dag_artifact, diagnostics, discovery, fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, plan_output, planner_registration, - project_assembly, refactor, references, render, rules, scopes, semantic_checks, + project_assembly, records, refactor, references, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, state_rows, type_system, }; use crate::bindings::models; #[pymodule] pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { + records::registration::register(module)?; rules::registration::register(module)?; cli::registration::register(module)?; module.add( diff --git a/fensu.toml b/fensu.toml index c51d431272..d80e5e7f3a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -23,11 +23,6 @@ exclude = ["tests/e2e/fixtures/**"] [targets.python.thresholds] max_positional_args = 1 -[[targets.python.rule_exceptions]] -rule = "FFR001" -path = "src/sqlbuild/cli/commands/models.py" -reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/lint/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1b8978598b..0a3f3823a3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -27,34 +27,6 @@ from sqlbuild.adapter.contract.types import ( TypeDialect, ) from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.commands.models import ( - AuditCommandRequest, - BuildCommandRequest, - BuildCostFinalization, - CheckCommandRequest, - CloneCommandRequest, - ContractCommandRequest, - CostCommandRequest, - DbtInitCommandRequest, - DbtSqlbuildWorkContext, - DiffCommandRequest, - FreshnessCommandRequest, - JanitorCommandRequest, - LineageCommandRequest, - LoadCommandRequest, - NestedProgressChildRow, - PlanCommandRequest, - PlanSafetyGates, - PlaygroundCommandRequest, - QueryDiffPreparation, - RefactorCommandRequest, - RulesCommandRequest, - ScenarioCaptureCommandRequest, - ScenarioTestCommandRequest, - ScopeCommandRequest, - SeedCommandRequest, - TestCommandRequest, -) from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader from sqlbuild.cli.compile.models import CompileAnalysis from sqlbuild.cli.compile_reuse.models import CompileReuseRequest @@ -65,6 +37,7 @@ from sqlbuild.cli.entry.models import ( CompileProfileFlags, ParsedCliInvocation, ResolvedWarehouse, + SelectorFileSummary, ) from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget from sqlbuild.compiler.auditing.models import ( @@ -329,6 +302,7 @@ from sqlbuild.cost.models import ( StatementExecutionTelemetry, StatementLedgerEntry, ) +from sqlbuild.cost.types import CostStatus from sqlbuild.cursor_algebra.models import ( AlignedInterval, DateValue, @@ -417,6 +391,7 @@ from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( + CostConfig, DbtConfig, DefaultsConfig, ExecutionLimitsConfig, @@ -7430,3 +7405,1036 @@ def render_snowflake_query_history_sql( ) -> str: ... def snowflake_statement_is_metadata_only(*, query_type: object, query_text: object) -> bool: ... def parse_sqlbuild_query_tag(value: object) -> dict[str, str | None] | None: ... + +class NestedProgressChildRow: + """One child row rendered below a completed nested progress item.""" + + label: str + name: str + status_text: str + detail: str + def __init__( + self, + label: str, + name: str, + status_text: str, + detail: str = "", + ) -> None: ... + +class AuditCommandRequest: + """CLI inputs for one audit command invocation.""" + + project_dir: Path | None + no_sql_validation: bool + defer_to: str | None + no_color: bool + selected_target: str | None + concurrency: int | None + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + json_output: bool + json_output_path: Path | None + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + defer_to: str | None = None, + no_color: bool = False, + selected_target: str | None = None, + concurrency: int | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + cli_vars: dict[str, object] | None = None, + json_output: bool = False, + json_output_path: Path | None = None, + ) -> None: ... + +class AuditInvocation: + """Resolved project, adapter, and reporter context for the audit command.""" + + effective_project_dir: Path + discovered_inputs: DiscoveredProjectInputs + adapter_name: str + adapter: BaseAdapter + connection_config: dict[str, object] + use_color: bool + progress_stream: TextIO + connection_progress: ConnectionProgressReporter + planning_progress: PlanningProgressReporter + def __init__( + self, + effective_project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + adapter_name: str, + adapter: BaseAdapter, + connection_config: dict[str, object], + use_color: bool, + progress_stream: TextIO, + connection_progress: ConnectionProgressReporter, + planning_progress: PlanningProgressReporter, + ) -> None: ... + +class BuildCommandRequest: + """CLI inputs for one build command invocation.""" + + project_dir: Path | None + no_sql_validation: bool + defer_to: str | None + defer_clone_from: str | None + defer_sources_to: str | None + selected_target: str | None + cursor_overrides: CursorOverrides | None + no_color: bool + fail_fast: bool + full_refresh: bool + load_sources: bool | None + reload_sources: bool + include_python: bool + allow_snapshot_full_refresh: bool + allow_table_type_downgrade: bool + allow_retention_decrease: bool + allow_missing_migration_origin: bool + allow_snapshot_schema_change: bool + concurrency: int | None + select: tuple[str, ...] + exclude: tuple[str, ...] + verbose: bool + debug: bool + cli_vars: dict[str, object] | None + run_tests: bool + run_audits: bool + manifest: bool + json_output: bool + json_output_path: Path | None + event_output_path: Path | None + no_cache: bool + max_microbatches: int | None + selection_diagnostics: bool + selector_files: tuple[SelectorFileSummary, ...] + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + defer_to: str | None = None, + defer_clone_from: str | None = None, + defer_sources_to: str | None = None, + selected_target: str | None = None, + cursor_overrides: CursorOverrides | None = None, + no_color: bool = False, + fail_fast: bool = False, + full_refresh: bool = False, + load_sources: bool | None = None, + reload_sources: bool = False, + include_python: bool = True, + allow_snapshot_full_refresh: bool = False, + allow_table_type_downgrade: bool = False, + allow_retention_decrease: bool = False, + allow_missing_migration_origin: bool = False, + allow_snapshot_schema_change: bool = False, + concurrency: int | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + verbose: bool = False, + debug: bool = False, + cli_vars: dict[str, object] | None = None, + run_tests: bool = True, + run_audits: bool = True, + manifest: bool = False, + json_output: bool = False, + json_output_path: Path | None = None, + event_output_path: Path | None = None, + no_cache: bool = False, + max_microbatches: int | None = None, + selection_diagnostics: bool = False, + selector_files: tuple[SelectorFileSummary, ...] = (), + ) -> None: ... + +class BuildInvocation: + """Resolved project, adapter, and reporter context for the build command.""" + + effective_project_dir: Path + discovered_inputs: DiscoveredProjectInputs + effective_defer_clone_from: str | None + adapter_name: str + adapter: BaseAdapter + connection_config: dict[str, object] + use_color: bool + progress_stream: TextIO + connection_progress: ConnectionProgressReporter + planning_progress: PlanningProgressReporter + should_load_sources: bool + effective_target_name: str | None + execution_limits: ExecutionLimitsConfig + def __init__( + self, + effective_project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + effective_defer_clone_from: str | None, + adapter_name: str, + adapter: BaseAdapter, + connection_config: dict[str, object], + use_color: bool, + progress_stream: TextIO, + connection_progress: ConnectionProgressReporter, + planning_progress: PlanningProgressReporter, + should_load_sources: bool, + effective_target_name: str | None = None, + execution_limits: ExecutionLimitsConfig = ..., + ) -> None: ... + +class CheckCommandRequest: + """CLI inputs for one check command invocation.""" + + project_dir: Path | None + no_sql_validation: bool + no_color: bool + selected_target: str | None + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + json_output: bool + json_output_path: Path | None + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + no_color: bool = False, + selected_target: str | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + cli_vars: dict[str, object] | None = None, + json_output: bool = False, + json_output_path: Path | None = None, + ) -> None: ... + +class CheckInvocation: + """Resolved project, adapter, and reporter context for the check command.""" + + effective_project_dir: Path + discovered_inputs: DiscoveredProjectInputs + adapter_name: str + adapter: BaseAdapter + connection_config: dict[str, object] + use_color: bool + progress_stream: TextIO + connection_progress: ConnectionProgressReporter + planning_progress: PlanningProgressReporter + def __init__( + self, + effective_project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + adapter_name: str, + adapter: BaseAdapter, + connection_config: dict[str, object], + use_color: bool, + progress_stream: TextIO, + connection_progress: ConnectionProgressReporter, + planning_progress: PlanningProgressReporter, + ) -> None: ... + +class CloneCommandRequest: + """CLI inputs for one clone command invocation.""" + + project_dir: Path | None + no_color: bool + no_sql_validation: bool + origin_target_name: str + destination_target_name: str | None + hard_copy: bool + select: tuple[str, ...] + exclude: tuple[str, ...] + verbose: bool + cli_vars: dict[str, object] | None + json_output_path: Path | None + event_output_path: Path | None + def __init__( + self, + project_dir: Path | None, + no_color: bool, + no_sql_validation: bool, + origin_target_name: str, + destination_target_name: str | None, + hard_copy: bool, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + verbose: bool = False, + cli_vars: dict[str, object] | None = None, + json_output_path: Path | None = None, + event_output_path: Path | None = None, + ) -> None: ... + +class DbtSqlbuildWorkContext: + """Shared execution context for one dbt interop SQLBuild work run.""" + + plan_output: PlanOutput + connection_config: dict[str, object] + adapter: BaseAdapter + adapter_name: str + output_stream: TextIO + use_color: bool + snapshots_config: SnapshotsConfig + execution_limits: ExecutionLimitsConfig + def __init__( + self, + plan_output: PlanOutput, + connection_config: dict[str, object], + adapter: BaseAdapter, + adapter_name: str, + output_stream: TextIO, + use_color: bool, + snapshots_config: SnapshotsConfig, + execution_limits: ExecutionLimitsConfig, + ) -> None: ... + +class PlanSafetyGates: + """Pre-confirmed plan safety gates and the guidance shown when one needs a terminal.""" + + allow_missing_migration_origin: bool + allow_snapshot_full_refresh: bool + allow_table_type_downgrade: bool + allow_retention_decrease: bool + missing_origin_help: str + snapshot_full_refresh_help: str + table_type_downgrade_help: str + retention_decrease_help: str + execution_limit_note: str + def __init__( + self, + allow_missing_migration_origin: bool, + allow_snapshot_full_refresh: bool, + allow_table_type_downgrade: bool, + allow_retention_decrease: bool, + missing_origin_help: str, + snapshot_full_refresh_help: str, + table_type_downgrade_help: str, + retention_decrease_help: str, + execution_limit_note: str, + ) -> None: ... + +class DbtInitCommandRequest: + """CLI inputs for one dbt init command invocation.""" + + cwd: Path + dbt_project_dir: str | None + profiles_dir: str | None + profile_name: str | None + target_name: str | None + sqb_output_dir: str | None + dry_run: bool + overwrite: bool + skip_dbt_debug: bool + def __init__( + self, + cwd: Path, + dbt_project_dir: str | None, + profiles_dir: str | None, + profile_name: str | None, + target_name: str | None, + sqb_output_dir: str | None, + dry_run: bool, + overwrite: bool, + skip_dbt_debug: bool, + ) -> None: ... + +class DiffCommandRequest: + """CLI inputs for one diff command invocation.""" + + project_dir: Path | None + no_color: bool + no_sql_validation: bool + from_name: str | None + to_name: str | None + full: bool + schema_only: bool + bounded: str | None + max_column_examples: int | None + max_row_only_examples: int | None + sample_rows: int | None + sample_seed: int | None + exhaustive: bool + json_output: bool + json_output_path: Path | None + max_models: int | None + max_columns: int | None + select: tuple[str, ...] + exclude: tuple[str, ...] + verbose: bool + cli_vars: dict[str, object] | None + selected_target: str | None + left_query: str | None + left_query_file: Path | None + left_label: str | None + right_query: str | None + right_query_file: Path | None + right_label: str | None + unique_key_override: tuple[str, ...] + unkeyed: bool + excluded_columns_override: tuple[str, ...] + tolerance_overrides: tuple[str, ...] + max_value_length: int | None + suppress_example_values: bool + full_example_values: bool + def __init__( + self, + project_dir: Path | None, + no_color: bool, + no_sql_validation: bool, + from_name: str | None, + to_name: str | None, + full: bool, + schema_only: bool, + bounded: str | None, + max_column_examples: int | None = None, + max_row_only_examples: int | None = None, + sample_rows: int | None = None, + sample_seed: int | None = None, + exhaustive: bool = False, + json_output: bool = False, + json_output_path: Path | None = None, + max_models: int | None = None, + max_columns: int | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + verbose: bool = False, + cli_vars: dict[str, object] | None = None, + selected_target: str | None = None, + left_query: str | None = None, + left_query_file: Path | None = None, + left_label: str | None = None, + right_query: str | None = None, + right_query_file: Path | None = None, + right_label: str | None = None, + unique_key_override: tuple[str, ...] = (), + unkeyed: bool = False, + excluded_columns_override: tuple[str, ...] = (), + tolerance_overrides: tuple[str, ...] = (), + max_value_length: int | None = None, + suppress_example_values: bool = False, + full_example_values: bool = False, + ) -> None: ... + +class QueryDiffPreparation: + """Resolved raw-query diff adapter, connection, SQL, and limits.""" + + adapter: BaseAdapter + connection_config: dict[str, object] + left_sql: str + right_sql: str + left_label: str + right_label: str + selected_target: str | None + database: str | None + schema: str + run_id: str + artifact_ttl: str + effective_max_column_examples: int + effective_max_row_only_examples: int + def __init__( + self, + adapter: BaseAdapter, + connection_config: dict[str, object], + left_sql: str, + right_sql: str, + left_label: str, + right_label: str, + selected_target: str | None, + database: str | None, + schema: str, + run_id: str, + artifact_ttl: str, + effective_max_column_examples: int, + effective_max_row_only_examples: int, + ) -> None: ... + +class FreshnessCommandRequest: + """CLI inputs for one `sqb freshness` invocation.""" + + project_dir: Path | None + no_sql_validation: bool + no_color: bool + selected_target: str | None + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + json_output: bool + json_output_path: Path | None + fail_on_error: bool + compare_state: bool + fail_on_stale: bool + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + no_color: bool = False, + selected_target: str | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + cli_vars: dict[str, object] | None = None, + json_output: bool = False, + json_output_path: Path | None = None, + fail_on_error: bool = False, + compare_state: bool = False, + fail_on_stale: bool = False, + ) -> None: ... + +class JanitorCommandRequest: + """CLI inputs for one janitor command invocation.""" + + project_dir: Path | None + no_color: bool + auto_approve: bool + retention_days: int | None + direct_state_history_versions: int | None + drop_old_name_views: tuple[str, ...] + selected_target: str | None + as_target: str | None + def __init__( + self, + project_dir: Path | None, + no_color: bool = False, + auto_approve: bool = False, + retention_days: int | None = None, + direct_state_history_versions: int | None = None, + drop_old_name_views: tuple[str, ...] = (), + selected_target: str | None = None, + as_target: str | None = None, + ) -> None: ... + +class JanitorInvocation: + """Resolved project discovery and output context for janitor.""" + + effective_project_dir: Path + discovered_inputs: DiscoveredProjectInputs + use_color: bool + selected_target: str | None + active_target_name: str | None + as_target: str | None + def __init__( + self, + effective_project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + use_color: bool, + selected_target: str | None = None, + active_target_name: str | None = None, + as_target: str | None = None, + ) -> None: ... + +class JanitorSettings: + """Validated effective janitor settings.""" + + retention_days: int + direct_state_history_versions: int + archive_retention_days: int + drop_old_name_views: tuple[str, ...] + def __init__( + self, + retention_days: int, + direct_state_history_versions: int, + archive_retention_days: int = 14, + drop_old_name_views: tuple[str, ...] = (), + ) -> None: ... + +class LoadCommandRequest: + """CLI inputs for one load command invocation.""" + + project_dir: Path | None + no_color: bool + selected_target: str | None + select: tuple[str, ...] + exclude: tuple[str, ...] + reload: bool + concurrency: int | None + cursor_overrides: CursorOverrides | None + cli_vars: dict[str, object] | None + json_output: bool + json_output_path: Path | None + def __init__( + self, + project_dir: Path | None, + no_color: bool = False, + selected_target: str | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + reload: bool = False, + concurrency: int | None = None, + cursor_overrides: CursorOverrides | None = None, + cli_vars: dict[str, object] | None = None, + json_output: bool = False, + json_output_path: Path | None = None, + ) -> None: ... + +class PlanCommandRequest: + """CLI inputs for one plan command invocation.""" + + project_dir: Path | None + no_sql_validation: bool + defer_to: str | None + defer_sources_to: str | None + selected_target: str | None + as_target: str | None + cursor_overrides: CursorOverrides | None + json_output: bool + full_refresh: bool + load_sources: bool | None + include_python: bool + no_color: bool + select: tuple[str, ...] + exclude: tuple[str, ...] + verbose: bool + cli_vars: dict[str, object] | None + no_cache: bool + max_microbatches: int | None + selection_diagnostics: bool + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + defer_to: str | None = None, + defer_sources_to: str | None = None, + selected_target: str | None = None, + as_target: str | None = None, + cursor_overrides: CursorOverrides | None = None, + json_output: bool = False, + full_refresh: bool = False, + load_sources: bool | None = None, + include_python: bool = True, + no_color: bool = False, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + verbose: bool = False, + cli_vars: dict[str, object] | None = None, + no_cache: bool = False, + max_microbatches: int | None = None, + selection_diagnostics: bool = False, + ) -> None: ... + +class PlaygroundCommandRequest: + """CLI inputs for one playground command invocation.""" + + project_dir: Path | None + target_path: str + template: str + def __init__( + self, + project_dir: Path | None, + target_path: str, + template: str = ..., + ) -> None: ... + +class AdapterConnectionContext: + """Resolved adapter and connection configuration for one CLI command.""" + + adapter_name: str + adapter: BaseAdapter + connection_config: dict[str, object] + def __init__( + self, + adapter_name: str, + adapter: BaseAdapter, + connection_config: dict[str, object], + ) -> None: ... + +class ScenarioSnapshotLimitInputs: + """CLI snapshot capture-limit overrides for one scenario run.""" + + max_snapshot_rows: int | None + max_snapshot_total_rows: int | None + max_snapshot_bytes: int | None + max_snapshot_total_bytes: int | None + force: bool + def __init__( + self, + max_snapshot_rows: int | None = None, + max_snapshot_total_rows: int | None = None, + max_snapshot_bytes: int | None = None, + max_snapshot_total_bytes: int | None = None, + force: bool = False, + ) -> None: ... + +class ScenarioTestCommandRequest: + """CLI inputs for one `sqb scenario test` invocation.""" + + project_dir: Path | None + scenario_namespace: str | None + no_sql_validation: bool + no_color: bool + selectors: tuple[str, ...] + exclude: tuple[str, ...] + retain: bool + local: bool + strict: bool + concurrency: int | None + sync_snapshots: bool + refresh: bool + limit_inputs: ScenarioSnapshotLimitInputs + json_output: bool + json_output_path: Path | None + def __init__( + self, + project_dir: Path | None = None, + scenario_namespace: str | None = None, + no_sql_validation: bool = False, + no_color: bool = False, + selectors: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + retain: bool = False, + local: bool = False, + strict: bool = False, + concurrency: int | None = None, + sync_snapshots: bool = False, + refresh: bool = False, + limit_inputs: ScenarioSnapshotLimitInputs = ..., + json_output: bool = False, + json_output_path: Path | None = None, + ) -> None: ... + +class ScenarioCaptureCommandRequest: + """CLI inputs for one `sqb scenario capture` invocation.""" + + project_dir: Path | None + scenario_namespace: str | None + no_sql_validation: bool + no_color: bool + selectors: tuple[str, ...] + exclude: tuple[str, ...] + retain: bool + limit_inputs: ScenarioSnapshotLimitInputs + def __init__( + self, + project_dir: Path | None = None, + scenario_namespace: str | None = None, + no_sql_validation: bool = False, + no_color: bool = False, + selectors: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + retain: bool = False, + limit_inputs: ScenarioSnapshotLimitInputs = ..., + ) -> None: ... + +class SeedCommandRequest: + """CLI inputs for one seed command invocation.""" + + project_dir: Path | None + no_color: bool + selected_target: str | None + select: tuple[str, ...] + exclude: tuple[str, ...] + concurrency: int | None + cli_vars: dict[str, object] | None + json_output: bool + json_output_path: Path | None + def __init__( + self, + project_dir: Path | None, + no_color: bool = False, + selected_target: str | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + concurrency: int | None = None, + cli_vars: dict[str, object] | None = None, + json_output: bool = False, + json_output_path: Path | None = None, + ) -> None: ... + +class SeedInvocation: + """Resolved project, adapter, connection, and output context for seed.""" + + effective_project_dir: Path + discovered_inputs: DiscoveredProjectInputs + adapter_name: str + adapter: BaseAdapter + connection_config: dict[str, object] + use_color: bool + progress_stream: TextIO + def __init__( + self, + effective_project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + adapter_name: str, + adapter: BaseAdapter, + connection_config: dict[str, object], + use_color: bool, + progress_stream: TextIO, + ) -> None: ... + +class TestCommandRequest: + """CLI inputs for one test command invocation.""" + + project_dir: Path | None + no_sql_validation: bool + no_color: bool + selected_target: str | None + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + json_output: bool + json_output_path: Path | None + concurrency: int | None + case_name: str | None + inspect: bool + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + no_color: bool = False, + selected_target: str | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + cli_vars: dict[str, object] | None = None, + json_output: bool = False, + json_output_path: Path | None = None, + concurrency: int | None = None, + case_name: str | None = None, + inspect: bool = False, + ) -> None: ... + +class RefactorCommandRequest: + """CLI inputs for one `sqb rename` or `sqb mv` invocation.""" + + command: str + target: str + new_name: str | None + destination: str | None + project_dir: Path | None + cascade: bool + dry_run: bool + json_output: bool + no_color: bool + def __init__( + self, + command: str, + target: str, + new_name: str | None = None, + destination: str | None = None, + project_dir: Path | None = None, + cascade: bool = False, + dry_run: bool = False, + json_output: bool = False, + no_color: bool = False, + ) -> None: ... + +class ScopeCommandRequest: + """CLI inputs for one offline declaration-scope inspection.""" + + project_dir: Path | None + target: str | None + at: str | None + as_path: str | None + browse: str | None + list_path: str | None + defined_under: str | None + kinds: tuple[str, ...] + match: str | None + used_only: bool + include_nearby: bool + nearby_depth: int + dependency_depth: int + explain: str | None + globals: str + page_size: int + after: str | None + paths: str + json_output: bool + no_cache: bool + verbose: bool + no_color: bool + def __init__( + self, + project_dir: Path | None = None, + target: str | None = None, + at: str | None = None, + as_path: str | None = None, + browse: str | None = None, + list_path: str | None = None, + defined_under: str | None = None, + kinds: tuple[str, ...] = (), + match: str | None = None, + used_only: bool = False, + include_nearby: bool = False, + nearby_depth: int = 1, + dependency_depth: int = 0, + explain: str | None = None, + globals: str = "summary", + page_size: int = 100, + after: str | None = None, + paths: str = "relative", + json_output: bool = False, + no_cache: bool = False, + verbose: bool = False, + no_color: bool = False, + ) -> None: ... + +class TestInvocation: + """Resolved project, adapter, and reporter context for the test command.""" + + effective_project_dir: Path + discovered_inputs: DiscoveredProjectInputs + adapter_name: str + adapter: BaseAdapter + connection_config: dict[str, object] + use_color: bool + progress_stream: TextIO + connection_progress: ConnectionProgressReporter + planning_progress: PlanningProgressReporter + def __init__( + self, + effective_project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + adapter_name: str, + adapter: BaseAdapter, + connection_config: dict[str, object], + use_color: bool, + progress_stream: TextIO, + connection_progress: ConnectionProgressReporter, + planning_progress: PlanningProgressReporter, + ) -> None: ... + +class RulesCommandRequest: + """Inputs for focused rule execution, inspection, or skill generation.""" + + project_dir: Path | None + json_output: bool + action: str + rule_selector: str | None + skills_check: bool + select: tuple[str, ...] + exclude: tuple[str, ...] + def __init__( + self, + project_dir: Path | None, + json_output: bool, + action: str, + rule_selector: str | None, + skills_check: bool, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + ) -> None: ... + +class CostCommandRequest: + """Inputs for local run-cost history and drill-down.""" + + project_dir: Path | None + selector: str + no_color: bool + limit: int | None + no_limit: bool + sort: str | None + order: str | None + since: str | None + until: str | None + json_output: bool + json_output_path: Path | None + def __init__( + self, + project_dir: Path | None, + selector: str, + no_color: bool, + limit: int | None, + no_limit: bool, + sort: str | None, + order: str | None, + since: str | None, + until: str | None, + json_output: bool, + json_output_path: Path | None, + ) -> None: ... + +class BuildCostFinalization: + """Inputs for nonfatal automatic build-cost finalization.""" + + project_dir: Path + adapter_name: str + adapter: object + connection_config: dict[str, object] + target_name: str | None + target_database: str | None + run_id: str + build_status: str + started_at: datetime + completed_at: datetime + config: CostConfig + output_stream: TextIO + use_color: bool + collect: bool + render: bool + cost_status: CostStatus + cost_message: str + had_executable_work: bool | None + def __init__( + self, + project_dir: Path, + adapter_name: str, + adapter: object, + connection_config: dict[str, object], + target_name: str | None, + target_database: str | None, + run_id: str, + build_status: str, + started_at: datetime, + completed_at: datetime, + config: CostConfig, + output_stream: TextIO, + use_color: bool, + collect: bool = True, + render: bool = True, + cost_status: CostStatus = ..., + cost_message: str = ..., + had_executable_work: bool | None = True, + ) -> None: ... + +class ContractCommandRequest: + """CLI inputs for physical contract comparison and repository adoption.""" + + action: str + from_target: str + project_dir: Path | None + select: tuple[str, ...] + exclude: tuple[str, ...] + write: bool + overwrite: bool + json_output: bool + no_color: bool + cli_vars: dict[str, object] | None + def __init__( + self, + action: str, + from_target: str, + project_dir: Path | None = None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + write: bool = False, + overwrite: bool = False, + json_output: bool = False, + no_color: bool = False, + cli_vars: dict[str, object] | None = None, + ) -> None: ... + +class LineageCommandRequest: + """CLI inputs for dependency and column lineage inspection.""" + + project_dir: Path | None + no_sql_validation: bool + targets: tuple[str, ...] + output_format: str + direction: str | None + depth: str + select: tuple[str, ...] + exclude: tuple[str, ...] + lineage_mode: ColumnLineageMode + include_uses: bool + cli_vars: dict[str, object] | None + def __init__( + self, + project_dir: Path | None, + no_sql_validation: bool = False, + targets: tuple[str, ...] = (), + output_format: str = "tree", + direction: str | None = None, + depth: str = "all", + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + lineage_mode: ColumnLineageMode = ..., + include_uses: bool = False, + cli_vars: dict[str, object] | None = None, + ) -> None: ... diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py deleted file mode 100644 index 9fbfa50fa6..0000000000 --- a/src/sqlbuild/cli/commands/models.py +++ /dev/null @@ -1,650 +0,0 @@ -"""CLI command runtime models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime -from pathlib import Path -from typing import TYPE_CHECKING, TextIO - -from sqlbuild.cli.commands.types import ( - PlaygroundTemplate, -) -from sqlbuild.cli.compile.models import ( # noqa: F401 - CompileAnalysis, - CompileWriteResult, - SqlTestArtifactCacheRecord, - SqlTestArtifactIdentityContext, -) -from sqlbuild.cli.entry.models import ( # noqa: F401 - CliEntrypointHandlers, - CompileCommandRequest, - CompileProfileFlags, - ParsedCliInvocation, - SelectorFileSummary, -) -from sqlbuild.cli.output.models import ( # noqa: F401 - SkillMaintenanceResult, - SkillUpdateResult, - WrittenTarget, -) -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.cost.types import CostStatus -from sqlbuild.spec.contracts.models import ( - CostConfig, - ExecutionLimitsConfig, - SnapshotsConfig, -) - -if TYPE_CHECKING: - from sqlbuild._native import ( - ConnectionProgressReporter, - PlanningProgressReporter, - ) - from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter - from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput - - -@dataclass(frozen=True) -class NestedProgressChildRow: - """One child row rendered below a completed nested progress item.""" - - label: str - name: str - status_text: str - detail: str = "" - - -@dataclass(frozen=True) -class AuditCommandRequest: - """CLI inputs for one audit command invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - defer_to: str | None = None - no_color: bool = False - selected_target: str | None = None - concurrency: int | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - json_output: bool = False - json_output_path: Path | None = None - - -@dataclass(frozen=True) -class AuditInvocation: - """Resolved project, adapter, and reporter context for the audit command.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - connection_progress: ConnectionProgressReporter - planning_progress: PlanningProgressReporter - - -@dataclass(frozen=True) -class BuildCommandRequest: - """CLI inputs for one build command invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - defer_to: str | None = None - defer_clone_from: str | None = None - defer_sources_to: str | None = None - selected_target: str | None = None - cursor_overrides: CursorOverrides | None = None - no_color: bool = False - fail_fast: bool = False - full_refresh: bool = False - load_sources: bool | None = None - reload_sources: bool = False - include_python: bool = True - allow_snapshot_full_refresh: bool = False - allow_table_type_downgrade: bool = False - allow_retention_decrease: bool = False - allow_missing_migration_origin: bool = False - allow_snapshot_schema_change: bool = False - concurrency: int | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - verbose: bool = False - debug: bool = False - cli_vars: dict[str, object] | None = None - run_tests: bool = True - run_audits: bool = True - manifest: bool = False - json_output: bool = False - json_output_path: Path | None = None - event_output_path: Path | None = None - no_cache: bool = False - max_microbatches: int | None = None - selection_diagnostics: bool = False - selector_files: tuple[SelectorFileSummary, ...] = () - - -@dataclass(frozen=True) -class BuildInvocation: - """Resolved project, adapter, and reporter context for the build command.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - effective_defer_clone_from: str | None - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - connection_progress: ConnectionProgressReporter - planning_progress: PlanningProgressReporter - should_load_sources: bool - effective_target_name: str | None = None - execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) - - -@dataclass(frozen=True) -class CheckCommandRequest: - """CLI inputs for one check command invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - no_color: bool = False - selected_target: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - json_output: bool = False - json_output_path: Path | None = None - - -@dataclass(frozen=True) -class CheckInvocation: - """Resolved project, adapter, and reporter context for the check command.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - connection_progress: ConnectionProgressReporter - planning_progress: PlanningProgressReporter - - -@dataclass(frozen=True) -class CloneCommandRequest: - """CLI inputs for one clone command invocation.""" - - project_dir: Path | None - no_color: bool - no_sql_validation: bool - origin_target_name: str - destination_target_name: str | None - hard_copy: bool - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - verbose: bool = False - cli_vars: dict[str, object] | None = None - json_output_path: Path | None = None - event_output_path: Path | None = None - - -@dataclass(frozen=True) -class DbtSqlbuildWorkContext: - """Shared execution context for one dbt interop SQLBuild work run.""" - - plan_output: PlanOutput - connection_config: dict[str, object] - adapter: BaseAdapter - adapter_name: str - output_stream: TextIO - use_color: bool - snapshots_config: SnapshotsConfig - execution_limits: ExecutionLimitsConfig - - -@dataclass(frozen=True) -class PlanSafetyGates: - """Pre-confirmed plan safety gates and the guidance shown when one needs a terminal.""" - - allow_missing_migration_origin: bool - allow_snapshot_full_refresh: bool - allow_table_type_downgrade: bool - allow_retention_decrease: bool - missing_origin_help: str - snapshot_full_refresh_help: str - table_type_downgrade_help: str - retention_decrease_help: str - execution_limit_note: str - - -@dataclass(frozen=True) -class DbtInitCommandRequest: - """CLI inputs for one dbt init command invocation.""" - - cwd: Path - dbt_project_dir: str | None - profiles_dir: str | None - profile_name: str | None - target_name: str | None - sqb_output_dir: str | None - dry_run: bool - overwrite: bool - skip_dbt_debug: bool - - -@dataclass(frozen=True) -class DiffCommandRequest: - """CLI inputs for one diff command invocation.""" - - project_dir: Path | None - no_color: bool - no_sql_validation: bool - from_name: str | None - to_name: str | None - full: bool - schema_only: bool - bounded: str | None - max_column_examples: int | None = None - max_row_only_examples: int | None = None - sample_rows: int | None = None - sample_seed: int | None = None - exhaustive: bool = False - json_output: bool = False - json_output_path: Path | None = None - max_models: int | None = None - max_columns: int | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - verbose: bool = False - cli_vars: dict[str, object] | None = None - selected_target: str | None = None - left_query: str | None = None - left_query_file: Path | None = None - left_label: str | None = None - right_query: str | None = None - right_query_file: Path | None = None - right_label: str | None = None - unique_key_override: tuple[str, ...] = () - unkeyed: bool = False - excluded_columns_override: tuple[str, ...] = () - tolerance_overrides: tuple[str, ...] = () - max_value_length: int | None = None - suppress_example_values: bool = False - full_example_values: bool = False - - -@dataclass(frozen=True) -class QueryDiffPreparation: - """Resolved raw-query diff adapter, connection, SQL, and limits.""" - - adapter: BaseAdapter - connection_config: dict[str, object] - left_sql: str - right_sql: str - left_label: str - right_label: str - selected_target: str | None - database: str | None - schema: str - run_id: str - artifact_ttl: str - effective_max_column_examples: int - effective_max_row_only_examples: int - - -@dataclass(frozen=True) -class FreshnessCommandRequest: - """CLI inputs for one `sqb freshness` invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - no_color: bool = False - selected_target: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - json_output: bool = False - json_output_path: Path | None = None - fail_on_error: bool = False - compare_state: bool = False - fail_on_stale: bool = False - - -@dataclass(frozen=True) -class JanitorCommandRequest: - """CLI inputs for one janitor command invocation.""" - - project_dir: Path | None - no_color: bool = False - auto_approve: bool = False - retention_days: int | None = None - direct_state_history_versions: int | None = None - drop_old_name_views: tuple[str, ...] = () - selected_target: str | None = None - as_target: str | None = None - - -@dataclass(frozen=True) -class JanitorInvocation: - """Resolved project discovery and output context for janitor.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - use_color: bool - selected_target: str | None = None - active_target_name: str | None = None - as_target: str | None = None - - -@dataclass(frozen=True) -class JanitorSettings: - """Validated effective janitor settings.""" - - retention_days: int - direct_state_history_versions: int - archive_retention_days: int = 14 - drop_old_name_views: tuple[str, ...] = () - - -@dataclass(frozen=True) -class LoadCommandRequest: - """CLI inputs for one load command invocation.""" - - project_dir: Path | None - no_color: bool = False - selected_target: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - reload: bool = False - concurrency: int | None = None - cursor_overrides: CursorOverrides | None = None - cli_vars: dict[str, object] | None = None - json_output: bool = False - json_output_path: Path | None = None - - -@dataclass(frozen=True) -class PlanCommandRequest: - """CLI inputs for one plan command invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - defer_to: str | None = None - defer_sources_to: str | None = None - selected_target: str | None = None - as_target: str | None = None - cursor_overrides: CursorOverrides | None = None - json_output: bool = False - full_refresh: bool = False - load_sources: bool | None = None - include_python: bool = True - no_color: bool = False - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - verbose: bool = False - cli_vars: dict[str, object] | None = None - no_cache: bool = False - max_microbatches: int | None = None - selection_diagnostics: bool = False - - -@dataclass(frozen=True) -class PlaygroundCommandRequest: - """CLI inputs for one playground command invocation.""" - - project_dir: Path | None - target_path: str - template: str = PlaygroundTemplate.WAFFLE_SHOP.value - - -@dataclass(frozen=True) -class AdapterConnectionContext: - """Resolved adapter and connection configuration for one CLI command.""" - - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - - -@dataclass(frozen=True) -class ScenarioSnapshotLimitInputs: - """CLI snapshot capture-limit overrides for one scenario run.""" - - max_snapshot_rows: int | None = None - max_snapshot_total_rows: int | None = None - max_snapshot_bytes: int | None = None - max_snapshot_total_bytes: int | None = None - force: bool = False - - -@dataclass(frozen=True) -class ScenarioTestCommandRequest: - """CLI inputs for one `sqb scenario test` invocation.""" - - project_dir: Path | None = None - scenario_namespace: str | None = None - no_sql_validation: bool = False - no_color: bool = False - selectors: tuple[str, ...] = () - exclude: tuple[str, ...] = () - retain: bool = False - local: bool = False - strict: bool = False - concurrency: int | None = None - sync_snapshots: bool = False - refresh: bool = False - limit_inputs: ScenarioSnapshotLimitInputs = ScenarioSnapshotLimitInputs() - json_output: bool = False - json_output_path: Path | None = None - - -@dataclass(frozen=True) -class ScenarioCaptureCommandRequest: - """CLI inputs for one `sqb scenario capture` invocation.""" - - project_dir: Path | None = None - scenario_namespace: str | None = None - no_sql_validation: bool = False - no_color: bool = False - selectors: tuple[str, ...] = () - exclude: tuple[str, ...] = () - retain: bool = False - limit_inputs: ScenarioSnapshotLimitInputs = ScenarioSnapshotLimitInputs() - - -@dataclass(frozen=True) -class SeedCommandRequest: - """CLI inputs for one seed command invocation.""" - - project_dir: Path | None - no_color: bool = False - selected_target: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - concurrency: int | None = None - cli_vars: dict[str, object] | None = None - json_output: bool = False - json_output_path: Path | None = None - - -@dataclass(frozen=True) -class SeedInvocation: - """Resolved project, adapter, connection, and output context for seed.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - - -@dataclass(frozen=True) -class TestCommandRequest: - """CLI inputs for one test command invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - no_color: bool = False - selected_target: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - json_output: bool = False - json_output_path: Path | None = None - concurrency: int | None = None - case_name: str | None = None - inspect: bool = False - - -@dataclass(frozen=True) -class RefactorCommandRequest: - """CLI inputs for one `sqb rename` or `sqb mv` invocation.""" - - command: str - target: str - new_name: str | None = None - destination: str | None = None - project_dir: Path | None = None - cascade: bool = False - dry_run: bool = False - json_output: bool = False - no_color: bool = False - - -@dataclass(frozen=True) -class ScopeCommandRequest: - """CLI inputs for one offline declaration-scope inspection.""" - - project_dir: Path | None = None - target: str | None = None - at: str | None = None - as_path: str | None = None - browse: str | None = None - list_path: str | None = None - defined_under: str | None = None - kinds: tuple[str, ...] = () - match: str | None = None - used_only: bool = False - include_nearby: bool = False - nearby_depth: int = 1 - dependency_depth: int = 0 - explain: str | None = None - globals: str = "summary" - page_size: int = 100 - after: str | None = None - paths: str = "relative" - json_output: bool = False - no_cache: bool = False - verbose: bool = False - no_color: bool = False - - -@dataclass(frozen=True) -class TestInvocation: - """Resolved project, adapter, and reporter context for the test command.""" - - effective_project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter_name: str - adapter: BaseAdapter - connection_config: dict[str, object] - use_color: bool - progress_stream: TextIO - connection_progress: ConnectionProgressReporter - planning_progress: PlanningProgressReporter - - -@dataclass(frozen=True) -class RulesCommandRequest: - """Inputs for focused rule execution, inspection, or skill generation.""" - - project_dir: Path | None - json_output: bool - action: str - rule_selector: str | None - skills_check: bool - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - - -@dataclass(frozen=True) -class CostCommandRequest: - """Inputs for local run-cost history and drill-down.""" - - project_dir: Path | None - selector: str - no_color: bool - limit: int | None - no_limit: bool - sort: str | None - order: str | None - since: str | None - until: str | None - json_output: bool - json_output_path: Path | None - - -@dataclass(frozen=True) -class BuildCostFinalization: - """Inputs for nonfatal automatic build-cost finalization.""" - - project_dir: Path - adapter_name: str - adapter: object - connection_config: dict[str, object] - target_name: str | None - target_database: str | None - run_id: str - build_status: str - started_at: datetime - completed_at: datetime - config: CostConfig - output_stream: TextIO - use_color: bool - collect: bool = True - render: bool = True - cost_status: CostStatus = CostStatus.COMPLETE - cost_message: str = "No executable warehouse work was planned for this build." - had_executable_work: bool | None = True - - -@dataclass(frozen=True) -class ContractCommandRequest: - """CLI inputs for physical contract comparison and repository adoption.""" - - action: str - from_target: str - project_dir: Path | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - write: bool = False - overwrite: bool = False - json_output: bool = False - no_color: bool = False - cli_vars: dict[str, object] | None = None - - -@dataclass(frozen=True) -class LineageCommandRequest: - """CLI inputs for dependency and column lineage inspection.""" - - project_dir: Path | None - no_sql_validation: bool = False - targets: tuple[str, ...] = () - output_format: str = "tree" - direction: str | None = None - depth: str = "all" - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - lineage_mode: ColumnLineageMode = ColumnLineageMode.RICH - include_uses: bool = False - cli_vars: dict[str, object] | None = None diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py index 1927cab32a..855c6ebc9c 100644 --- a/src/sqlbuild/cli/commands/types.py +++ b/src/sqlbuild/cli/commands/types.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: - from sqlbuild.cli.commands.models import LineageCommandRequest, ScopeCommandRequest + from sqlbuild._native import LineageCommandRequest, ScopeCommandRequest from sqlbuild.compiler.scopes.models import ScopeIndex diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py index 023e02cbbc..78df74c758 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_output_encoding.py @@ -6,8 +6,8 @@ import pytest +from sqlbuild._native import PlaygroundCommandRequest from sqlbuild._native import run_playground_command as run_playground -from sqlbuild.cli.commands.models import PlaygroundCommandRequest from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( LegacyCodePageOutputTestCase, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py index f2c1d042cc..cbf5b1c997 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -5,8 +5,8 @@ import pytest +from sqlbuild._native import PlaygroundCommandRequest from sqlbuild._native import run_playground_command as run_playground -from sqlbuild.cli.commands.models import PlaygroundCommandRequest from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( PlaygroundCompileBuildTestCase, PythonNodesPlaygroundLifecycleTestCase, diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index a25a7ae59a..7fe6fd9263 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -21,8 +21,8 @@ materialize, ) +from sqlbuild._native import PlaygroundCommandRequest from sqlbuild._native import run_playground_command as run_playground -from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.integrations.dagster import ( SqlBuildCliResource, SqlBuildDagsterTranslator, diff --git a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py index 64e2a2854d..a72e977e61 100644 --- a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py @@ -11,8 +11,8 @@ import pytest +from sqlbuild._native import PlaygroundCommandRequest from sqlbuild._native import run_playground_command as run_playground -from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets from sqlbuild.integrations.rivers._helpers.assets import build_asset_defs from sqlbuild.integrations.rivers._helpers.dag import load_sqlbuild_dag diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py index efd1bba37a..400a9b170f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import run_refactor_command -from sqlbuild.cli.commands.models import RefactorCommandRequest +from sqlbuild._native import RefactorCommandRequest, run_refactor_command from sqlbuild.cli.commands.types import CliCommand from tests.integration.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( BareTargetRefactorTestCase, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index 9cba7be2a3..9dffe4bb19 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -11,7 +11,6 @@ import pytest from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( - ExpectedBooleanTestCase, ExpectedCountTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( @@ -60,55 +59,6 @@ def test_given_plain_compile_when_running_fresh_cli_then_optional_artifact_impor } -@pytest.mark.parametrize( - "test_case", - [ - ExpectedBooleanTestCase( - description="compatibility exports retain identity", expected_result=True - ) - ], - ids=lambda case: case.description, -) -def test_given_legacy_model_imports_when_loading_focused_models_then_exports_keep_identity( - test_case: ExpectedBooleanTestCase, -) -> None: - script = """ -import json -from sqlbuild.cli.commands import models -from sqlbuild.cli.compile import models as compile_models -from sqlbuild.cli.entry import models as entry_models -from sqlbuild.cli.output import models as output_models - -expected_modules = { - "CompileAnalysis": compile_models, - "CompileCommandRequest": entry_models, - "CompileProfileFlags": entry_models, - "CompileWriteResult": compile_models, - "SqlTestArtifactCacheRecord": compile_models, - "SqlTestArtifactIdentityContext": compile_models, - "CliEntrypointHandlers": entry_models, - "ParsedCliInvocation": entry_models, - "SelectorFileSummary": entry_models, - "SkillMaintenanceResult": output_models, - "SkillUpdateResult": output_models, - "WrittenTarget": output_models, -} -print(json.dumps({ - name: getattr(models, name) is getattr(module, name) - for name, module in expected_modules.items() -})) -""" - - result: subprocess.CompletedProcess[str] = subprocess.run( - [sys.executable, "-c", script], - check=True, - capture_output=True, - text=True, - ) - - assert all(json.loads(result.stdout).values()) is test_case.expected_result - - @pytest.mark.parametrize( "test_case", [ExpectedCountTestCase(description="dag command remains available", expected_count=1)], diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index db6c1fc58e..012838e072 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -14,6 +14,12 @@ from duckdb import DuckDBPyConnection from sqlbuild._native import ( + AuditCommandRequest, + BuildCommandRequest, + LoadCommandRequest, + PlanCommandRequest, + ScenarioTestCommandRequest, + TestCommandRequest, discover_project_inputs, format_load_execution_json, run_audit_command, @@ -27,14 +33,6 @@ from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - AuditCommandRequest, - BuildCommandRequest, - LoadCommandRequest, - PlanCommandRequest, - ScenarioTestCommandRequest, - TestCommandRequest, -) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorOverrides diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 5a2ae9f03a..be5252782a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -53,7 +53,6 @@ PRESENTATION_AND_PROVIDER_MODULES: tuple[str, ...] = ( "pydantic_settings", "rich", - "sqlbuild.cli.commands.models", ) PARSE_ONLY_FORBIDDEN: tuple[str, ...] = ( *WAREHOUSE_AND_INTEGRATION_MODULES, diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index 04d94ba4fc..249996f765 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -9,11 +9,10 @@ import duckdb import pytest -from sqlbuild._native import execute_query_diff +from sqlbuild._native import DiffCommandRequest, QueryDiffPreparation, execute_query_diff from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands.models import DiffCommandRequest, QueryDiffPreparation from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index 9249d6654f..2eccf2fc78 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -2,9 +2,9 @@ from pathlib import Path +from sqlbuild._native import PlanSafetyGates from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.cli.commands.models import PlanSafetyGates from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py index cee2f9b1a8..4f758851b9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/helpers.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any -from sqlbuild.cli.commands.models import BuildCostFinalization +from sqlbuild._native import BuildCostFinalization from sqlbuild.spec.contracts.models import CostConfig diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index 25acc186fd..4988768b0c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -9,8 +9,12 @@ import pytest -from sqlbuild._native import read_cost_run, refresh_pending_cost_run, write_cost_run -from sqlbuild.cli.commands.models import CostCommandRequest +from sqlbuild._native import ( + CostCommandRequest, + read_cost_run, + refresh_pending_cost_run, + write_cost_run, +) from sqlbuild.cost.models import CostRunRecord, RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py index ef798251b1..0073bcb8e7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_output.py @@ -2,13 +2,11 @@ from __future__ import annotations -from dataclasses import replace from io import StringIO import pytest -from sqlbuild._native import render_scope_result, run_scope_command -from sqlbuild.cli.commands.models import ScopeCommandRequest +from sqlbuild._native import ScopeCommandRequest, render_scope_result, run_scope_command from tests.unit.src.sqlbuild.cli.commands._helpers.scope._test_types import ( ScopeColourCase, ScopeDetailLevelCase, @@ -23,9 +21,6 @@ _YELLOW: str = "\033[33m" _RED: str = "\033[38;5;167m" _RESET: str = "\033[0m" -_MOVE_REQUEST: ScopeCommandRequest = ScopeCommandRequest( - target="model:orders", as_path="models/marts/orders.sql" -) @pytest.mark.parametrize( @@ -76,7 +71,11 @@ def test_given_move_preview_when_rendering_then_detail_level_follows_verbose( stream: StringIO = StringIO() exit_code: int = run_scope_command( - request=replace(_MOVE_REQUEST, verbose=test_case.verbose), + request=ScopeCommandRequest( + target="model:orders", + as_path="models/marts/orders.sql", + verbose=test_case.verbose, + ), load_scope_index=lambda **_kwargs: report_scope_lookup().index, output_stream=stream, ) @@ -127,7 +126,9 @@ def test_given_colour_setting_when_rendering_move_preview_then_applies_colour_ro test_case: ScopeColourCase, ) -> None: rendered: str = render_scope_result( - result=orders_move_report(), request=_MOVE_REQUEST, use_color=test_case.use_color + result=orders_move_report(), + request=ScopeCommandRequest(target="model:orders", as_path="models/marts/orders.sql"), + use_color=test_case.use_color, ) assert all(fragment in rendered for fragment in test_case.expected_fragments), rendered diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py index 769ad2aac9..cbd0d288d2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import run_scope_command +from sqlbuild._native import ScopeCommandRequest, run_scope_command from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ScopeCommandRequest from tests.unit.src.sqlbuild.cli.commands._helpers.scope._test_types import ( ScopeTargetCase, ScopeTargetErrorCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_import_footprint.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_import_footprint.py index 23406d5953..830f4327f9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_import_footprint.py @@ -24,7 +24,6 @@ CompileImportFootprintTestCase( description="compile skips build and execution command models", checked_modules=( - "sqlbuild.cli.commands.models", "sqlbuild.executor.build.models", "sqlbuild.executor.run.models", ), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py index 240bb1bba0..9325a2e1c3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_init.py @@ -7,7 +7,7 @@ import sqlbuild._native as dbt_init_module import sqlbuild.integrations.dbt._helpers.profile.init as dbt_profile_init -from sqlbuild.cli.commands.models import DbtInitCommandRequest +from sqlbuild._native import DbtInitCommandRequest from sqlbuild.integrations.dbt.exceptions import DbtProfileError from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 2bda854f13..c1ad3c82b6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -9,10 +9,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import run_cli as _main_with_dependencies -from sqlbuild._native import run_cli as main -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( +from sqlbuild._native import ( AuditCommandRequest, BuildCommandRequest, CloneCommandRequest, @@ -29,6 +26,9 @@ ScenarioCaptureCommandRequest, ScenarioTestCommandRequest, ) +from sqlbuild._native import run_cli as _main_with_dependencies +from sqlbuild._native import run_cli as main +from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.exceptions import CompileInputError diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_scope.py index 32855986ec..577d080ba3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_scope.py @@ -2,12 +2,14 @@ from __future__ import annotations +from dataclasses import FrozenInstanceError + import pytest -from sqlbuild._native import run_cli as _main_with_dependencies -from sqlbuild.cli.commands.models import ( +from sqlbuild._native import ( ScopeCommandRequest, ) +from sqlbuild._native import run_cli as _main_with_dependencies from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( ScopeArgumentMisuseCase, ScopeEntryCase, @@ -88,7 +90,8 @@ def record(request: ScopeCommandRequest) -> int: no_color=True, ) ] - assert ScopeCommandRequest.__dataclass_params__.frozen is True + with pytest.raises(FrozenInstanceError): + received[0].verbose = False @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py index a72e7bdbb0..5a3cf41bf1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py @@ -7,9 +7,8 @@ import pytest -from sqlbuild._native import run_build_command +from sqlbuild._native import BuildCommandRequest, run_build_command from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import BuildCommandRequest from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( BuildDurationLimitTestCase, BuildProcessReportTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py index f20b93c541..b20328cb18 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py @@ -7,8 +7,11 @@ import pytest -from sqlbuild._native import NativeProgressProjector, run_scenario_test_command -from sqlbuild.cli.commands.models import ScenarioTestCommandRequest +from sqlbuild._native import ( + NativeProgressProjector, + ScenarioTestCommandRequest, + run_scenario_test_command, +) from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py index ff9c9e1f8b..fa0aeba44d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py @@ -2,7 +2,7 @@ from decimal import Decimal from pathlib import Path -from sqlbuild.cli.commands.models import CostCommandRequest +from sqlbuild._native import CostCommandRequest from sqlbuild.cost.models import CostRunRecord, ResourceCost, RunCostSummary from sqlbuild.cost.types import CostStatus diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index d4843a3838..1a044e4a19 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -5,9 +5,12 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import maintain_sqlbuild_skills, run_playground_command +from sqlbuild._native import ( + PlaygroundCommandRequest, + maintain_sqlbuild_skills, + run_playground_command, +) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import PlaygroundCommandRequest from sqlbuild.cli.output.models import SkillMaintenanceResult from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py index 913badbca7..2af8533318 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -8,9 +8,8 @@ import pytest from sqlbuild import _native as native -from sqlbuild._native import run_scope_command +from sqlbuild._native import ScopeCommandRequest, run_scope_command from sqlbuild.adapter.discovery.main import resolve_adapter as adapter_registry -from sqlbuild.cli.commands.models import ScopeCommandRequest from sqlbuild.compiler.scopes.models import ScopeCompleteness, ScopeIndex from tests.unit.src.sqlbuild.cli.commands.main.scope._test_types import ScopeCommandCase from tests.unit.src.sqlbuild.compiler.scopes.helpers import report_scope_lookup diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 535810c1d7..526c540419 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -10,6 +10,7 @@ from sqlbuild._native import ( BuildProgressCallbacks, NestedCommandProgressCallbacks, + NestedProgressChildRow, build_test_expectation_rows, format_build_footer, resolve_test_name_width, @@ -17,7 +18,6 @@ ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.commands.models import NestedProgressChildRow from sqlbuild.compiler.auditing.types import AuditOutcome from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.types import SkipMode From eaa963c1b9cd474fdf2b1f108add6db1a5f12d26 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 06:57:33 +0100 Subject: [PATCH 10/23] refactor: build CLI entry, compile output and skill results as native records --- crates/sqlbuild-cli/src/dispatch/constants.rs | 2 +- .../src/bindings/_helpers/cli/argv.rs | 2 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../command_runtime/warehouse_scope.rs | 2 +- .../_helpers/command_runtime/warehouses.rs | 39 +-- .../_helpers/compile_command/target_writer.rs | 2 +- .../_helpers/compile_driver/analysis.rs | 2 +- .../_helpers/compile_driver/phases.rs | 2 +- .../bindings/_helpers/records/constants.rs | 190 +++++++++++- .../src/bindings/_helpers/records/fields.rs | 1 + .../src/bindings/_helpers/records/models.rs | 1 + .../bindings/_helpers/records/registration.rs | 34 ++- .../_helpers/refactor/offline_compile.rs | 2 +- .../_helpers/workspace_commands/commands.rs | 2 +- fensu.toml | 7 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 281 ++++++++++++++++-- src/sqlbuild/cli/compile/__init__.py | 0 src/sqlbuild/cli/compile/models.py | 67 ----- src/sqlbuild/cli/entry/models.py | 136 --------- src/sqlbuild/cli/output/models.py | 35 --- src/sqlbuild/cli/output/types.py | 8 - .../main/test_startup_import_footprint.py | 3 - .../commands/_helpers/entry/test_parsing.py | 6 +- .../_helpers/runtime/test_warehouses.py | 3 +- .../cli/commands/main/compile/helpers.py | 2 +- .../main/compile/test_compile_command.py | 3 +- .../main/compile/test_compile_output.py | 4 +- .../compile/test_compile_target_writer.py | 4 +- .../test_staged_artifact_publication.py | 3 +- .../cli/commands/main/entry/helpers.py | 4 +- .../main/entry/test_compute_capture.py | 2 +- .../cli/commands/main/entry/test_main.py | 2 +- .../main/playground/test_playground.py | 2 +- .../cli/commands/main/skills/helpers.py | 3 +- .../main/skills/test_skill_directory.py | 2 +- .../cli/commands/main/skills/test_skills.py | 6 +- 37 files changed, 517 insertions(+), 351 deletions(-) delete mode 100644 src/sqlbuild/cli/compile/__init__.py delete mode 100644 src/sqlbuild/cli/compile/models.py delete mode 100644 src/sqlbuild/cli/entry/models.py delete mode 100644 src/sqlbuild/cli/output/models.py delete mode 100644 src/sqlbuild/cli/output/types.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 3bd77ad282..7ecac4974c 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -12,7 +12,7 @@ pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init"; pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const ENTRY_MODELS: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs index a4be411f9d..80a6fa14cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::cli::namespace::new_cli_namespace; use crate::bindings::_helpers::cli::surrogates::{python_text, rust_text, uses_placeholders}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; +const ENTRY_MODELS_MODULE: &str = "sqlbuild._native"; const INT_CONVERTER: &str = "int"; const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; const VARS_CONVERTER: &str = "parse_cli_vars"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 3c6625fcf7..5732239d8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -21,7 +21,7 @@ use sqlbuild_core::text::main::python_strip::python_strip; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; +const ENTRY_MODELS_MODULE: &str = "sqlbuild._native"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index 53167885f7..b7023de7f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -12,7 +12,7 @@ use sqlbuild_config::command_connection::constants::COMMAND_WAREHOUSE_GROUPS; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const ENTRY_MODELS: &str = "sqlbuild._native"; const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild.spec.contracts.types", "WarehouseGroup"); const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index d9b469c8ed..0b24d368a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::discovery::configuration::discover_project_configuration; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; -const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const ENTRY_MODELS: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SESSION_WAREHOUSE_MODULE: &str = "sqlbuild.adapter.discovery.main._session_warehouse_support"; @@ -303,42 +303,6 @@ fn validate_cli_warehouse( Ok(()) } -/// Find the active target's authored, unexpanded warehouse for one command group. -#[pyfunction] -#[pyo3(signature = (*, discovered_inputs, selected_target, group))] -fn find_target_group_warehouse<'py>( - py: Python<'py>, - discovered_inputs: Bound<'py, PyAny>, - selected_target: Bound<'py, PyAny>, - group: Bound<'py, PyAny>, -) -> PyResult> { - let inputs: WarehouseInputs<'_, 'py> = WarehouseInputs { - discovered_inputs: &discovered_inputs, - selected_target: &selected_target, - }; - let Some(found) = find_authored(inputs, &group)? else { - return Ok(none(py)); - }; - call_with( - &imported(py, ENTRY_MODELS, "AuthoredTargetWarehouse")?, - &[ - ( - "warehouse", - PyString::new(py, &found.authored.warehouse).into_any(), - ), - ( - "target_name", - PyString::new(py, &found.authored.target_name).into_any(), - ), - ("group", found.group), - ( - "file_name", - PyString::new(py, &found.authored.file_name).into_any(), - ), - ], - ) -} - /// Resolve one group's warehouse: CLI, then target group, then connection, then default. #[pyfunction] #[pyo3(signature = (**options))] @@ -458,7 +422,6 @@ fn describe_connection_warehouse<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(validate_cli_warehouse, module)?)?; - module.add_function(wrap_pyfunction!(find_target_group_warehouse, module)?)?; module.add_function(wrap_pyfunction!(resolve_command_warehouse, module)?)?; module.add_function(wrap_pyfunction!(apply_command_warehouse, module)?)?; module.add_function(wrap_pyfunction!(describe_connection_warehouse, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 773c0ff562..d8df2df8e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -44,7 +44,7 @@ const SQL_FILE_SUFFIX: &str = ".sql"; const LANGUAGE_DIR_DEPTH: usize = 2; const PUBLISHED_STAGING_CHANGED: &str = "changed"; const PUBLISHED_TREE: &str = "tree"; -const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; +const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 126168f78a..3cdcc843f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -33,7 +33,7 @@ const RULE_THREADS_MODULE: &str = "sqlbuild.rule_engine._helpers.run.rule_thread const RULE_MODELS_MODULE: &str = "sqlbuild.rule_engine.models"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild.cli.compile.models"; +const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild._native"; const GENERIC_DIALECT: &str = "generic"; const TARGET_DIRECTORY: &str = "target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index abd42d48c9..c407dc3e15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; const NATIVE_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; -const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; +const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index addd9c8db4..812a28946a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -6,9 +6,18 @@ pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; /// The class attribute holding a record class's index into `RECORDS`. pub(crate) const RECORD_SPEC_ATTRIBUTE: &str = "__native_record__"; pub(crate) const FROZEN_INSTANCE_ERROR: (&str, &str) = ("dataclasses", "FrozenInstanceError"); +pub(crate) const DATACLASSES_MODULE: &str = "dataclasses"; +pub(crate) const DATACLASS_FIELDS_ATTRIBUTE: &str = "__dataclass_fields__"; +/// The marker `dataclasses` gives ordinary (not class or init-only) fields. +pub(crate) const DATACLASS_FIELD_KIND: &str = "_FIELD"; /// Every native record group, in registration order. -pub(crate) const RECORDS: &[&[RecordSpec]] = &[COMMAND_RECORDS]; +pub(crate) const RECORDS: &[&[RecordSpec]] = &[ + COMMAND_RECORDS, + ENTRY_RECORDS, + OUTPUT_RECORDS, + COMPILE_RECORDS, +]; const NESTED_PROGRESS_CHILD_ROW: RecordSpec = RecordSpec { name: "NestedProgressChildRow", @@ -675,3 +684,182 @@ const COMMAND_RECORDS: &[RecordSpec] = &[ CONTRACT_COMMAND_REQUEST, LINEAGE_COMMAND_REQUEST, ]; + +const COMPILE_PROFILE_FLAGS: RecordSpec = RecordSpec { + name: "CompileProfileFlags", + doc: "Profiling toggles that skip compile phases for benchmarking.", + fields: &[ + FieldSpec::new("skip_discovery_sql_analysis", FieldDefault::False), + FieldSpec::new("skip_column_inference", FieldDefault::False), + FieldSpec::new("skip_contracts", FieldDefault::False), + FieldSpec::new("skip_write", FieldDefault::False), + ], +}; +const COMPILE_COMMAND_REQUEST: RecordSpec = RecordSpec { + name: "CompileCommandRequest", + doc: "CLI inputs for one `sqb compile` invocation.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("defer_to", FieldDefault::None), + FieldSpec::new("selected_target", FieldDefault::None), + FieldSpec::new("json_output", FieldDefault::False), + FieldSpec::new("manifest", FieldDefault::False), + FieldSpec::new("dag_path", FieldDefault::None), + FieldSpec::new("no_color", FieldDefault::False), + FieldSpec::new( + "lineage_mode", + FieldDefault::Member("sqlbuild.cli.commands.types", "CompileLineageMode", "FAST"), + ), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("cli_vars", FieldDefault::None), + FieldSpec::new( + "profile_flags", + FieldDefault::Call("sqlbuild._native", "CompileProfileFlags"), + ), + FieldSpec::new("no_cache", FieldDefault::False), + FieldSpec::new("debug", FieldDefault::False), + ], +}; +const SELECTOR_FILE_SUMMARY: RecordSpec = RecordSpec { + name: "SelectorFileSummary", + doc: "Non-expanded provenance for one selector file.", + fields: &[ + FieldSpec::required("path"), + FieldSpec::required("selector_count"), + ], +}; +const PARSED_CLI_INVOCATION: RecordSpec = RecordSpec { + name: "ParsedCliInvocation", + doc: "Outcome of parsing CLI arguments: either a namespace or an exit code.", + fields: &[ + FieldSpec::required("args"), + FieldSpec::required("exit_code"), + ], +}; +const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { + name: "CliEntrypointHandlers", + doc: "Injected command handlers for the CLI entrypoint.", + fields: &[ + FieldSpec::required("run_compile"), + FieldSpec::required("run_cost"), + FieldSpec::required("run_dag"), + FieldSpec::required("run_plan"), + FieldSpec::required("run_dbt_plan"), + FieldSpec::required("run_dbt_run"), + FieldSpec::required("run_dbt_build"), + FieldSpec::required("run_dbt_debug"), + FieldSpec::required("run_dbt_init"), + FieldSpec::required("run_build"), + FieldSpec::required("run_freshness"), + FieldSpec::required("run_test"), + FieldSpec::required("run_check"), + FieldSpec::required("run_audit"), + FieldSpec::required("run_seed"), + FieldSpec::required("run_load"), + FieldSpec::required("run_clone"), + FieldSpec::required("run_diff"), + FieldSpec::required("run_query"), + FieldSpec::required("run_debug"), + FieldSpec::required("run_lineage"), + FieldSpec::required("run_janitor"), + FieldSpec::required("run_init"), + FieldSpec::required("run_playground"), + FieldSpec::required("run_skills_update"), + FieldSpec::required("run_format"), + FieldSpec::required("run_scenario"), + FieldSpec::required("run_scenario_capture"), + FieldSpec::required("run_rules"), + FieldSpec::required("run_scope"), + FieldSpec::new("run_contract", FieldDefault::None), + FieldSpec::new("run_refactor", FieldDefault::None), + ], +}; +const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { + name: "CommandWarehouseScope", + doc: "The warehouse group and CLI override active for one command invocation.", + fields: &[ + FieldSpec::required("group"), + FieldSpec::new("cli_warehouse", FieldDefault::None), + FieldSpec::new("cli_vars", FieldDefault::EmptyDict), + ], +}; +const RESOLVED_WAREHOUSE: RecordSpec = RecordSpec { + name: "ResolvedWarehouse", + doc: "The warehouse one command connects with and where that value came from.", + fields: &[ + FieldSpec::required("warehouse"), + FieldSpec::required("source"), + ], +}; + +/// Parsed `sqb` invocations, compile requests, entry handlers and command warehouses. +const ENTRY_RECORDS: &[RecordSpec] = &[ + COMPILE_PROFILE_FLAGS, + COMPILE_COMMAND_REQUEST, + SELECTOR_FILE_SUMMARY, + PARSED_CLI_INVOCATION, + CLI_ENTRYPOINT_HANDLERS, + COMMAND_WAREHOUSE_SCOPE, + RESOLVED_WAREHOUSE, +]; + +const WRITTEN_TARGET: RecordSpec = RecordSpec { + name: "WrittenTarget", + doc: "Result of writing compiled output to target/.", + fields: &[ + FieldSpec::required("model_count"), + FieldSpec::required("seed_count"), + FieldSpec::required("function_count"), + FieldSpec::required("audit_count"), + FieldSpec::required("test_count"), + FieldSpec::required("target_dir"), + FieldSpec::new("diagnostics", FieldDefault::EmptyTuple), + ], +}; +const SKILL_MAINTENANCE_RESULT: RecordSpec = RecordSpec { + name: "SkillMaintenanceResult", + doc: "Non-blocking generated-skill maintenance result.", + fields: &[FieldSpec::new("message", FieldDefault::Text(""))], +}; +const SKILL_UPDATE_RESULT: RecordSpec = RecordSpec { + name: "SkillUpdateResult", + doc: "Result of installing or updating skill files.", + fields: &[FieldSpec::required("written_paths")], +}; + +/// Compile output and skill maintenance results. +const OUTPUT_RECORDS: &[RecordSpec] = &[ + WRITTEN_TARGET, + SKILL_MAINTENANCE_RESULT, + SKILL_UPDATE_RESULT, +]; + +const COMPILE_ANALYSIS: RecordSpec = RecordSpec { + name: "CompileAnalysis", + doc: "Compiled project analysis shared by compile output phases.", + fields: &[ + FieldSpec::required("discovered_inputs"), + FieldSpec::required("adapter"), + FieldSpec::required("graph"), + FieldSpec::required("selected_keys"), + FieldSpec::required("lineage"), + FieldSpec::required("diagnostics"), + FieldSpec::required("discover_ms"), + FieldSpec::required("graph_ms"), + FieldSpec::required("lineage_ms"), + FieldSpec::required("contract_ms"), + FieldSpec::required("contract_cpu_ms"), + FieldSpec::new("built_in_rules_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_rules_ms", FieldDefault::Int(0)), + FieldSpec::new("custom_rules_cpu_ms", FieldDefault::Int(0)), + FieldSpec::new("early_lint_wait_ms", FieldDefault::Int(0)), + FieldSpec::new("rule_cache_hits", FieldDefault::Int(0)), + FieldSpec::new("rule_cache_misses", FieldDefault::Int(0)), + FieldSpec::new("skipped_type_proof_rules", FieldDefault::EmptyTuple), + ], +}; + +/// The compiled project analysis shared by compile output phases. +const COMPILE_RECORDS: &[RecordSpec] = &[COMPILE_ANALYSIS]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index 00b0d5f2fa..e7bc8eb51b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -99,6 +99,7 @@ fn default_value<'py>( FieldDefault::Int(value) => value.into_bound_py_any(py), FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), + FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), FieldDefault::Member(module, class, member) => imported(py, module, class)?.getattr(member), FieldDefault::Call(module, name) => imported(py, module, name)?.call0(), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index fc8e397341..3842e720f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -19,6 +19,7 @@ pub(crate) enum FieldDefault { Int(i64), Text(&'static str), EmptyTuple, + EmptyDict, /// `getattr(getattr(import(module), class), member)`. Member(&'static str, &'static str, &'static str), /// `getattr(import(module), name)()`, evaluated for every instance. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index fcf81a0926..5159e705ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -3,7 +3,10 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use crate::bindings::_helpers::records::constants::{NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE}; +use crate::bindings::_helpers::records::constants::{ + DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, NATIVE_MODULE, + RECORD_SPEC_ATTRIBUTE, +}; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::models::{NativeRecord, RecordAttribute, RecordSpec}; @@ -13,12 +16,39 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; let base: Bound<'_, PyAny> = py.get_type::().into_any(); let metaclass: Bound<'_, PyAny> = base.get_type().into_any(); + let dataclasses: Bound<'_, PyModule> = py.import(DATACLASSES_MODULE)?; for (index, spec) in all_specs().enumerate() { - module.add(spec.name, record_class(&metaclass, &base, (index, spec))?)?; + let class: Bound<'_, PyAny> = record_class(&metaclass, &base, (index, spec))?; + class.setattr( + DATACLASS_FIELDS_ATTRIBUTE, + dataclass_fields(&dataclasses, spec)?, + )?; + module.add(spec.name, class)?; } Ok(()) } +/// The `dataclasses.Field` table that lets `dataclasses.replace` and `fields` read a record. +fn dataclass_fields<'py>( + dataclasses: &Bound<'py, PyModule>, + spec: &RecordSpec, +) -> PyResult> { + let py: Python<'py> = dataclasses.py(); + let field_kind: Bound<'py, PyAny> = dataclasses.getattr(DATACLASS_FIELD_KIND)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in spec.fields { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("compare", field.compared)?; + options.set_item("repr", field.compared)?; + let declared: Bound<'py, PyAny> = dataclasses.getattr("field")?.call((), Some(&options))?; + declared.setattr("name", field.name)?; + declared.setattr("type", "object")?; + declared.setattr("_field_type", &field_kind)?; + fields.set_item(field.name, declared)?; + } + Ok(fields) +} + /// A `NativeRecord` subclass laid out as `spec`. fn record_class<'py>( metaclass: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index 2a914f63f2..b2adf6af7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -15,7 +15,7 @@ const COMPILE_ERRORS: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; const SPEC_ERRORS: &str = "sqlbuild.spec.contracts.exceptions"; -const ENTRY_MODELS: &str = "sqlbuild.cli.entry.models"; +const ENTRY_MODELS: &str = "sqlbuild._native"; const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; const REFACTOR_MODELS: &str = "sqlbuild.compiler.refactoring.models"; const DEFAULT_ERROR_CODE: &str = "E001"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index dfe56f585b..58b50f56b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; const TEMPLATES_DIRECTORY: &str = "playground_templates"; const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const OUTPUT_MODELS: &str = "sqlbuild.cli.output.models"; +const OUTPUT_MODELS: &str = "sqlbuild._native"; const SKILLS_OPTIONS: (&[&str], &[&str]) = (&["project_dir"], &["global_install", "targets", "force"]); const UPDATE_OPTIONS: (&[&str], &[&str]) = ( diff --git a/fensu.toml b/fensu.toml index d80e5e7f3a..b207ddeddd 100644 --- a/fensu.toml +++ b/fensu.toml @@ -33,11 +33,6 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/constants.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/cli/compile/__init__.py" -reason = "Native bindings own the compile command, its reuse attempt and its analysis; this package only declares the compile analysis models the native code builds." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" @@ -186,7 +181,7 @@ reason = "Graph algorithms and views run natively; the package keeps only the ed [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/output/__init__.py" -reason = "Execution-result output is rendered natively; the package keeps only the integration-result models, constants and types its callers and readers share." +reason = "Execution-result output is rendered natively; the package keeps only the integration-result constants its callers and readers share." [[targets.python.rule_exceptions]] rule = "FFR309" diff --git a/python-boundary.toml b/python-boundary.toml index 47a0d41149..28290c17fd 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 11290 +remaining_lines = 10394 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0a3f3823a3..54f15b9ea3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -27,19 +27,19 @@ from sqlbuild.adapter.contract.types import ( TypeDialect, ) from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode, ScopeIndexLoader -from sqlbuild.cli.compile.models import CompileAnalysis -from sqlbuild.cli.compile_reuse.models import CompileReuseRequest -from sqlbuild.cli.entry.models import ( - AuthoredTargetWarehouse, - CommandWarehouseScope, - CompileCommandRequest, - CompileProfileFlags, - ParsedCliInvocation, - ResolvedWarehouse, - SelectorFileSummary, +from sqlbuild.cli.commands.types import ( + CliCommand, + CompileLineageMode, + DagCommandHandler, + DebugCommandHandler, + FormatCommandHandler, + LineageCommandHandler, + QueryCommandHandler, + ScopeCommandHandler, + ScopeIndexLoader, + SkillsUpdateCommandHandler, ) -from sqlbuild.cli.output.models import SkillMaintenanceResult, SkillUpdateResult, WrittenTarget +from sqlbuild.cli.compile_reuse.models import CompileReuseRequest from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, MeasurementContract, @@ -2975,12 +2975,6 @@ def run_skills_command( ) -> int: ... def run_playground_command(request: PlaygroundCommandRequest) -> int: ... def validate_cli_warehouse(*, cli_warehouse: str | None, project_dir: Path) -> None: ... -def find_target_group_warehouse( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - group: WarehouseGroup, -) -> AuthoredTargetWarehouse | None: ... def resolve_command_warehouse( *, discovered_inputs: DiscoveredProjectInputs | None, @@ -8438,3 +8432,254 @@ class LineageCommandRequest: include_uses: bool = False, cli_vars: dict[str, object] | None = None, ) -> None: ... + +class CompileProfileFlags: + """Profiling toggles that skip compile phases for benchmarking.""" + + skip_discovery_sql_analysis: bool + skip_column_inference: bool + skip_contracts: bool + skip_write: bool + def __init__( + self, + skip_discovery_sql_analysis: bool = False, + skip_column_inference: bool = False, + skip_contracts: bool = False, + skip_write: bool = False, + ) -> None: ... + +class CompileCommandRequest: + """CLI inputs for one `sqb compile` invocation.""" + + project_dir: Path | None + no_sql_validation: bool + defer_to: str | None + selected_target: str | None + json_output: bool + manifest: bool + dag_path: str | None + no_color: bool + lineage_mode: CompileLineageMode + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + profile_flags: CompileProfileFlags + no_cache: bool + debug: bool + def __init__( + self, + project_dir: Path | None = None, + no_sql_validation: bool = False, + defer_to: str | None = None, + selected_target: str | None = None, + json_output: bool = False, + manifest: bool = False, + dag_path: str | None = None, + no_color: bool = False, + lineage_mode: CompileLineageMode = ..., + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + cli_vars: dict[str, object] | None = None, + profile_flags: CompileProfileFlags = ..., + no_cache: bool = False, + debug: bool = False, + ) -> None: ... + +class SelectorFileSummary: + """Non-expanded provenance for one selector file.""" + + path: Path + selector_count: int + def __init__( + self, + path: Path, + selector_count: int, + ) -> None: ... + +class ParsedCliInvocation: + """Outcome of parsing CLI arguments: either a namespace or an exit code.""" + + args: CliNamespace | None + exit_code: int | None + def __init__( + self, + args: CliNamespace | None, + exit_code: int | None, + ) -> None: ... + +class CliEntrypointHandlers: + """Injected command handlers for the CLI entrypoint.""" + + run_compile: Callable[[CompileCommandRequest], int] + run_cost: Callable[..., int] + run_dag: DagCommandHandler + run_plan: Callable[..., int] + run_dbt_plan: Callable[[Path | None, tuple[str, ...], bool], int] + run_dbt_run: Callable[[Path | None, tuple[str, ...], bool], int] + run_dbt_build: Callable[[Path | None, tuple[str, ...], bool], int] + run_dbt_debug: Callable[[Path | None, tuple[str, ...], bool], int] + run_dbt_init: Callable[..., int] + run_build: Callable[..., int] + run_freshness: Callable[..., int] + run_test: Callable[..., int] + run_check: Callable[..., int] + run_audit: Callable[..., int] + run_seed: Callable[..., int] + run_load: Callable[..., int] + run_clone: Callable[..., int] + run_diff: Callable[..., int] + run_query: QueryCommandHandler + run_debug: DebugCommandHandler + run_lineage: LineageCommandHandler + run_janitor: Callable[..., int] + run_init: Callable[[Path | None], int] + run_playground: Callable[..., int] + run_skills_update: SkillsUpdateCommandHandler + run_format: FormatCommandHandler + run_scenario: Callable[..., int] + run_scenario_capture: Callable[..., int] + run_rules: Callable[..., int] + run_scope: ScopeCommandHandler + run_contract: Callable[..., int] | None + run_refactor: Callable[..., int] | None + def __init__( + self, + run_compile: Callable[[CompileCommandRequest], int], + run_cost: Callable[..., int], + run_dag: DagCommandHandler, + run_plan: Callable[..., int], + run_dbt_plan: Callable[[Path | None, tuple[str, ...], bool], int], + run_dbt_run: Callable[[Path | None, tuple[str, ...], bool], int], + run_dbt_build: Callable[[Path | None, tuple[str, ...], bool], int], + run_dbt_debug: Callable[[Path | None, tuple[str, ...], bool], int], + run_dbt_init: Callable[..., int], + run_build: Callable[..., int], + run_freshness: Callable[..., int], + run_test: Callable[..., int], + run_check: Callable[..., int], + run_audit: Callable[..., int], + run_seed: Callable[..., int], + run_load: Callable[..., int], + run_clone: Callable[..., int], + run_diff: Callable[..., int], + run_query: QueryCommandHandler, + run_debug: DebugCommandHandler, + run_lineage: LineageCommandHandler, + run_janitor: Callable[..., int], + run_init: Callable[[Path | None], int], + run_playground: Callable[..., int], + run_skills_update: SkillsUpdateCommandHandler, + run_format: FormatCommandHandler, + run_scenario: Callable[..., int], + run_scenario_capture: Callable[..., int], + run_rules: Callable[..., int], + run_scope: ScopeCommandHandler, + run_contract: Callable[..., int] | None = None, + run_refactor: Callable[..., int] | None = None, + ) -> None: ... + +class CommandWarehouseScope: + """The warehouse group and CLI override active for one command invocation.""" + + group: WarehouseGroup | None + cli_warehouse: str | None + cli_vars: dict[str, object] + def __init__( + self, + group: WarehouseGroup | None, + cli_warehouse: str | None = None, + cli_vars: dict[str, object] = ..., + ) -> None: ... + +class ResolvedWarehouse: + """The warehouse one command connects with and where that value came from.""" + + warehouse: str | None + source: str + def __init__( + self, + warehouse: str | None, + source: str, + ) -> None: ... + +class WrittenTarget: + """Result of writing compiled output to target/.""" + + model_count: int + seed_count: int + function_count: int + audit_count: int + test_count: int + target_dir: Path + diagnostics: tuple[CompilerDiagnostic, ...] + def __init__( + self, + model_count: int, + seed_count: int, + function_count: int, + audit_count: int, + test_count: int, + target_dir: Path, + diagnostics: tuple[CompilerDiagnostic, ...] = ..., + ) -> None: ... + +class SkillMaintenanceResult: + """Non-blocking generated-skill maintenance result.""" + + message: str + def __init__( + self, + message: str = "", + ) -> None: ... + +class SkillUpdateResult: + """Result of installing or updating skill files.""" + + written_paths: tuple[Path, ...] + def __init__( + self, + written_paths: tuple[Path, ...], + ) -> None: ... + +class CompileAnalysis: + """Compiled project analysis shared by compile output phases.""" + + discovered_inputs: DiscoveredProjectInputs + adapter: BaseAdapter + graph: ProjectGraph + selected_keys: frozenset[CompiledObjectKey] + lineage: ProjectColumnLineage | None + diagnostics: tuple[CompilerDiagnostic, ...] + discover_ms: int + graph_ms: int + lineage_ms: int + contract_ms: int + contract_cpu_ms: int + built_in_rules_ms: int + custom_rules_ms: int + custom_rules_cpu_ms: int + early_lint_wait_ms: int + rule_cache_hits: int + rule_cache_misses: int + skipped_type_proof_rules: tuple[str, ...] + def __init__( + self, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + graph: ProjectGraph, + selected_keys: frozenset[CompiledObjectKey], + lineage: ProjectColumnLineage | None, + diagnostics: tuple[CompilerDiagnostic, ...], + discover_ms: int, + graph_ms: int, + lineage_ms: int, + contract_ms: int, + contract_cpu_ms: int, + built_in_rules_ms: int = 0, + custom_rules_ms: int = 0, + custom_rules_cpu_ms: int = 0, + early_lint_wait_ms: int = 0, + rule_cache_hits: int = 0, + rule_cache_misses: int = 0, + skipped_type_proof_rules: tuple[str, ...] = (), + ) -> None: ... diff --git a/src/sqlbuild/cli/compile/__init__.py b/src/sqlbuild/cli/compile/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py deleted file mode 100644 index 076731c0ba..0000000000 --- a/src/sqlbuild/cli/compile/models.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Data models used by the compile command path.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path - -from sqlbuild._native import ProjectColumnLineage -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.output.models import WrittenTarget -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompilerDiagnostic, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph - - -@dataclass(frozen=True) -class CompileAnalysis: - """Compiled project analysis shared by compile output phases.""" - - discovered_inputs: DiscoveredProjectInputs - adapter: BaseAdapter - graph: ProjectGraph - selected_keys: frozenset[CompiledObjectKey] - lineage: ProjectColumnLineage | None - diagnostics: tuple[CompilerDiagnostic, ...] - discover_ms: int - graph_ms: int - lineage_ms: int - contract_ms: int - contract_cpu_ms: int - built_in_rules_ms: int = 0 - custom_rules_ms: int = 0 - custom_rules_cpu_ms: int = 0 - early_lint_wait_ms: int = 0 - rule_cache_hits: int = 0 - rule_cache_misses: int = 0 - skipped_type_proof_rules: tuple[str, ...] = () - - -@dataclass(frozen=True) -class CompileWriteResult: - """Result of writing compiled artifacts with its elapsed time.""" - - written: WrittenTarget - write_ms: int - - -@dataclass(frozen=True) -class SqlTestArtifactCacheRecord: - """Validated metadata for one previously written SQL test artifact.""" - - identity: str - relative_path: Path - size: int - mtime_ns: int - content_sha256: str - - -@dataclass(frozen=True) -class SqlTestArtifactIdentityContext: - """Project-wide identity fragments shared by every SQL test.""" - - common_identity: str - model_identities: dict[str, str] diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py deleted file mode 100644 index eb9ca9c7ec..0000000000 --- a/src/sqlbuild/cli/entry/models.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Data models and handler relationships for CLI entry dispatch.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild._native import CliNamespace -from sqlbuild.cli.commands.types import ( - CompileLineageMode, - DagCommandHandler, - DebugCommandHandler, - FormatCommandHandler, - LineageCommandHandler, - QueryCommandHandler, - ScopeCommandHandler, - SkillsUpdateCommandHandler, -) -from sqlbuild.spec.contracts.types import WarehouseGroup - - -@dataclass(frozen=True) -class CompileProfileFlags: - """Profiling toggles that skip compile phases for benchmarking.""" - - skip_discovery_sql_analysis: bool = False - skip_column_inference: bool = False - skip_contracts: bool = False - skip_write: bool = False - - -@dataclass(frozen=True) -class CompileCommandRequest: - """CLI inputs for one `sqb compile` invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - defer_to: str | None = None - selected_target: str | None = None - json_output: bool = False - manifest: bool = False - dag_path: str | None = None - no_color: bool = False - lineage_mode: CompileLineageMode = CompileLineageMode.FAST - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - profile_flags: CompileProfileFlags = CompileProfileFlags() - no_cache: bool = False - debug: bool = False - - -@dataclass(frozen=True) -class SelectorFileSummary: - """Non-expanded provenance for one selector file.""" - - path: Path - selector_count: int - - -@dataclass(frozen=True) -class ParsedCliInvocation: - """Outcome of parsing CLI arguments: either a namespace or an exit code.""" - - args: CliNamespace | None - exit_code: int | None - - -@dataclass(frozen=True) -class CliEntrypointHandlers: - """Injected command handlers for the CLI entrypoint.""" - - run_compile: Callable[[CompileCommandRequest], int] - run_cost: Callable[..., int] - run_dag: DagCommandHandler - run_plan: Callable[..., int] - run_dbt_plan: Callable[[Path | None, tuple[str, ...], bool], int] - run_dbt_run: Callable[[Path | None, tuple[str, ...], bool], int] - run_dbt_build: Callable[[Path | None, tuple[str, ...], bool], int] - run_dbt_debug: Callable[[Path | None, tuple[str, ...], bool], int] - run_dbt_init: Callable[..., int] - run_build: Callable[..., int] - run_freshness: Callable[..., int] - run_test: Callable[..., int] - run_check: Callable[..., int] - run_audit: Callable[..., int] - run_seed: Callable[..., int] - run_load: Callable[..., int] - run_clone: Callable[..., int] - run_diff: Callable[..., int] - run_query: QueryCommandHandler - run_debug: DebugCommandHandler - run_lineage: LineageCommandHandler - run_janitor: Callable[..., int] - run_init: Callable[[Path | None], int] - run_playground: Callable[..., int] - run_skills_update: SkillsUpdateCommandHandler - run_format: FormatCommandHandler - run_scenario: Callable[..., int] - run_scenario_capture: Callable[..., int] - run_rules: Callable[..., int] - run_scope: ScopeCommandHandler - run_contract: Callable[..., int] | None = None - run_refactor: Callable[..., int] | None = None - - -@dataclass(frozen=True) -class CommandWarehouseScope: - """The warehouse group and CLI override active for one command invocation.""" - - group: WarehouseGroup | None - cli_warehouse: str | None = None - cli_vars: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class AuthoredTargetWarehouse: - """One target's authored, unexpanded warehouse for a command group and its file.""" - - warehouse: str - target_name: str - group: WarehouseGroup - file_name: str - - @property - def source(self) -> str: - return f"{self.file_name} [targets.{self.target_name}.warehouses] {self.group.value}" - - -@dataclass(frozen=True) -class ResolvedWarehouse: - """The warehouse one command connects with and where that value came from.""" - - warehouse: str | None - source: str diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py deleted file mode 100644 index d40b3996ce..0000000000 --- a/src/sqlbuild/cli/output/models.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Structured values shared by CLI output renderers.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild.cli.output.types import WrittenTargetDiagnostic - - -@dataclass(frozen=True) -class WrittenTarget: - """Result of writing compiled output to target/.""" - - model_count: int - seed_count: int - function_count: int - audit_count: int - test_count: int - target_dir: Path - diagnostics: tuple[WrittenTargetDiagnostic, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class SkillMaintenanceResult: - """Non-blocking generated-skill maintenance result.""" - - message: str = "" - - -@dataclass(frozen=True) -class SkillUpdateResult: - """Result of installing or updating skill files.""" - - written_paths: tuple[Path, ...] diff --git a/src/sqlbuild/cli/output/types.py b/src/sqlbuild/cli/output/types.py deleted file mode 100644 index 753c8e8c0a..0000000000 --- a/src/sqlbuild/cli/output/types.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Type-layer declarations for CLI output.""" - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from sqlbuild.compiler.compile.models import CompilerDiagnostic - -type WrittenTargetDiagnostic = CompilerDiagnostic diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index be5252782a..005153b360 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -41,8 +41,6 @@ "pydantic", "yaml", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.cli.compile.models", - "sqlbuild.cli.output.models", "sqlbuild.compiler.compile.models", "sqlbuild.compiler.discovery.models", "sqlbuild.compiler.planner.models", @@ -67,7 +65,6 @@ "polyglot_sql", "pydantic", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.cli.compile.models", "sqlbuild.compiler.compile.models", "sqlbuild.compiler.pipeline", "sqlbuild.compiler.planner.models", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index bb6f34880b..e770071e95 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -7,11 +7,11 @@ import pytest -from sqlbuild._native import parse_cli_invocation -from sqlbuild.cli.commands.types import CompileLineageMode, PlaygroundTemplate -from sqlbuild.cli.entry.models import ( +from sqlbuild._native import ( ParsedCliInvocation, + parse_cli_invocation, ) +from sqlbuild.cli.commands.types import CompileLineageMode, PlaygroundTemplate from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index d703c13970..a50bfb0da6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -7,6 +7,8 @@ import pytest from sqlbuild._native import ( + ParsedCliInvocation, + ResolvedWarehouse, command_warehouse_scope, current_command_warehouse_scope, describe_connection_warehouse, @@ -15,7 +17,6 @@ ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.cli.entry.models import ParsedCliInvocation, ResolvedWarehouse from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig from sqlbuild.spec.contracts.types import WarehouseGroup diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 0c85d6fe57..d9e5b92ba6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -7,12 +7,12 @@ from sqlbuild._native import ( PreparedCompileArtifacts, + WrittenTarget, project_with, ) from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index bd858d132b..1fb53e8c4f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -12,10 +12,9 @@ import sqlbuild._native as discovery_entry import sqlbuild._native as plan_artifacts_module -from sqlbuild._native import run_compile +from sqlbuild._native import CompileCommandRequest, run_compile from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 81860c21b6..5f8746aa9c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -8,15 +8,13 @@ from sqlbuild._native import ( ProjectColumnLineage, + WrittenTarget, compiled_model_record, format_compile_json, format_compile_text, project_with, ) from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.output.models import ( - WrittenTarget, -) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileModelConfig from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.discovery.models import SqlHookEntry diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 118615bd48..4b59180665 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -11,6 +11,7 @@ import sqlbuild._native as model_chain_module import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( + WrittenTarget, build_manifest, compiled_model_record, project_with, @@ -18,9 +19,6 @@ write_static_compile_target, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.output.models import ( - WrittenTarget, -) from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import ( CompiledModelView, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py index ced212f287..7d31f329cb 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py @@ -11,10 +11,9 @@ import pytest from filelock import FileLock, Timeout -from sqlbuild._native import PreparedCompileArtifacts +from sqlbuild._native import PreparedCompileArtifacts, WrittenTarget from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError -from sqlbuild.cli.output.models import WrittenTarget from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( StagedPublishTestCase, StagingInterleavingTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index e96da9a59c..a22e51e0ac 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -6,9 +6,9 @@ from pathlib import Path from typing import Any, Protocol, cast -from sqlbuild._native import loader_destination_parts -from sqlbuild.cli.entry.models import ( +from sqlbuild._native import ( CliEntrypointHandlers, + loader_destination_parts, ) from sqlbuild.spec.contracts.exceptions import SpecConfigError diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py index 474c7a919d..a01f30c0e6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py @@ -6,9 +6,9 @@ import pytest from _pytest.capture import CaptureResult +from sqlbuild._native import CompileCommandRequest from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild.cli.commands.main.entrypoint import _dispatch_with_compute_logs as compute_dispatch -from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.diagnostics.classes.dynamic_stderr_handler import DynamicStderrHandler from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index c1ad3c82b6..57a5178f85 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -13,6 +13,7 @@ AuditCommandRequest, BuildCommandRequest, CloneCommandRequest, + CompileCommandRequest, CostCommandRequest, DbtInitCommandRequest, DiffCommandRequest, @@ -30,7 +31,6 @@ from sqlbuild._native import run_cli as main from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.lineage.types import ColumnLineageMode diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 1a044e4a19..4980c30bad 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( PlaygroundCommandRequest, + SkillMaintenanceResult, maintain_sqlbuild_skills, run_playground_command, ) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.output.models import SkillMaintenanceResult from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, RunPlaygroundTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py index a45491a958..0b5d648b92 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/helpers.py @@ -3,8 +3,7 @@ import re from pathlib import Path, PurePosixPath -from sqlbuild._native import update_sqlbuild_skills -from sqlbuild.cli.output.models import SkillUpdateResult +from sqlbuild._native import SkillUpdateResult, update_sqlbuild_skills def write_project_files(*, project_dir: Path, files: dict[Path, str]) -> None: diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py index c5fea0e828..ef86465c73 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + SkillMaintenanceResult, load_packaged_skill_files, maintain_sqlbuild_skills, update_sqlbuild_skills, ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand -from sqlbuild.cli.output.models import SkillMaintenanceResult from tests.unit.src.sqlbuild.cli.commands.main.skills._test_types import ( SkillContentTestCase, SkillDirectoryCollisionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py index 8bd8cee382..7e3f608426 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py @@ -5,15 +5,13 @@ import pytest from sqlbuild._native import ( + SkillMaintenanceResult, + SkillUpdateResult, load_packaged_skill_files, maintain_sqlbuild_skills, update_sqlbuild_skills, ) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.output.models import ( - SkillMaintenanceResult, - SkillUpdateResult, -) from tests.unit.src.sqlbuild.cli.commands.main.skills._test_types import ( SkillMaintenanceTestCase, SkillUpdateErrorTestCase, From fbb6063f1ed65ca85d6e32696cf4875806d3585b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:09:09 +0100 Subject: [PATCH 11/23] refactor: build dbt interop manifests, arguments and plans as native records --- .../_helpers/audit_command/command.rs | 3 +- .../_helpers/build_command/completion.rs | 3 +- .../_helpers/build_command/invocation.rs | 3 +- .../bindings/_helpers/check/check_command.rs | 3 +- .../_helpers/command_runtime/invocation.rs | 3 +- .../bindings/_helpers/dbt_command/command.rs | 3 +- .../bindings/_helpers/dbt_events/results.rs | 3 +- .../src/bindings/_helpers/dbt_interop/cli.rs | 8 +- .../bindings/_helpers/dbt_interop/graph.rs | 4 +- .../bindings/_helpers/dbt_interop/manifest.rs | 10 +- .../bindings/_helpers/dbt_interop/pipeline.rs | 3 +- .../_helpers/dbt_interop/pipeline_phases.rs | 6 +- .../_helpers/dbt_interop/pipeline_prologue.rs | 10 +- .../src/bindings/_helpers/dbt_interop/plan.rs | 3 +- .../_helpers/dbt_interop/plan_output.rs | 6 +- .../_helpers/dbt_interop/runtime_config.rs | 4 +- .../_helpers/dbt_interop/runtime_execution.rs | 6 +- .../_helpers/dbt_interop/runtime_results.rs | 3 +- .../_helpers/janitor/janitor_command.rs | 5 +- .../bindings/_helpers/records/constants.rs | 283 +++++++++++ .../bindings/_helpers/seed_command/command.rs | 3 +- .../_helpers/sql_test_command/test_command.rs | 5 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 439 +++++++++++++++++- src/sqlbuild/integrations/dbt/models.py | 274 +---------- .../sqlbuild/integrations/dbt/test_graph.py | 7 +- .../sqlbuild/integrations/dbt/test_plan.py | 3 +- .../sqlbuild/cli/commands/main/dbt/helpers.py | 3 +- .../cli/commands/main/dbt/test_dbt.py | 9 +- .../planner/_helpers/resolve/test_refs.py | 2 +- .../sqlbuild/integrations/dbt/_test_types.py | 7 +- .../src/sqlbuild/integrations/dbt/helpers.py | 7 +- .../sqlbuild/integrations/dbt/test_args.py | 8 +- .../sqlbuild/integrations/dbt/test_config.py | 3 +- .../sqlbuild/integrations/dbt/test_execute.py | 23 +- .../sqlbuild/integrations/dbt/test_graph.py | 8 +- .../integrations/dbt/test_manifest.py | 8 +- .../sqlbuild/integrations/dbt/test_plan.py | 9 +- .../dbt/test_plan_orchestration.py | 11 +- .../sqlbuild/integrations/dbt/test_runner.py | 12 +- .../integrations/dbt/test_selection.py | 3 +- 41 files changed, 809 insertions(+), 409 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index a18e28ee47..a78d925e01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -15,7 +15,6 @@ use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_com use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -484,7 +483,7 @@ fn complete( fn run_audit_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, - &imported(py, NATIVE_RECORDS, "AuditInvocation")?, + &imported(py, NATIVE_MODULE, "AuditInvocation")?, true, )?; let pipeline: Bound<'_, PyAny> = compile_audits(request, &invocation)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index ba47db5aa5..2800f33c32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -26,7 +26,6 @@ use crate::bindings::_helpers::cost_report::degradation::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const COST_TYPES: &str = "sqlbuild.cost.types"; const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; @@ -76,7 +75,7 @@ pub(crate) fn finalize_build_cost<'py>( ]; fields.extend(cost.overrides); let finalization: Bound<'py, PyAny> = call_with( - &imported(py, NATIVE_RECORDS, "BuildCostFinalization")?, + &imported(py, NATIVE_MODULE, "BuildCostFinalization")?, &fields, )?; finalize_cost(&finalization) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index ef46c37a05..0845f9e4b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const SETTINGS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; @@ -196,7 +195,7 @@ pub(crate) fn resolve_build_context<'py>( load_sources.is_truthy()? }; let invocation: Bound<'py, PyAny> = call_with( - &imported(py, NATIVE_RECORDS, "BuildInvocation")?, + &imported(py, NATIVE_MODULE, "BuildInvocation")?, &[ ("effective_project_dir", project_dir), ("discovered_inputs", discovered), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 838260c9c4..0179390a01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -26,7 +26,6 @@ use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; @@ -663,7 +662,7 @@ fn complete<'py>( fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, - &imported(py, NATIVE_RECORDS, "CheckInvocation")?, + &imported(py, NATIVE_MODULE, "CheckInvocation")?, true, )?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 2aecfd0649..9d52b3acc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -8,7 +8,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; /// The project directory, discovered inputs, adapter context, colour and progress stream. type InvocationContext<'py> = ( @@ -50,7 +49,7 @@ pub(crate) fn resolve_adapter_connection_context<'py>( ], )?; call_with( - &imported(py, NATIVE_RECORDS, "AdapterConnectionContext")?, + &imported(py, NATIVE_MODULE, "AdapterConnectionContext")?, &[ ("adapter_name", adapter_name), ("adapter", adapter), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs index 5c41b0c1cb..847766f82d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::dbt_interop::runtime_config::validate_dbt_executi const NATIVE_MODULE: &str = "sqlbuild._native"; const PLAN_MODULE: &str = "sqlbuild._native"; const EXECUTE_MODULE: &str = "sqlbuild._native"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; const PLAN_COMMAND: &str = "plan"; const DEBUG_COMMAND: &str = "debug"; @@ -117,7 +116,7 @@ fn run_execution( let progress: Progress<'_> = Progress::new(py, &options, no_color)?; progress.blank_line()?; let request: Bound<'_, PyAny> = call_with( - &imported(py, DBT_MODELS, "DbtInteropExecutionRequest")?, + &imported(py, NATIVE_MODULE, "DbtInteropExecutionRequest")?, &[ ("command", command.clone()), ("project_dir", project_dir.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs index 80858efe42..450683c195 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs @@ -7,6 +7,7 @@ use sqlbuild_core::dbt_events::models::{DbtNodeMessage, DbtNodeResult}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; fn optional<'py, T: pyo3::IntoPyObject<'py>>( @@ -32,7 +33,7 @@ pub(crate) fn python_result<'py>( py: Python<'py>, result: &DbtNodeResult, ) -> PyResult> { - let message_class: Bound<'py, PyAny> = imported(py, DBT_MODELS, "DbtNodeMessage")?; + let message_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS, "DbtNodeMessage")?; let messages: Vec> = result .messages .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs index eabf25abdb..9748a99b29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const DBT_EXECUTABLE_ENV_VAR: &str = "DBT_EXECUTABLE"; @@ -219,7 +219,7 @@ fn ls_node<'py>( #[pyfunction] #[pyo3(signature = (*, stdout))] fn parse_dbt_ls_json_lines<'py>(py: Python<'py>, stdout: &str) -> PyResult> { - let node_class: Bound<'py, PyAny> = imported(py, DBT_MODELS, "DbtLsNode")?; + let node_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS, "DbtLsNode")?; let mut nodes: Vec> = Vec::new(); for line in python_splitlines(stdout) { let stripped: &str = python_strip(line); @@ -297,7 +297,7 @@ fn route_dbt_interop_args<'py>( let command: Bound<'py, PyAny> = imported(py, DBT_TYPES, "DbtInteropCommand")?.call1((kind.name(),))?; call_with( - &imported(py, DBT_MODELS, "DbtInteropRoutedArgs")?, + &imported(py, NATIVE_RECORDS, "DbtInteropRoutedArgs")?, &[ ("command", command), ("select", PyTuple::new(py, flags.select)?.into_any()), @@ -378,7 +378,7 @@ fn resolve_dbt_config<'py>( |target| PyString::new(py, &target).into_any(), ); call_with( - &imported(py, DBT_MODELS, "ResolvedDbtConfig")?, + &imported(py, NATIVE_RECORDS, "ResolvedDbtConfig")?, &[ ( "project_dir", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs index d0d82f18f8..ae9fcfc08d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const SELECTOR_EXPANSION_MODULE: &str = "sqlbuild._native"; const MODEL_SELECTORS_MODULE: &str = "sqlbuild._native"; @@ -276,7 +276,7 @@ fn selection_result<'py>(py: Python<'py>, selection: DbtSelection) -> PyResult>()?; call_with( - &imported(py, DBT_MODELS, "DbtInteropSelectionResult")?, + &imported(py, NATIVE_RECORDS, "DbtInteropSelectionResult")?, &[ ( "sqlbuild_model_names", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs index c31414eccb..44d7aa4732 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const DBT_REF_KIND: &str = "dbt_ref"; const RELATION_OPTIONS: (&[&str], &[&str]) = @@ -139,7 +139,7 @@ fn model_object<'py>( ) -> PyResult> { let py: Python<'py> = raw.py(); call_with( - &imported(py, DBT_MODELS, "DbtManifestModel")?, + &imported(py, NATIVE_RECORDS, "DbtManifestModel")?, &[ ("unique_id", unique_id.clone()), ("package_name", text(py, &model.package_name)), @@ -170,7 +170,7 @@ fn seed_object<'py>( ) -> PyResult> { let py: Python<'py> = raw.py(); call_with( - &imported(py, DBT_MODELS, "DbtManifestSeed")?, + &imported(py, NATIVE_RECORDS, "DbtManifestSeed")?, &[ ("unique_id", unique_id.clone()), ("package_name", text(py, &seed.package_name)), @@ -191,7 +191,7 @@ fn source_object<'py>( ) -> PyResult> { let py: Python<'py> = raw.py(); call_with( - &imported(py, DBT_MODELS, "DbtManifestSource")?, + &imported(py, NATIVE_RECORDS, "DbtManifestSource")?, &[ ("unique_id", unique_id.clone()), ("package_name", text(py, &source.package_name)), @@ -337,7 +337,7 @@ fn manifest_index<'py>(raw_data: &Bound<'py, PyAny>) -> PyResult( ) -> PyResult> { let args = KeywordArguments::read(py, "plan_dbt_interop_from_project", options, PLAN_OPTIONS)?; let request = call_with( - &imported(py, MODELS, "DbtInteropExecutionRequest")?, + &imported(py, NATIVE_RECORDS, "DbtInteropExecutionRequest")?, &[ ( "command", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 6f5fa1bd1a..485fddb95b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::plan_output::{connection_config, connection_hooks}; -const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const OPTIONS: (&[&str], &[&str]) = ( &["request", "invocation"], &[ @@ -31,7 +31,7 @@ fn environment<'py>(args: &KeywordArguments<'py>) -> PyResult> let request = args.value("request")?; let compiled = args.value("compiled")?; call_with( - &imported(request.py(), MODELS, "DbtPlanEnvironment")?, + &imported(request.py(), NATIVE_RECORDS, "DbtPlanEnvironment")?, &[ ("project_dir", request.getattr("project_dir")?), ( @@ -203,7 +203,7 @@ fn resolve_sqlbuild_execution_plan_output<'py>( ], )?; let request = call_with( - &imported(py, MODELS, "DbtSqlbuildPlanRequest")?, + &imported(py, NATIVE_RECORDS, "DbtSqlbuildPlanRequest")?, &[ ("selected_model_names", selected), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index 1c7f11b482..b12d63d053 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -15,7 +15,7 @@ use crate::bindings::_helpers::dbt_interop::runtime_config::{ use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; -const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const COMPILE_OPTIONS: (&[&str], &[&str]) = ( &[ "project_dir", @@ -107,7 +107,7 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( } let dbt_stream = request.getattr("dbt_stdout_stream")?; call_with( - &imported(py, MODELS, "DbtInteropInvocation")?, + &imported(py, NATIVE_RECORDS, "DbtInteropInvocation")?, &[ ("dbt_executable", executable), ("output_stream", stream.clone()), @@ -214,7 +214,7 @@ fn compile_dbt_interop_project<'py>( )?; completed(&callback, "Compiled SQLBuild project.", start)?; call_with( - &imported(py, MODELS, "DbtInteropCompiledProject")?, + &imported(py, NATIVE_RECORDS, "DbtInteropCompiledProject")?, &[ ("adapter_name", name), ("adapter", adapter), @@ -249,7 +249,7 @@ fn resolve_dbt_interop_plan<'py>( let routed = invocation.getattr("routed")?; let tuple = py.import("builtins")?.getattr("tuple")?; let command_args = call_with( - &imported(py, MODELS, "DbtInteropCommandArgs")?, + &imported(py, NATIVE_RECORDS, "DbtInteropCommandArgs")?, &[ ("select", tuple.call1((routed.getattr("select")?,))?), ("exclude", tuple.call1((routed.getattr("exclude")?,))?), @@ -279,7 +279,7 @@ fn resolve_dbt_interop_plan<'py>( )?; completed(&callback, "Resolved dbt and SQLBuild selection.", start)?; call_with( - &imported(py, MODELS, "DbtInteropPlanResolution")?, + &imported(py, NATIVE_RECORDS, "DbtInteropPlanResolution")?, &[("graph", graph), ("plan", plan)], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs index 5d2fd1314f..f36212e5ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -13,7 +13,6 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; const BUILD_OPTIONS: (&[&str], &[&str]) = ( @@ -153,7 +152,7 @@ pub(crate) fn build_dbt_interop_plan<'py>( dbt_has_work || sqlbuild_has_work, ); call_with( - &imported(py, DBT_MODELS, "DbtInteropPlan")?, + &imported(py, NATIVE_MODULE, "DbtInteropPlan")?, &[ ("command", command), ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 570b82d1fc..084519dff4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; -const MODELS: &str = "sqlbuild.integrations.dbt.models"; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const HOOKS: &str = "sqlbuild.runtime.contracts.models"; const BUILD_OPTIONS: (&[&str], &[&str]) = (&["environment", "request", "hooks"], &[]); @@ -240,7 +240,7 @@ fn attach_sqlbuild_plan_output<'py>( let compiled = args.value("compiled")?; let plan = args.value("plan")?; let environment = call_with( - &imported(py, MODELS, "DbtPlanEnvironment")?, + &imported(py, NATIVE_RECORDS, "DbtPlanEnvironment")?, &[ ("project_dir", args.value("project_dir")?), ("discovered_inputs", args.value("discovered_inputs")?), @@ -250,7 +250,7 @@ fn attach_sqlbuild_plan_output<'py>( ], )?; let request = call_with( - &imported(py, MODELS, "DbtSqlbuildPlanRequest")?, + &imported(py, NATIVE_RECORDS, "DbtSqlbuildPlanRequest")?, &[ ( "selected_model_names", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs index 574b733d81..b2316c094d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs @@ -104,7 +104,7 @@ pub(crate) fn parsed_flags( fn parsed_args_model<'py>(py: Python<'py>, flags: DbtParsedFlags) -> PyResult> { call_with( - &imported(py, DBT_MODELS, "DbtInteropParsedArgs")?, + &imported(py, NATIVE_MODULE, "DbtInteropParsedArgs")?, &[ ("select", PyTuple::new(py, flags.select)?.into_any()), ("exclude", PyTuple::new(py, flags.exclude)?.into_any()), @@ -184,7 +184,7 @@ pub(crate) fn config_overrides<'py>( ) -> PyResult> { let value = |name: &str| optional_text(py, dbt_flag_value(args, name)); call_with( - &imported(py, DBT_MODELS, "DbtCliConfigOverrides")?, + &imported(py, NATIVE_MODULE, "DbtCliConfigOverrides")?, &[ ("project_dir", value("--project-dir")), ("profiles_dir", value("--profiles-dir")), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs index c62b84ea12..9fc861a255 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs @@ -111,11 +111,7 @@ fn execute_dbt_commands<'py>( &imported(py, "sqlbuild._native", "CliStyle")?, &[("use_color", arguments.value("use_color")?)], )?; - let result_type = imported( - py, - "sqlbuild.integrations.dbt.models", - "DbtCommandExecutionResult", - )?; + let result_type = imported(py, "sqlbuild._native", "DbtCommandExecutionResult")?; if argv.is_none() { let message = arguments.value("skip_message")?; let message = if message.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs index 8aaa940d20..bd0752a308 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs @@ -8,6 +8,7 @@ use pyo3::types::{ }; use std::collections::HashSet; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const MODELS: &str = "sqlbuild.integrations.dbt.models"; pub(crate) fn result_ids(results: &Bound<'_, PyAny>) -> PyResult> { @@ -48,7 +49,7 @@ fn messages<'py>(raw: &Bound<'py, PyDict>) -> PyResult> { PyTuple::new( py, [call_with( - &imported(py, MODELS, "DbtNodeMessage")?, + &imported(py, NATIVE_RECORDS, "DbtNodeMessage")?, &[ ("level", PyString::new(py, "error").into_any()), ("message", text), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index 576dae6261..a293f77af0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -28,7 +28,6 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; @@ -117,7 +116,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( ), ], )?; - imported(py, NATIVE_RECORDS, "JanitorSettings")?.call((), Some(&options)) + imported(py, NATIVE_MODULE, "JanitorSettings")?.call((), Some(&options)) } /// Ask for the exact confirmation phrase; an interrupt cancels. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 812a28946a..7c962f1528 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -17,6 +17,7 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ ENTRY_RECORDS, OUTPUT_RECORDS, COMPILE_RECORDS, + DBT_RECORDS, ]; const NESTED_PROGRESS_CHILD_ROW: RecordSpec = RecordSpec { @@ -863,3 +864,285 @@ const COMPILE_ANALYSIS: RecordSpec = RecordSpec { /// The compiled project analysis shared by compile output phases. const COMPILE_RECORDS: &[RecordSpec] = &[COMPILE_ANALYSIS]; + +const DBT_MANIFEST_MODEL: RecordSpec = RecordSpec { + name: "DbtManifestModel", + doc: "One dbt model node needed for SQLBuild dbt_ref resolution.", + fields: &[ + FieldSpec::required("unique_id"), + FieldSpec::required("package_name"), + FieldSpec::required("name"), + FieldSpec::required("relation_name"), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("alias", FieldDefault::None), + FieldSpec::new("node_checksum", FieldDefault::None), + FieldSpec::new("fqn", FieldDefault::EmptyTuple), + FieldSpec::new("query_sql", FieldDefault::Text("")), + FieldSpec::new("depends_on_nodes", FieldDefault::EmptyTuple), + FieldSpec::new("payload", FieldDefault::EmptyDict), + ], +}; +const DBT_MANIFEST_SOURCE: RecordSpec = RecordSpec { + name: "DbtManifestSource", + doc: "One dbt source node needed for graph and lineage resolution.", + fields: &[ + FieldSpec::required("unique_id"), + FieldSpec::required("package_name"), + FieldSpec::required("source_name"), + FieldSpec::required("name"), + FieldSpec::required("relation_name"), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("identifier", FieldDefault::None), + FieldSpec::new("payload", FieldDefault::EmptyDict), + ], +}; +const DBT_MANIFEST_SEED: RecordSpec = RecordSpec { + name: "DbtManifestSeed", + doc: "One dbt seed node, a leaf data relation used for column lineage.", + fields: &[ + FieldSpec::required("unique_id"), + FieldSpec::required("package_name"), + FieldSpec::required("name"), + FieldSpec::required("relation_name"), + FieldSpec::new("database", FieldDefault::None), + FieldSpec::new("schema", FieldDefault::None), + FieldSpec::new("alias", FieldDefault::None), + FieldSpec::new("payload", FieldDefault::EmptyDict), + ], +}; +const DBT_MANIFEST_INDEX: RecordSpec = RecordSpec { + name: "DbtManifestIndex", + doc: "Lookup indexes for dbt model and source nodes in a manifest.", + fields: &[ + FieldSpec::required("models_by_unique_id"), + FieldSpec::required("models_by_name"), + FieldSpec::required("models_by_package_and_name"), + FieldSpec::new("sources_by_unique_id", FieldDefault::EmptyDict), + FieldSpec::new("seeds_by_unique_id", FieldDefault::EmptyDict), + FieldSpec::new( + "validation_depends_on_nodes_by_unique_id", + FieldDefault::EmptyDict, + ), + ], +}; +const DBT_CLI_CONFIG_OVERRIDES: RecordSpec = RecordSpec { + name: "DbtCliConfigOverrides", + doc: "dbt config values supplied by CLI flags.", + fields: &[ + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("profiles_dir", FieldDefault::None), + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("target_path", FieldDefault::None), + ], +}; +const RESOLVED_DBT_CONFIG: RecordSpec = RecordSpec { + name: "ResolvedDbtConfig", + doc: "Resolved dbt config after applying CLI overrides.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("profiles_dir"), + FieldSpec::required("target"), + FieldSpec::required("target_path"), + ], +}; +const DBT_NODE_MESSAGE: RecordSpec = RecordSpec { + name: "DbtNodeMessage", + doc: "One dbt log message attached to a dbt node.", + fields: &[FieldSpec::required("level"), FieldSpec::required("message")], +}; +const DBT_COMMAND_EXECUTION_RESULT: RecordSpec = RecordSpec { + name: "DbtCommandExecutionResult", + doc: "dbt command execution output from streamed JSON events.", + fields: &[ + FieldSpec::required("returncode"), + FieldSpec::new("node_results", FieldDefault::EmptyTuple), + ], +}; +const DBT_LS_NODE: RecordSpec = RecordSpec { + name: "DbtLsNode", + doc: "One node returned by `dbt ls --output json`.", + fields: &[ + FieldSpec::required("unique_id"), + FieldSpec::new("resource_type", FieldDefault::None), + FieldSpec::new("package_name", FieldDefault::None), + FieldSpec::new("name", FieldDefault::None), + FieldSpec::new("fqn", FieldDefault::EmptyTuple), + FieldSpec::new("original_file_path", FieldDefault::None), + ], +}; +const DBT_INTEROP_PARSED_ARGS: RecordSpec = RecordSpec { + name: "DbtInteropParsedArgs", + doc: "Declared `sqb dbt` flags parsed by argparse, before routing to buckets.", + fields: &[ + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("vars", FieldDefault::None), + FieldSpec::new("threads", FieldDefault::None), + FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("event_time_start", FieldDefault::None), + FieldSpec::new("event_time_end", FieldDefault::None), + FieldSpec::new("project_dir", FieldDefault::None), + FieldSpec::new("profiles_dir", FieldDefault::None), + FieldSpec::new("profile", FieldDefault::None), + FieldSpec::new("target", FieldDefault::None), + FieldSpec::new("target_path", FieldDefault::None), + FieldSpec::new("state", FieldDefault::None), + FieldSpec::new("indirect_selection", FieldDefault::None), + FieldSpec::new("defer", FieldDefault::False), + FieldSpec::new("start_cursor_ts", FieldDefault::None), + FieldSpec::new("end_cursor_ts", FieldDefault::None), + FieldSpec::new("start_cursor_int", FieldDefault::None), + FieldSpec::new("end_cursor_int", FieldDefault::None), + FieldSpec::new("defer_to", FieldDefault::None), + FieldSpec::new("fail_fast", FieldDefault::False), + FieldSpec::new("hard_copy", FieldDefault::False), + FieldSpec::new("dbt_passthrough", FieldDefault::EmptyTuple), + ], +}; +const DBT_INTEROP_ROUTED_ARGS: RecordSpec = RecordSpec { + name: "DbtInteropRoutedArgs", + doc: "Arguments split for a future `sqb dbt` command execution.", + fields: &[ + FieldSpec::required("command"), + FieldSpec::new("select", FieldDefault::EmptyTuple), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_args", FieldDefault::EmptyTuple), + FieldSpec::new("sqlbuild_args", FieldDefault::EmptyTuple), + ], +}; +const DBT_INTEROP_SELECTION_RESULT: RecordSpec = RecordSpec { + name: "DbtInteropSelectionResult", + doc: "SQLBuild-side selection produced from `sqb dbt` selectors.", + fields: &[ + FieldSpec::new("sqlbuild_model_names", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_required_unique_ids", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_anchor_terms", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_anchor_unique_ids_by_term", FieldDefault::EmptyDict), + FieldSpec::new("path_translations", FieldDefault::EmptyTuple), + ], +}; +const DBT_INTEROP_PLAN: RecordSpec = RecordSpec { + name: "DbtInteropPlan", + doc: "Plan output for one future `sqb dbt` command.", + fields: &[ + FieldSpec::required("command"), + FieldSpec::required("dbt_command_argv"), + FieldSpec::required("dbt_selected_nodes"), + FieldSpec::required("dbt_selected_unique_ids"), + FieldSpec::required("sqlbuild_command_argvs"), + FieldSpec::required("selection"), + FieldSpec::new("sqlbuild_plan_output", FieldDefault::None), + FieldSpec::new("dbt_required_selector_terms", FieldDefault::EmptyTuple), + FieldSpec::new("supplemental_dbt_command_argvs", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_skip_reason", FieldDefault::None), + FieldSpec::new("sqlbuild_skip_reason", FieldDefault::None), + FieldSpec::new("warnings", FieldDefault::EmptyTuple), + ], +}; +const DBT_INTEROP_EXECUTION_REQUEST: RecordSpec = RecordSpec { + name: "DbtInteropExecutionRequest", + doc: "Caller inputs for one dbt interop execution command.", + fields: &[ + FieldSpec::required("command"), + FieldSpec::required("project_dir"), + FieldSpec::required("args"), + FieldSpec::new("dbt_runner", FieldDefault::None), + FieldSpec::new("dbt_executable", FieldDefault::None), + FieldSpec::new("sqlbuild_executable", FieldDefault::Text("sqb")), + FieldSpec::new("no_sql_validation", FieldDefault::False), + FieldSpec::new("fail_fast", FieldDefault::False), + FieldSpec::new("on_progress", FieldDefault::None), + FieldSpec::new("progress_stream", FieldDefault::None), + FieldSpec::new("dbt_stdout_stream", FieldDefault::None), + FieldSpec::new("use_color", FieldDefault::False), + FieldSpec::new("verbose", FieldDefault::False), + FieldSpec::new("json_output", FieldDefault::False), + ], +}; +const DBT_INTEROP_INVOCATION: RecordSpec = RecordSpec { + name: "DbtInteropInvocation", + doc: "Resolved execution environment for one dbt interop command.", + fields: &[ + FieldSpec::required("dbt_executable"), + FieldSpec::required("output_stream"), + FieldSpec::required("dbt_output_stream"), + FieldSpec::required("routed"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("effective_sqlbuild_args"), + FieldSpec::required("dbt_options"), + FieldSpec::required("dbt_vars"), + FieldSpec::required("runner"), + ], +}; +const DBT_INTEROP_COMPILED_PROJECT: RecordSpec = RecordSpec { + name: "DbtInteropCompiledProject", + doc: "Adapter and compiled SQLBuild project for dbt interop planning.", + fields: &[ + FieldSpec::required("adapter_name"), + FieldSpec::required("adapter"), + FieldSpec::required("project"), + ], +}; +const DBT_INTEROP_PLAN_RESOLUTION: RecordSpec = RecordSpec { + name: "DbtInteropPlanResolution", + doc: "Combined dbt graph and the resolved interop plan.", + fields: &[FieldSpec::required("graph"), FieldSpec::required("plan")], +}; +const DBT_INTEROP_COMMAND_ARGS: RecordSpec = RecordSpec { + name: "DbtInteropCommandArgs", + doc: "Selection, passthrough args, and executables for a dbt interop command.", + fields: &[ + FieldSpec::required("select"), + FieldSpec::new("exclude", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_command_args", FieldDefault::EmptyTuple), + FieldSpec::new("sqlbuild_command_args", FieldDefault::EmptyTuple), + FieldSpec::new("dbt_executable", FieldDefault::Text("dbt")), + FieldSpec::new("sqlbuild_executable", FieldDefault::Text("sqb")), + ], +}; +const DBT_PLAN_ENVIRONMENT: RecordSpec = RecordSpec { + name: "DbtPlanEnvironment", + doc: "Compiled project and adapter environment for dbt plan-output builds.", + fields: &[ + FieldSpec::required("project_dir"), + FieldSpec::required("discovered_inputs"), + FieldSpec::required("project"), + FieldSpec::required("adapter"), + FieldSpec::required("adapter_name"), + ], +}; +const DBT_SQLBUILD_PLAN_REQUEST: RecordSpec = RecordSpec { + name: "DbtSqlbuildPlanRequest", + doc: "Selection and native planner inputs for a SQLBuild plan-output build.", + fields: &[ + FieldSpec::required("selected_model_names"), + FieldSpec::required("sqlbuild_args"), + FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), + ], +}; + +/// dbt manifest indexes, interop arguments, plans and invocations. +const DBT_RECORDS: &[RecordSpec] = &[ + DBT_MANIFEST_MODEL, + DBT_MANIFEST_SOURCE, + DBT_MANIFEST_SEED, + DBT_MANIFEST_INDEX, + DBT_CLI_CONFIG_OVERRIDES, + RESOLVED_DBT_CONFIG, + DBT_NODE_MESSAGE, + DBT_COMMAND_EXECUTION_RESULT, + DBT_LS_NODE, + DBT_INTEROP_PARSED_ARGS, + DBT_INTEROP_ROUTED_ARGS, + DBT_INTEROP_SELECTION_RESULT, + DBT_INTEROP_PLAN, + DBT_INTEROP_EXECUTION_REQUEST, + DBT_INTEROP_INVOCATION, + DBT_INTEROP_COMPILED_PROJECT, + DBT_INTEROP_PLAN_RESOLUTION, + DBT_INTEROP_COMMAND_ARGS, + DBT_PLAN_ENVIRONMENT, + DBT_SQLBUILD_PLAN_REQUEST, +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index d9727ea938..28e5f913bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -19,7 +19,6 @@ use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hook use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; const NATIVE_MODULE: &str = "sqlbuild._native"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -416,7 +415,7 @@ fn load_seeds<'py>( #[pyo3(signature = (*, request))] fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let invocation: Bound<'_, PyAny> = - resolve_command_invocation(request, &imported(py, NATIVE_RECORDS, "SeedInvocation")?)?; + resolve_command_invocation(request, &imported(py, NATIVE_MODULE, "SeedInvocation")?)?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; let concurrency: i64 = if requested.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 2bf64631a9..df0faa4614 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -32,7 +32,6 @@ use crate::bindings::_helpers::compile_outputs::runtime_target::write_test_runti use crate::bindings::_helpers::result_output::integration_values::json_value; const NATIVE_MODULE: &str = "sqlbuild._native"; -const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; @@ -153,7 +152,7 @@ fn outcome_status(outcome: &Bound<'_, PyAny>) -> PyResult<&'static str> { /// Nested `expect` rows of a completed test, one per step. fn expectation_rows<'py>(result: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = result.py(); - let row_class: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS, "NestedProgressChildRow")?; + let row_class: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NestedProgressChildRow")?; let name: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_name")?; let detail: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "format_expectation_detail")?; let mut rows: Vec> = Vec::new(); @@ -636,7 +635,7 @@ fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, - &imported(py, NATIVE_RECORDS, "TestInvocation")?, + &imported(py, NATIVE_MODULE, "TestInvocation")?, !inspect_only, )?; let pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; diff --git a/python-boundary.toml b/python-boundary.toml index 28290c17fd..3fd082b138 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 10394 +remaining_lines = 10122 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 54f15b9ea3..b3efb69bfa 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -329,30 +329,13 @@ from sqlbuild.executor.scenario.models import ScenarioRunResult, ScenarioSnapsho from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.models import ( - DbtCliConfigOverrides, DbtCliOptions, - DbtCommandExecutionResult, DbtCommandResult, - DbtInteropCommandArgs, - DbtInteropCompiledProject, - DbtInteropExecutionRequest, - DbtInteropInvocation, - DbtInteropParsedArgs, - DbtInteropPlan, - DbtInteropPlanResolution, - DbtInteropRoutedArgs, - DbtInteropSelectionResult, - DbtLsNode, - DbtManifestIndex, - DbtManifestModel, DbtNodeExecutionResult, - DbtPlanEnvironment, - DbtSqlbuildPlanRequest, NormalizedDbtProfileConnection, - ResolvedDbtConfig, ResolvedDbtProfileOutput, ) -from sqlbuild.integrations.dbt.types import DbtInteropCommand +from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason from sqlbuild.lint.models import ( CompileFacts, ExpandedLintFile, @@ -8683,3 +8666,423 @@ class CompileAnalysis: rule_cache_misses: int = 0, skipped_type_proof_rules: tuple[str, ...] = (), ) -> None: ... + +class DbtManifestModel: + """One dbt model node needed for SQLBuild dbt_ref resolution.""" + + unique_id: str + package_name: str + name: str + relation_name: str + database: str | None + schema: str | None + alias: str | None + node_checksum: str | None + fqn: tuple[str, ...] + query_sql: str + depends_on_nodes: tuple[str, ...] + payload: dict[str, object] + def __init__( + self, + unique_id: str, + package_name: str, + name: str, + relation_name: str, + database: str | None = None, + schema: str | None = None, + alias: str | None = None, + node_checksum: str | None = None, + fqn: tuple[str, ...] = ..., + query_sql: str = "", + depends_on_nodes: tuple[str, ...] = ..., + payload: dict[str, object] = ..., + ) -> None: ... + +class DbtManifestSource: + """One dbt source node needed for graph and lineage resolution.""" + + unique_id: str + package_name: str + source_name: str + name: str + relation_name: str + database: str | None + schema: str | None + identifier: str | None + payload: dict[str, object] + def __init__( + self, + unique_id: str, + package_name: str, + source_name: str, + name: str, + relation_name: str, + database: str | None = None, + schema: str | None = None, + identifier: str | None = None, + payload: dict[str, object] = ..., + ) -> None: ... + +class DbtManifestSeed: + """One dbt seed node, a leaf data relation used for column lineage.""" + + unique_id: str + package_name: str + name: str + relation_name: str + database: str | None + schema: str | None + alias: str | None + payload: dict[str, object] + def __init__( + self, + unique_id: str, + package_name: str, + name: str, + relation_name: str, + database: str | None = None, + schema: str | None = None, + alias: str | None = None, + payload: dict[str, object] = ..., + ) -> None: ... + +class DbtManifestIndex: + """Lookup indexes for dbt model and source nodes in a manifest.""" + + models_by_unique_id: dict[str, DbtManifestModel] + models_by_name: dict[str, tuple[DbtManifestModel, ...]] + models_by_package_and_name: dict[tuple[str, str], DbtManifestModel] + sources_by_unique_id: dict[str, DbtManifestSource] + seeds_by_unique_id: dict[str, DbtManifestSeed] + validation_depends_on_nodes_by_unique_id: dict[str, tuple[str, ...]] + def __init__( + self, + models_by_unique_id: dict[str, DbtManifestModel], + models_by_name: dict[str, tuple[DbtManifestModel, ...]], + models_by_package_and_name: dict[tuple[str, str], DbtManifestModel], + sources_by_unique_id: dict[str, DbtManifestSource] = ..., + seeds_by_unique_id: dict[str, DbtManifestSeed] = ..., + validation_depends_on_nodes_by_unique_id: dict[str, tuple[str, ...]] = ..., + ) -> None: ... + +class DbtCliConfigOverrides: + """dbt config values supplied by CLI flags.""" + + project_dir: str | None + profiles_dir: str | None + target: str | None + target_path: str | None + def __init__( + self, + project_dir: str | None = None, + profiles_dir: str | None = None, + target: str | None = None, + target_path: str | None = None, + ) -> None: ... + +class ResolvedDbtConfig: + """Resolved dbt config after applying CLI overrides.""" + + project_dir: Path | None + profiles_dir: Path | None + target: str | None + target_path: Path | None + def __init__( + self, + project_dir: Path | None, + profiles_dir: Path | None, + target: str | None, + target_path: Path | None, + ) -> None: ... + +class DbtNodeMessage: + """One dbt log message attached to a dbt node.""" + + level: str + message: str + def __init__( + self, + level: str, + message: str, + ) -> None: ... + +class DbtCommandExecutionResult: + """dbt command execution output from streamed JSON events.""" + + returncode: int + node_results: tuple[DbtNodeExecutionResult, ...] + def __init__( + self, + returncode: int, + node_results: tuple[DbtNodeExecutionResult, ...] = ..., + ) -> None: ... + +class DbtLsNode: + """One node returned by `dbt ls --output json`.""" + + unique_id: str + resource_type: str | None + package_name: str | None + name: str | None + fqn: tuple[str, ...] + original_file_path: str | None + def __init__( + self, + unique_id: str, + resource_type: str | None = None, + package_name: str | None = None, + name: str | None = None, + fqn: tuple[str, ...] = ..., + original_file_path: str | None = None, + ) -> None: ... + +class DbtInteropParsedArgs: + """Declared `sqb dbt` flags parsed by argparse, before routing to buckets.""" + + select: tuple[str, ...] + exclude: tuple[str, ...] + vars: str | None + threads: str | None + full_refresh: bool + event_time_start: str | None + event_time_end: str | None + project_dir: str | None + profiles_dir: str | None + profile: str | None + target: str | None + target_path: str | None + state: str | None + indirect_selection: str | None + defer: bool + start_cursor_ts: str | None + end_cursor_ts: str | None + start_cursor_int: str | None + end_cursor_int: str | None + defer_to: str | None + fail_fast: bool + hard_copy: bool + dbt_passthrough: tuple[str, ...] + def __init__( + self, + select: tuple[str, ...] = ..., + exclude: tuple[str, ...] = ..., + vars: str | None = None, + threads: str | None = None, + full_refresh: bool = False, + event_time_start: str | None = None, + event_time_end: str | None = None, + project_dir: str | None = None, + profiles_dir: str | None = None, + profile: str | None = None, + target: str | None = None, + target_path: str | None = None, + state: str | None = None, + indirect_selection: str | None = None, + defer: bool = False, + start_cursor_ts: str | None = None, + end_cursor_ts: str | None = None, + start_cursor_int: str | None = None, + end_cursor_int: str | None = None, + defer_to: str | None = None, + fail_fast: bool = False, + hard_copy: bool = False, + dbt_passthrough: tuple[str, ...] = ..., + ) -> None: ... + +class DbtInteropRoutedArgs: + """Arguments split for a future `sqb dbt` command execution.""" + + command: DbtInteropCommand + select: tuple[str, ...] + exclude: tuple[str, ...] + dbt_args: tuple[str, ...] + sqlbuild_args: tuple[str, ...] + def __init__( + self, + command: DbtInteropCommand, + select: tuple[str, ...] = ..., + exclude: tuple[str, ...] = ..., + dbt_args: tuple[str, ...] = ..., + sqlbuild_args: tuple[str, ...] = ..., + ) -> None: ... + +class DbtInteropSelectionResult: + """SQLBuild-side selection produced from `sqb dbt` selectors.""" + + sqlbuild_model_names: tuple[str, ...] + dbt_required_unique_ids: tuple[str, ...] + dbt_anchor_terms: tuple[str, ...] + dbt_anchor_unique_ids_by_term: dict[str, tuple[str, ...]] + path_translations: tuple[tuple[str, str], ...] + def __init__( + self, + sqlbuild_model_names: tuple[str, ...] = ..., + dbt_required_unique_ids: tuple[str, ...] = ..., + dbt_anchor_terms: tuple[str, ...] = ..., + dbt_anchor_unique_ids_by_term: dict[str, tuple[str, ...]] = ..., + path_translations: tuple[tuple[str, str], ...] = ..., + ) -> None: ... + +class DbtInteropPlan: + """Plan output for one future `sqb dbt` command.""" + + command: DbtInteropCommand + dbt_command_argv: tuple[str, ...] + dbt_selected_nodes: tuple[DbtLsNode, ...] + dbt_selected_unique_ids: tuple[str, ...] + sqlbuild_command_argvs: tuple[tuple[str, ...], ...] + selection: DbtInteropSelectionResult + sqlbuild_plan_output: PlanOutput | None + dbt_required_selector_terms: tuple[str, ...] + supplemental_dbt_command_argvs: tuple[tuple[str, ...], ...] + dbt_skip_reason: DbtInteropSkipReason | None + sqlbuild_skip_reason: DbtInteropSkipReason | None + warnings: tuple[str, ...] + def __init__( + self, + command: DbtInteropCommand, + dbt_command_argv: tuple[str, ...], + dbt_selected_nodes: tuple[DbtLsNode, ...], + dbt_selected_unique_ids: tuple[str, ...], + sqlbuild_command_argvs: tuple[tuple[str, ...], ...], + selection: DbtInteropSelectionResult, + sqlbuild_plan_output: PlanOutput | None = None, + dbt_required_selector_terms: tuple[str, ...] = ..., + supplemental_dbt_command_argvs: tuple[tuple[str, ...], ...] = ..., + dbt_skip_reason: DbtInteropSkipReason | None = None, + sqlbuild_skip_reason: DbtInteropSkipReason | None = None, + warnings: tuple[str, ...] = ..., + ) -> None: ... + +class DbtInteropExecutionRequest: + """Caller inputs for one dbt interop execution command.""" + + command: DbtInteropCommand + project_dir: Path + args: tuple[str, ...] + dbt_runner: Any | None + dbt_executable: str | None + sqlbuild_executable: str + no_sql_validation: bool + fail_fast: bool + on_progress: Callable[[str], None] | None + progress_stream: TextIO | None + dbt_stdout_stream: TextIO | None + use_color: bool + verbose: bool + json_output: bool + def __init__( + self, + command: DbtInteropCommand, + project_dir: Path, + args: tuple[str, ...], + dbt_runner: Any | None = None, + dbt_executable: str | None = None, + sqlbuild_executable: str = "sqb", + no_sql_validation: bool = False, + fail_fast: bool = False, + on_progress: Callable[[str], None] | None = None, + progress_stream: TextIO | None = None, + dbt_stdout_stream: TextIO | None = None, + use_color: bool = False, + verbose: bool = False, + json_output: bool = False, + ) -> None: ... + +class DbtInteropInvocation: + """Resolved execution environment for one dbt interop command.""" + + dbt_executable: str + output_stream: TextIO + dbt_output_stream: TextIO + routed: DbtInteropRoutedArgs + discovered_inputs: DiscoveredProjectInputs + effective_sqlbuild_args: tuple[str, ...] + dbt_options: DbtCliOptions + dbt_vars: dict[str, object] + runner: Any + def __init__( + self, + dbt_executable: str, + output_stream: TextIO, + dbt_output_stream: TextIO, + routed: DbtInteropRoutedArgs, + discovered_inputs: DiscoveredProjectInputs, + effective_sqlbuild_args: tuple[str, ...], + dbt_options: DbtCliOptions, + dbt_vars: dict[str, object], + runner: Any, + ) -> None: ... + +class DbtInteropCompiledProject: + """Adapter and compiled SQLBuild project for dbt interop planning.""" + + adapter_name: str + adapter: BaseAdapter + project: CompiledProjectView + def __init__( + self, + adapter_name: str, + adapter: BaseAdapter, + project: CompiledProjectView, + ) -> None: ... + +class DbtInteropPlanResolution: + """Combined dbt graph and the resolved interop plan.""" + + graph: DbtCombinedGraph + plan: DbtInteropPlan + def __init__( + self, + graph: DbtCombinedGraph, + plan: DbtInteropPlan, + ) -> None: ... + +class DbtInteropCommandArgs: + """Selection, passthrough args, and executables for a dbt interop command.""" + + select: tuple[str, ...] + exclude: tuple[str, ...] + dbt_command_args: tuple[str, ...] + sqlbuild_command_args: tuple[str, ...] + dbt_executable: str + sqlbuild_executable: str + def __init__( + self, + select: tuple[str, ...], + exclude: tuple[str, ...] = (), + dbt_command_args: tuple[str, ...] = (), + sqlbuild_command_args: tuple[str, ...] = (), + dbt_executable: str = "dbt", + sqlbuild_executable: str = "sqb", + ) -> None: ... + +class DbtPlanEnvironment: + """Compiled project and adapter environment for dbt plan-output builds.""" + + project_dir: Path + discovered_inputs: DiscoveredProjectInputs + project: CompiledProjectView + adapter: BaseAdapter + adapter_name: str + def __init__( + self, + project_dir: Path, + discovered_inputs: DiscoveredProjectInputs, + project: CompiledProjectView, + adapter: BaseAdapter, + adapter_name: str, + ) -> None: ... + +class DbtSqlbuildPlanRequest: + """Selection and native planner inputs for a SQLBuild plan-output build.""" + + selected_model_names: tuple[str, ...] + sqlbuild_args: tuple[str, ...] + external_blocked_model_names: tuple[str, ...] + def __init__( + self, + selected_model_names: tuple[str, ...], + sqlbuild_args: tuple[str, ...], + external_blocked_model_names: tuple[str, ...] = (), + ) -> None: ... diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 742e719653..9c8d1aba92 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -5,75 +5,8 @@ from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path -from typing import Any, TextIO -from sqlbuild._native import DbtCombinedGraph, dbt_fqn_selector_term -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason - - -@dataclass(frozen=True) -class DbtManifestModel: - """One dbt model node needed for SQLBuild dbt_ref resolution.""" - - unique_id: str - package_name: str - name: str - relation_name: str - database: str | None = None - schema: str | None = None - alias: str | None = None - node_checksum: str | None = None - fqn: tuple[str, ...] = field(default_factory=tuple) - query_sql: str = "" - depends_on_nodes: tuple[str, ...] = field(default_factory=tuple) - payload: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DbtManifestSource: - """One dbt source node needed for graph and lineage resolution.""" - - unique_id: str - package_name: str - source_name: str - name: str - relation_name: str - database: str | None = None - schema: str | None = None - identifier: str | None = None - payload: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DbtManifestSeed: - """One dbt seed node, a leaf data relation used for column lineage.""" - - unique_id: str - package_name: str - name: str - relation_name: str - database: str | None = None - schema: str | None = None - alias: str | None = None - payload: dict[str, object] = field(default_factory=dict) - - -@dataclass(frozen=True) -class DbtManifestIndex: - """Lookup indexes for dbt model and source nodes in a manifest.""" - - models_by_unique_id: dict[str, DbtManifestModel] - models_by_name: dict[str, tuple[DbtManifestModel, ...]] - models_by_package_and_name: dict[tuple[str, str], DbtManifestModel] - sources_by_unique_id: dict[str, DbtManifestSource] = field(default_factory=dict) - seeds_by_unique_id: dict[str, DbtManifestSeed] = field(default_factory=dict) - validation_depends_on_nodes_by_unique_id: dict[str, tuple[str, ...]] = field( - default_factory=dict - ) +from sqlbuild._native import DbtLsNode, DbtNodeMessage @dataclass(frozen=True) @@ -89,26 +22,6 @@ class DbtCliOptions: defer: bool = False -@dataclass(frozen=True) -class DbtCliConfigOverrides: - """dbt config values supplied by CLI flags.""" - - project_dir: str | None = None - profiles_dir: str | None = None - target: str | None = None - target_path: str | None = None - - -@dataclass(frozen=True) -class ResolvedDbtConfig: - """Resolved dbt config after applying CLI overrides.""" - - project_dir: Path | None - profiles_dir: Path | None - target: str | None - target_path: Path | None - - @dataclass(frozen=True) class DbtProjectProfileMetadata: """dbt project metadata needed for profile resolution.""" @@ -214,14 +127,6 @@ class DbtCommandResult: stderr: str = "" -@dataclass(frozen=True) -class DbtNodeMessage: - """One dbt log message attached to a dbt node.""" - - level: str - message: str - - @dataclass(frozen=True) class DbtNodeExecutionResult: """One dbt node execution result parsed from JSON logs.""" @@ -241,186 +146,9 @@ class DbtNodeExecutionResult: messages: tuple[DbtNodeMessage, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class DbtCommandExecutionResult: - """dbt command execution output from streamed JSON events.""" - - returncode: int - node_results: tuple[DbtNodeExecutionResult, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtLsNode: - """One node returned by `dbt ls --output json`.""" - - unique_id: str - resource_type: str | None = None - package_name: str | None = None - name: str | None = None - fqn: tuple[str, ...] = field(default_factory=tuple) - original_file_path: str | None = None - - @property - def selector_term(self) -> str: - return dbt_fqn_selector_term(fqn=self.fqn, fallback=self.name or self.unique_id) - - @dataclass(frozen=True) class DbtLsResult: """Parsed result from one dbt ls invocation.""" nodes: tuple[DbtLsNode, ...] command: DbtCommandResult - - -@dataclass(frozen=True) -class DbtInteropParsedArgs: - """Declared `sqb dbt` flags parsed by argparse, before routing to buckets.""" - - select: tuple[str, ...] = field(default_factory=tuple) - exclude: tuple[str, ...] = field(default_factory=tuple) - vars: str | None = None - threads: str | None = None - full_refresh: bool = False - event_time_start: str | None = None - event_time_end: str | None = None - project_dir: str | None = None - profiles_dir: str | None = None - profile: str | None = None - target: str | None = None - target_path: str | None = None - state: str | None = None - indirect_selection: str | None = None - defer: bool = False - start_cursor_ts: str | None = None - end_cursor_ts: str | None = None - start_cursor_int: str | None = None - end_cursor_int: str | None = None - defer_to: str | None = None - fail_fast: bool = False - hard_copy: bool = False - dbt_passthrough: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtInteropRoutedArgs: - """Arguments split for a future `sqb dbt` command execution.""" - - command: DbtInteropCommand - select: tuple[str, ...] = field(default_factory=tuple) - exclude: tuple[str, ...] = field(default_factory=tuple) - dbt_args: tuple[str, ...] = field(default_factory=tuple) - sqlbuild_args: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtInteropSelectionResult: - """SQLBuild-side selection produced from `sqb dbt` selectors.""" - - sqlbuild_model_names: tuple[str, ...] = field(default_factory=tuple) - dbt_required_unique_ids: tuple[str, ...] = field(default_factory=tuple) - dbt_anchor_terms: tuple[str, ...] = field(default_factory=tuple) - dbt_anchor_unique_ids_by_term: dict[str, tuple[str, ...]] = field(default_factory=dict) - path_translations: tuple[tuple[str, str], ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtInteropPlan: - """Plan output for one future `sqb dbt` command.""" - - command: DbtInteropCommand - dbt_command_argv: tuple[str, ...] - dbt_selected_nodes: tuple[DbtLsNode, ...] - dbt_selected_unique_ids: tuple[str, ...] - sqlbuild_command_argvs: tuple[tuple[str, ...], ...] - selection: DbtInteropSelectionResult - sqlbuild_plan_output: PlanOutput | None = None - dbt_required_selector_terms: tuple[str, ...] = field(default_factory=tuple) - supplemental_dbt_command_argvs: tuple[tuple[str, ...], ...] = field(default_factory=tuple) - dbt_skip_reason: DbtInteropSkipReason | None = None - sqlbuild_skip_reason: DbtInteropSkipReason | None = None - warnings: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class DbtInteropExecutionRequest: - """Caller inputs for one dbt interop execution command.""" - - command: DbtInteropCommand - project_dir: Path - args: tuple[str, ...] - dbt_runner: Any | None = None - dbt_executable: str | None = None - sqlbuild_executable: str = "sqb" - no_sql_validation: bool = False - fail_fast: bool = False - on_progress: Callable[[str], None] | None = None - progress_stream: TextIO | None = None - dbt_stdout_stream: TextIO | None = None - use_color: bool = False - verbose: bool = False - json_output: bool = False - - -@dataclass(frozen=True) -class DbtInteropInvocation: - """Resolved execution environment for one dbt interop command.""" - - dbt_executable: str - output_stream: TextIO - dbt_output_stream: TextIO - routed: DbtInteropRoutedArgs - discovered_inputs: DiscoveredProjectInputs - effective_sqlbuild_args: tuple[str, ...] - dbt_options: DbtCliOptions - dbt_vars: dict[str, object] - runner: Any - - -@dataclass(frozen=True) -class DbtInteropCompiledProject: - """Adapter and compiled SQLBuild project for dbt interop planning.""" - - adapter_name: str - adapter: BaseAdapter - project: CompiledProjectView - - -@dataclass(frozen=True) -class DbtInteropPlanResolution: - """Combined dbt graph and the resolved interop plan.""" - - graph: DbtCombinedGraph - plan: DbtInteropPlan - - -@dataclass(frozen=True) -class DbtInteropCommandArgs: - """Selection, passthrough args, and executables for a dbt interop command.""" - - select: tuple[str, ...] - exclude: tuple[str, ...] = () - dbt_command_args: tuple[str, ...] = () - sqlbuild_command_args: tuple[str, ...] = () - dbt_executable: str = "dbt" - sqlbuild_executable: str = "sqb" - - -@dataclass(frozen=True) -class DbtPlanEnvironment: - """Compiled project and adapter environment for dbt plan-output builds.""" - - project_dir: Path - discovered_inputs: DiscoveredProjectInputs - project: CompiledProjectView - adapter: BaseAdapter - adapter_name: str - - -@dataclass(frozen=True) -class DbtSqlbuildPlanRequest: - """Selection and native planner inputs for a SQLBuild plan-output build.""" - - selected_model_names: tuple[str, ...] - sqlbuild_args: tuple[str, ...] - external_blocked_model_names: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index 647476caad..4cd962a4ce 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( DbtCombinedGraph, + DbtManifestIndex, assemble_project, build_compile_inputs, build_dbt_combined_graph, @@ -16,11 +17,7 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandResult, - DbtManifestIndex, -) +from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtCombinedGraphTestCase, ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py index 7ec1102362..dbb431e2b0 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import plan_dbt_interop_from_project -from sqlbuild.integrations.dbt.models import DbtInteropPlan +from sqlbuild._native import DbtInteropPlan, plan_dbt_interop_from_project from sqlbuild.integrations.dbt.types import DbtInteropSkipReason from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtInteropPlanTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py index 760b9d84af..57327532d8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -2,8 +2,7 @@ from pathlib import Path -from sqlbuild._native import build_dbt_interop_plan -from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult +from sqlbuild._native import DbtInteropPlan, DbtInteropSelectionResult, build_dbt_interop_plan from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import DbtAutoInitTestCase diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index 8e6edb1ba5..6d16407fa3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -7,13 +7,8 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import run_dbt_command -from sqlbuild.integrations.dbt.models import ( - DbtInitRequest, - DbtInitResult, - DbtInteropExecutionRequest, - DbtInteropPlan, -) +from sqlbuild._native import DbtInteropExecutionRequest, DbtInteropPlan, run_dbt_command +from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import ( DbtAutoInitTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index e790e6ecb7..9a83879c95 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( DbtCompileReferenceResolver, + DbtManifestIndex, apply_deferred_locations, build_dbt_manifest_index, resolve_dbt_ref_references, @@ -21,7 +22,6 @@ ) from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import CursorBounds -from sqlbuild.integrations.dbt.models import DbtManifestIndex from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, RefResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py index b26afb0d0e..2c4faefd40 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/_test_types.py @@ -3,11 +3,8 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandResult, - DbtInteropParsedArgs, -) +from sqlbuild._native import DbtInteropParsedArgs +from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index dbe8f4735e..46e4b5ded1 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( BackfillResult, + DbtCliConfigOverrides, DbtCombinedGraph, DbtEventStream, DbtEventStreamUpdate, @@ -42,11 +43,7 @@ ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.models import ( - DbtCliConfigOverrides, - DbtCliOptions, - DbtCommandResult, -) +from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py index 20cd87212c..dbd1174d51 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py @@ -2,9 +2,13 @@ import pytest -from sqlbuild._native import parse_dbt_execution_args, route_dbt_interop_args +from sqlbuild._native import ( + DbtInteropParsedArgs, + DbtInteropRoutedArgs, + parse_dbt_execution_args, + route_dbt_interop_args, +) from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import DbtInteropParsedArgs, DbtInteropRoutedArgs from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtArgParseErrorTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py index 083a9795a0..de65571383 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_config.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_config.py @@ -4,9 +4,8 @@ import pytest -from sqlbuild._native import resolve_dbt_config, resolve_dbt_vars_mapping +from sqlbuild._native import ResolvedDbtConfig, resolve_dbt_config, resolve_dbt_vars_mapping from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError -from sqlbuild.integrations.dbt.models import ResolvedDbtConfig from sqlbuild.spec.contracts.models import DbtConfig, LocalDbtConfig from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtConfigErrorTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 1f94a81d74..9e2d338288 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -13,18 +13,6 @@ from sqlbuild import _native as plan_module from sqlbuild._native import ( DbtCombinedGraph, - build_dbt_interop_plan, - build_failed_sqlbuild_model_names, - build_merged_dbt_execution_argv, - execute_dbt_commands, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.main.runtime import _execute_json_event_stream as event_module -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, DbtCommandExecutionResult, DbtInteropCompiledProject, DbtInteropExecutionRequest, @@ -35,8 +23,17 @@ DbtInteropSelectionResult, DbtLsNode, DbtManifestIndex, - DbtNodeExecutionResult, + build_dbt_interop_plan, + build_failed_sqlbuild_model_names, + build_merged_dbt_execution_argv, + execute_dbt_commands, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner +from sqlbuild.integrations.dbt.main.runtime import _execute_json_event_stream as event_module +from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtNodeExecutionResult from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py index a030759270..0d0aec220d 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py @@ -2,9 +2,13 @@ import pytest -from sqlbuild._native import DbtCombinedGraph, build_dbt_combined_graph, build_dbt_manifest_index +from sqlbuild._native import ( + DbtCombinedGraph, + DbtManifestIndex, + build_dbt_combined_graph, + build_dbt_manifest_index, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.integrations.dbt.models import DbtManifestIndex from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtCombinedGraphTestCase from tests.unit.src.sqlbuild.integrations.dbt.helpers import ( build_compiled_project_with_models, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py index a0f9cafdca..588e28708b 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_manifest.py @@ -3,15 +3,13 @@ import pytest from sqlbuild._native import ( - build_dbt_manifest_index, - resolve_dbt_manifest_model, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.integrations.dbt.models import ( DbtManifestIndex, DbtManifestModel, DbtManifestSource, + build_dbt_manifest_index, + resolve_dbt_manifest_model, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtManifestIndexErrorTestCase, DbtManifestResolutionErrorTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py index f386c5a12c..4fc93401ce 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan.py @@ -2,8 +2,13 @@ import pytest -from sqlbuild._native import build_dbt_interop_plan, render_dbt_interop_plan -from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode +from sqlbuild._native import ( + DbtInteropPlan, + DbtInteropSelectionResult, + DbtLsNode, + build_dbt_interop_plan, + render_dbt_interop_plan, +) from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtPlanTestCase diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index 47de125bf9..51e6793cc8 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -6,6 +6,9 @@ from sqlbuild._native import ( DbtCombinedGraph, + DbtInteropCommandArgs, + DbtInteropPlan, + DbtManifestIndex, build_dbt_combined_graph, build_dbt_ls_argv, build_dbt_manifest_index, @@ -14,13 +17,7 @@ from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.models import ( - DbtCliOptions, - DbtCommandResult, - DbtInteropCommandArgs, - DbtInteropPlan, - DbtManifestIndex, -) +from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtPlanOrchestrationErrorTestCase, DbtPlanOrchestrationTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py b/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py index caae1125c2..e28c248bf8 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_runner.py @@ -5,14 +5,16 @@ import pytest from sqlbuild._native import ( + DbtLsNode, build_dbt_compile_argv, build_dbt_debug_argv, build_dbt_deps_argv, build_dbt_ls_argv, + dbt_fqn_selector_term, parse_dbt_ls_json_lines, ) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult, DbtLsNode, DbtLsResult +from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtCommandResult, DbtLsResult from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtArgvTestCase, DbtLsParseTestCase, @@ -233,7 +235,13 @@ def test_given_dbt_ls_output_when_parsing_then_returns_unique_id_nodes( assert tuple(node.unique_id for node in result) == test_case.expected_unique_ids assert tuple(node.resource_type for node in result) == test_case.expected_resource_types - assert tuple(node.selector_term for node in result) == test_case.expected_selector_terms + assert ( + tuple( + dbt_fqn_selector_term(fqn=node.fqn, fallback=node.name or node.unique_id) + for node in result + ) + == test_case.expected_selector_terms + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py index 8e552420cc..8f96f16465 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py @@ -4,13 +4,14 @@ from sqlbuild._native import ( DbtCombinedGraph, + DbtInteropSelectionResult, + DbtManifestIndex, build_dbt_combined_graph, build_dbt_manifest_index, resolve_dbt_interop_sqlbuild_selection, ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.models import DbtInteropSelectionResult, DbtManifestIndex from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtSelectionErrorTestCase, DbtSelectionTestCase, From 3c2f8dec7be3a14332187507b1227b48e1484a24 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:29:05 +0100 Subject: [PATCH 12/23] refactor: build source freshness observations and outcomes as native records --- .../_helpers/freshness/observation.rs | 37 +++-- .../bindings/_helpers/freshness/planning.rs | 14 +- .../bindings/_helpers/freshness/state_io.rs | 5 +- .../bindings/_helpers/freshness/support.rs | 1 - .../bindings/_helpers/records/constants.rs | 122 +++++++++++++++- .../src/bindings/_helpers/records/fields.rs | 11 +- .../src/bindings/_helpers/records/mod.rs | 1 + .../src/bindings/_helpers/records/models.rs | 9 ++ .../bindings/_helpers/records/registration.rs | 26 +++- .../src/bindings/_helpers/records/types.rs | 6 + .../bindings/_helpers/state_rows/freshness.rs | 4 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 133 ++++++++++++++++-- .../_helpers/freshness/source_freshness.py | 9 +- .../compiler/source_freshness/models.py | 96 +------------ .../compiler/source_freshness/types.py | 32 ----- .../cli/commands/main/janitor/helpers.py | 11 +- .../source_freshness/main/_test_types.py | 3 +- .../source_freshness/main/test_read_write.py | 13 +- .../_helpers/freshness/_test_types.py | 3 +- .../sqlbuild/cli/commands/_helpers/helpers.py | 11 +- .../_helpers/test_source_freshness.py | 7 +- .../_helpers/test_selection_staleness.py | 2 + .../source_freshness/main/_test_types.py | 2 +- .../test_observation.py | 2 +- .../compiler/source_freshness/main/helpers.py | 8 +- .../main/test_adapter_observation.py | 8 +- .../source_freshness/main/test_age_policy.py | 3 +- .../source_freshness/main/test_models.py | 6 +- .../source_freshness/main/test_observation.py | 2 +- .../source_freshness/main/test_planning.py | 2 +- .../source_freshness/main/test_propagation.py | 8 +- .../source_freshness/main/test_read.py | 10 +- 33 files changed, 388 insertions(+), 221 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/types.rs delete mode 100644 src/sqlbuild/compiler/source_freshness/types.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs index 2d8525fd51..458b0a11b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/observation.rs @@ -1,7 +1,7 @@ //! Observe configured and adapter-metadata source freshness through adapter callbacks. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDateTime, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_state::freshness::main::single_value_problem::single_value_problem; use sqlbuild_state::freshness::models::QueryShape; @@ -9,13 +9,15 @@ use sqlbuild_state::freshness::models::QueryShape; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::freshness::support::{ ADAPTER_MODELS, ADAPTER_TYPES, CONTRACT_MODELS, CONTRACT_TYPES, FRESHNESS_MODELS, - FRESHNESS_TYPES, LIFECYCLE_MODULE, OBSERVABILITY_MODELS, OBSERVATION_ERROR, exception_text, - freshness_error, presumed_utc_datetime, presumed_utc_value, text, + LIFECYCLE_MODULE, OBSERVABILITY_MODELS, OBSERVATION_ERROR, exception_text, freshness_error, + presumed_utc_datetime, presumed_utc_value, text, }; +use crate::bindings::_helpers::records::constants::{DATA_VERSION_FIELD, OBSERVED_AT_FIELD}; use crate::bindings::_helpers::references::source_relations::render_source_relation; use crate::bindings::_helpers::state_rows::freshness::{normalize, version_hash}; use crate::bindings::_helpers::state_rows::timestamps::{in_utc, offset_aware}; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const ADAPTER_STRATEGY: &str = "adapter"; const COLUMN_STRATEGY: &str = "column"; const DEFAULT_STRATEGY: &str = "sql"; @@ -117,7 +119,7 @@ fn observation<'py>( call_with( &imported( source_name.py(), - FRESHNESS_MODELS, + NATIVE_RECORDS, "SourceFreshnessObservation", )?, &[ @@ -307,12 +309,12 @@ pub(crate) fn unknown<'py>( ) -> PyResult> { let py: Python<'py> = source_name.py(); call_with( - &imported(py, FRESHNESS_MODELS, "SourceFreshnessUnknown")?, + &imported(py, NATIVE_RECORDS, "SourceFreshnessUnknown")?, &[ ("source_name", source_name.clone()), ( "reason", - imported(py, FRESHNESS_TYPES, "SourceFreshnessUnknownReason")?.getattr(reason)?, + imported(py, NATIVE_RECORDS, "SourceFreshnessUnknownReason")?.getattr(reason)?, ), ("message", message), ], @@ -404,7 +406,7 @@ pub(crate) fn observe_adapter_batch<'py>( ) -> PyResult> { let py: Python<'py> = observer.adapter.py(); let batch_class: Bound<'py, PyAny> = - imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?; + imported(py, NATIVE_RECORDS, "AdapterSourceFreshnessBatch")?; if sources.is_empty() { return batch_class.call0(); } @@ -432,7 +434,7 @@ fn observe_adapter_batch_inner<'py>( ) -> PyResult> { let py: Python<'py> = observer.adapter.py(); let batch_class: Bound<'py, PyAny> = - imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?; + imported(py, NATIVE_RECORDS, "AdapterSourceFreshnessBatch")?; let unknown_sources: Bound<'py, PyDict> = PyDict::new(py); let requests: Bound<'py, PyDict> = PyDict::new(py); let request_class: Bound<'py, PyAny> = imported(py, ADAPTER_MODELS, "TableFreshnessRequest")?; @@ -602,6 +604,23 @@ fn require_aware_utc_datetime<'py>( in_utc(value) } +/// Hold an observation's `observed_at` and datetime `data_version` as aware UTC values. +pub(crate) fn normalize_observation( + py: Python<'_>, + mut values: Vec>, +) -> PyResult>> { + let observed_at: Bound<'_, PyAny> = + require_aware_utc_datetime(values[OBSERVED_AT_FIELD].bind(py), "observed_at")?; + values[OBSERVED_AT_FIELD] = observed_at.unbind(); + let data_version: &Bound<'_, PyAny> = values[DATA_VERSION_FIELD].bind(py); + if data_version.is_instance_of::() { + let data_version: Bound<'_, PyAny> = + require_aware_utc_datetime(data_version, "data_version")?; + values[DATA_VERSION_FIELD] = data_version.unbind(); + } + Ok(values) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(require_aware_utc_datetime, module)?)?; module.add_function(wrap_pyfunction!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index e352a88857..c4f56b8921 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -16,12 +16,12 @@ use crate::bindings::_helpers::freshness::observation::{ }; use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::freshness::support::{ - ADAPTER_EXCEPTIONS, EXCEPTIONS_MODULE, FRESHNESS_MODELS, FRESHNESS_TYPES, OBSERVATION_ERROR, - text, + ADAPTER_EXCEPTIONS, EXCEPTIONS_MODULE, FRESHNESS_MODELS, OBSERVATION_ERROR, text, }; use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const ADAPTER_STRATEGY: &str = "adapter"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; @@ -124,7 +124,7 @@ fn adapter_batch<'py>( unknown_sources.set_item(&name, unknown(&name, ("ERROR", message.clone()))?)?; } call_with( - &imported(py, FRESHNESS_MODELS, "AdapterSourceFreshnessBatch")?, + &imported(py, NATIVE_RECORDS, "AdapterSourceFreshnessBatch")?, &[("unknown", unknown_sources.into_any())], ) } @@ -345,7 +345,7 @@ pub(crate) fn build_direct_source_freshness_propagation_result<'py>( unknown_sources.set_item(name, models)?; } let error_status: Bound<'py, PyAny> = - imported(py, FRESHNESS_TYPES, "SourceFreshnessAgeStatus")?.getattr("ERROR")?; + imported(py, NATIVE_RECORDS, "SourceFreshnessAgeStatus")?.getattr("ERROR")?; for (identity, status) in source_freshness .getattr("age_statuses")? .downcast::()? @@ -360,11 +360,7 @@ pub(crate) fn build_direct_source_freshness_propagation_result<'py>( errors.set_item(identity, models)?; } call_with( - &imported( - py, - FRESHNESS_MODELS, - "DirectSourceFreshnessPropagationResult", - )?, + &imported(py, NATIVE_RECORDS, "DirectSourceFreshnessPropagationResult")?, &[ ("changed_source_model_names", changed.into_any()), ("unknown_source_model_names", unknown_sources.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs index be7f68cdd4..9ebc788dde 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/state_io.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::freshness::support::{ }; use crate::bindings::_helpers::state_rows::timestamps::date_time_typed; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const MISSING_SCHEMA_MESSAGE: &str = "source freshness table requires a target schema"; /// The qualified state table name the adapter renders for one target schema. @@ -110,7 +111,7 @@ pub(crate) struct StateRead<'py> { fn empty_set<'py>(schema: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = schema.py(); call_with( - &imported(py, FRESHNESS_MODELS, "SourceFreshnessSet")?, + &imported(py, NATIVE_RECORDS, "SourceFreshnessSet")?, &[ ("schema", schema.clone()), ("records", PyDict::new(py).into_any()), @@ -174,7 +175,7 @@ freshness table to regenerate source freshness state." records.set_item(record.getattr("identity")?, record)?; } call_with( - &imported(py, FRESHNESS_MODELS, "SourceFreshnessSet")?, + &imported(py, NATIVE_RECORDS, "SourceFreshnessSet")?, &[("schema", schema.clone()), ("records", records.into_any())], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 610d3e5aeb..2ce560bc3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -10,7 +10,6 @@ use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; pub(crate) const FRESHNESS_MODELS: &str = "sqlbuild.compiler.source_freshness.models"; -pub(crate) const FRESHNESS_TYPES: &str = "sqlbuild.compiler.source_freshness.types"; pub(crate) const FRESHNESS_CONSTANTS: &str = "sqlbuild.compiler.source_freshness.constants"; pub(crate) const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; pub(crate) const CONTRACT_TYPES: &str = "sqlbuild.spec.contracts.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 7c962f1528..d361e71616 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1,6 +1,8 @@ //! The layouts of every native record class. -use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; +use crate::bindings::_helpers::freshness::observation::normalize_observation; +use crate::bindings::_helpers::records::models::{EnumSpec, FieldDefault, FieldSpec, RecordSpec}; +use crate::bindings::_helpers::records::types::RecordFinish; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; /// The class attribute holding a record class's index into `RECORDS`. @@ -11,6 +13,12 @@ pub(crate) const DATACLASS_FIELDS_ATTRIBUTE: &str = "__dataclass_fields__"; /// The marker `dataclasses` gives ordinary (not class or init-only) fields. pub(crate) const DATACLASS_FIELD_KIND: &str = "_FIELD"; +pub(crate) const ENUM_MODULE: &str = "enum"; +pub(crate) const STR_ENUM: &str = "StrEnum"; + +/// Every native string enum group, registered before the records. +pub(crate) const ENUMS: &[&[EnumSpec]] = &[FRESHNESS_ENUMS]; + /// Every native record group, in registration order. pub(crate) const RECORDS: &[&[RecordSpec]] = &[ COMMAND_RECORDS, @@ -18,8 +26,16 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ OUTPUT_RECORDS, COMPILE_RECORDS, DBT_RECORDS, + FRESHNESS_RECORDS, ]; +/// Records whose bound values are normalized before the record is created. +pub(crate) const RECORD_FINISHERS: &[(&str, RecordFinish)] = + &[("SourceFreshnessObservation", normalize_observation)]; +/// `SourceFreshnessObservation` field positions. +pub(crate) const DATA_VERSION_FIELD: usize = 2; +pub(crate) const OBSERVED_AT_FIELD: usize = 4; + const NESTED_PROGRESS_CHILD_ROW: RecordSpec = RecordSpec { name: "NestedProgressChildRow", doc: "One child row rendered below a completed nested progress item.", @@ -1146,3 +1162,107 @@ const DBT_RECORDS: &[RecordSpec] = &[ DBT_PLAN_ENVIRONMENT, DBT_SQLBUILD_PLAN_REQUEST, ]; + +const SOURCE_FRESHNESS_RENDERERS: RecordSpec = RecordSpec { + name: "SourceFreshnessRenderers", + doc: "Adapter SQL renderers used when writing source freshness rows.", + fields: &[ + FieldSpec::required("render_qualified_name"), + FieldSpec::required("render_framework_type"), + FieldSpec::required("render_insert_records_sql"), + FieldSpec::new("render_create_table_sql", FieldDefault::None), + FieldSpec::new("render_create_index_sqls", FieldDefault::None), + ], +}; +const SOURCE_FRESHNESS_OBSERVATION: RecordSpec = RecordSpec { + name: "SourceFreshnessObservation", + doc: "One comparable observation whose datetime values are always aware UTC.", + fields: &[ + FieldSpec::required("source_name"), + FieldSpec::required("strategy"), + FieldSpec::required("data_version"), + FieldSpec::required("value_kind"), + FieldSpec::required("observed_at"), + ], +}; +const SOURCE_FRESHNESS_UNKNOWN: RecordSpec = RecordSpec { + name: "SourceFreshnessUnknown", + doc: "One source whose freshness could not be observed, with the reason.", + fields: &[ + FieldSpec::required("source_name"), + FieldSpec::required("reason"), + FieldSpec::required("message"), + ], +}; +const ADAPTER_SOURCE_FRESHNESS_BATCH: RecordSpec = RecordSpec { + name: "AdapterSourceFreshnessBatch", + doc: "Per-source outcomes of one batched adapter metadata observation.", + fields: &[ + FieldSpec::new("observations", FieldDefault::EmptyDict), + FieldSpec::new("unknown", FieldDefault::EmptyDict), + ], +}; +const SOURCE_FRESHNESS_IDENTITY: RecordSpec = RecordSpec { + name: "SourceFreshnessIdentity", + doc: "Stable identity for one source freshness stream in direct state.", + fields: &[ + FieldSpec::required("source_name"), + FieldSpec::required("target_database"), + FieldSpec::required("target_schema"), + FieldSpec::required("target_name"), + ], +}; +const SOURCE_FRESHNESS_SET: RecordSpec = RecordSpec { + name: "SourceFreshnessSet", + doc: "Latest direct source freshness records for one target schema.", + fields: &[ + FieldSpec::required("schema"), + FieldSpec::required("records"), + ], +}; +const DIRECT_SOURCE_FRESHNESS_PROPAGATION_RESULT: RecordSpec = RecordSpec { + name: "DirectSourceFreshnessPropagationResult", + doc: "Downstream model impact derived from direct source freshness roots.", + fields: &[ + FieldSpec::new("changed_source_model_names", FieldDefault::EmptyDict), + FieldSpec::new("unknown_source_model_names", FieldDefault::EmptyDict), + FieldSpec::new("error_source_model_names", FieldDefault::EmptyDict), + FieldSpec::new("stale_model_names", FieldDefault::EmptyFrozenset), + FieldSpec::new("blocked_model_names", FieldDefault::EmptyFrozenset), + ], +}; + +/// Source freshness observations, identities and planning results. +const FRESHNESS_RECORDS: &[RecordSpec] = &[ + SOURCE_FRESHNESS_RENDERERS, + SOURCE_FRESHNESS_OBSERVATION, + SOURCE_FRESHNESS_UNKNOWN, + ADAPTER_SOURCE_FRESHNESS_BATCH, + SOURCE_FRESHNESS_IDENTITY, + SOURCE_FRESHNESS_SET, + DIRECT_SOURCE_FRESHNESS_PROPAGATION_RESULT, +]; + +const SOURCE_FRESHNESS_AGE_STATUS: EnumSpec = EnumSpec { + name: "SourceFreshnessAgeStatus", + doc: "", + members: &[ + ("PASS", "pass"), + ("WARN", "warn"), + ("ERROR", "error"), + ("UNKNOWN", "unknown"), + ], +}; +const SOURCE_FRESHNESS_UNKNOWN_REASON: EnumSpec = EnumSpec { + name: "SourceFreshnessUnknownReason", + doc: "Why one source's freshness could not be observed.", + members: &[ + ("UNAVAILABLE", "unavailable"), + ("ERROR", "error"), + ("NO_FRESHNESS_CONFIG", "no_freshness_config"), + ], +}; + +/// Source freshness age statuses and unknown reasons. +const FRESHNESS_ENUMS: &[EnumSpec] = + &[SOURCE_FRESHNESS_AGE_STATUS, SOURCE_FRESHNESS_UNKNOWN_REASON]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index e7bc8eb51b..def82d50c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -4,12 +4,13 @@ use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyTypeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{ - PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods, PyType, PyTypeMethods, + PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods, PyType, + PyTypeMethods, }; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::records::constants::{ - FROZEN_INSTANCE_ERROR, RECORD_SPEC_ATTRIBUTE, RECORDS, + FROZEN_INSTANCE_ERROR, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, }; use crate::bindings::_helpers::records::models::{FieldDefault, NativeRecord, RecordSpec}; @@ -80,7 +81,10 @@ pub(crate) fn build_values( }; values.push(value.unbind()); } - Ok(values) + match RECORD_FINISHERS.iter().find(|(record, _)| *record == name) { + Some((_, finish)) => finish(py, values), + None => Ok(values), + } } fn default_value<'py>( @@ -100,6 +104,7 @@ fn default_value<'py>( FieldDefault::Text(value) => Ok(PyString::new(py, value).into_any()), FieldDefault::EmptyTuple => Ok(PyTuple::empty(py).into_any()), FieldDefault::EmptyDict => Ok(PyDict::new(py).into_any()), + FieldDefault::EmptyFrozenset => Ok(PyFrozenSet::empty(py)?.into_any()), FieldDefault::Member(module, class, member) => imported(py, module, class)?.getattr(member), FieldDefault::Call(module, name) => imported(py, module, name)?.call0(), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs index bd3477fe45..21ee303956 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod constants; pub(crate) mod fields; pub(crate) mod models; pub(crate) mod registration; +pub(crate) mod types; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 3842e720f7..beffb44b1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -20,6 +20,7 @@ pub(crate) enum FieldDefault { Text(&'static str), EmptyTuple, EmptyDict, + EmptyFrozenset, /// `getattr(getattr(import(module), class), member)`. Member(&'static str, &'static str, &'static str), /// `getattr(import(module), name)()`, evaluated for every instance. @@ -42,6 +43,14 @@ pub(crate) struct RecordSpec { pub(crate) fields: &'static [FieldSpec], } +/// The members of one native string enum, in declaration order. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EnumSpec { + pub(crate) name: &'static str, + pub(crate) doc: &'static str, + pub(crate) members: &'static [(&'static str, &'static str)], +} + /// Base of every native frozen record, built, compared, shown and pickled as a dataclass. #[derive(Debug)] #[pyclass(module = "sqlbuild._native", subclass, frozen)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 5159e705ce..3942d7e3d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -4,11 +4,13 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ - DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, NATIVE_MODULE, - RECORD_SPEC_ATTRIBUTE, + DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, + NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, }; use crate::bindings::_helpers::records::fields::all_specs; -use crate::bindings::_helpers::records::models::{NativeRecord, RecordAttribute, RecordSpec}; +use crate::bindings::_helpers::records::models::{ + EnumSpec, NativeRecord, RecordAttribute, RecordSpec, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { let py: Python<'_> = module.py(); @@ -16,6 +18,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; let base: Bound<'_, PyAny> = py.get_type::().into_any(); let metaclass: Bound<'_, PyAny> = base.get_type().into_any(); + let str_enum: Bound<'_, PyAny> = py.import(ENUM_MODULE)?.getattr(STR_ENUM)?; + for spec in ENUMS.iter().flat_map(|group| group.iter()) { + module.add(spec.name, string_enum(&str_enum, spec)?)?; + } let dataclasses: Bound<'_, PyModule> = py.import(DATACLASSES_MODULE)?; for (index, spec) in all_specs().enumerate() { let class: Bound<'_, PyAny> = record_class(&metaclass, &base, (index, spec))?; @@ -28,6 +34,20 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +/// `StrEnum(name, members)` living in `sqlbuild._native`. +fn string_enum<'py>(str_enum: &Bound<'py, PyAny>, spec: &EnumSpec) -> PyResult> { + let py: Python<'py> = str_enum.py(); + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("module", NATIVE_MODULE)?; + options.set_item("qualname", spec.name)?; + let members: Vec<(&str, &str)> = spec.members.to_vec(); + let enum_class: Bound<'py, PyAny> = str_enum.call((spec.name, members), Some(&options))?; + if !spec.doc.is_empty() { + enum_class.setattr("__doc__", spec.doc)?; + } + Ok(enum_class) +} + /// The `dataclasses.Field` table that lets `dataclasses.replace` and `fields` read a record. fn dataclass_fields<'py>( dataclasses: &Bound<'py, PyModule>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs new file mode 100644 index 0000000000..4b90f1cc6a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/types.rs @@ -0,0 +1,6 @@ +//! Hooks applied while building native records. + +use pyo3::prelude::{Py, PyAny, PyResult, Python}; + +/// Normalizes the bound field values of one record before it is created. +pub(crate) type RecordFinish = fn(Python<'_>, Vec>) -> PyResult>>; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs index 9b30e3252c..578d6b0c4e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/freshness.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::state_rows::timestamps::{ date_time_typed, in_utc, offset_aware, presumed_utc, }; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; -const TYPES_MODULE: &str = "sqlbuild.compiler.source_freshness.types"; const TIMESTAMP_KIND: &str = "timestamp"; const INTEGER_KIND: &str = "integer"; const STRING_KIND: &str = "string"; @@ -206,7 +206,7 @@ pub(crate) fn age_policy<'py>( if policy.is_none() { return Ok(None); } - let statuses: Bound<'py, PyAny> = imported(py, TYPES_MODULE, "SourceFreshnessAgeStatus")?; + let statuses: Bound<'py, PyAny> = imported(py, NATIVE_RECORDS, "SourceFreshnessAgeStatus")?; if !date_time_typed(data_version)? { return statuses.getattr("UNKNOWN").map(Some); } diff --git a/python-boundary.toml b/python-boundary.toml index 3fd082b138..1edcd598b8 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 10122 +remaining_lines = 10007 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b3efb69bfa..17cc6787ad 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -11,6 +11,7 @@ from enum import StrEnum from pathlib import Path, PurePath from threading import Event from typing import Any, ClassVar, Literal, Protocol, TextIO, TypedDict +from typing import Any, Literal, Protocol, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -271,18 +272,8 @@ from sqlbuild.compiler.scopes.types import ( DeclarationKind, ) from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, - SourceFreshnessIdentity, - SourceFreshnessObservation, SourceFreshnessRecord, - SourceFreshnessRenderers, - SourceFreshnessSet, -) -from sqlbuild.compiler.source_freshness.types import ( - SourceFreshnessAgeStatus, - SourceFreshnessComparableRecord, ) from sqlbuild.compiler.sql_analysis.models import ( SqlBindingResult, @@ -9086,3 +9077,125 @@ class DbtSqlbuildPlanRequest: sqlbuild_args: tuple[str, ...], external_blocked_model_names: tuple[str, ...] = (), ) -> None: ... + +class SourceFreshnessComparableRecord(Protocol): + @property + def data_version_hash(self) -> str: ... + @property + def value_kind(self) -> str: ... + @property + def data_version(self) -> str | None: ... + +class SourceFreshnessAgeStatus(StrEnum): + PASS = "pass" + WARN = "warn" + ERROR = "error" + UNKNOWN = "unknown" + +class SourceFreshnessUnknownReason(StrEnum): + """Why one source's freshness could not be observed.""" + + UNAVAILABLE = "unavailable" + ERROR = "error" + NO_FRESHNESS_CONFIG = "no_freshness_config" + +class SourceFreshnessRenderers: + """Adapter SQL renderers used when writing source freshness rows.""" + + render_qualified_name: Callable[..., str | None] + render_framework_type: Callable[[FrameworkType], str] + render_insert_records_sql: Callable[..., str] + render_create_table_sql: Callable[..., str] | None + render_create_index_sqls: Callable[..., tuple[str, ...]] | None + def __init__( + self, + render_qualified_name: Callable[..., str | None], + render_framework_type: Callable[[FrameworkType], str], + render_insert_records_sql: Callable[..., str], + render_create_table_sql: Callable[..., str] | None = None, + render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, + ) -> None: ... + +class SourceFreshnessObservation: + """One comparable observation whose datetime values are always aware UTC.""" + + source_name: str + strategy: SourceFreshnessStrategy + data_version: object + value_kind: SourceFreshnessValueKind + observed_at: datetime + def __init__( + self, + source_name: str, + strategy: SourceFreshnessStrategy, + data_version: object, + value_kind: SourceFreshnessValueKind, + observed_at: datetime, + ) -> None: ... + +class SourceFreshnessUnknown: + """One source whose freshness could not be observed, with the reason.""" + + source_name: str + reason: SourceFreshnessUnknownReason + message: str + def __init__( + self, + source_name: str, + reason: SourceFreshnessUnknownReason, + message: str, + ) -> None: ... + +class AdapterSourceFreshnessBatch: + """Per-source outcomes of one batched adapter metadata observation.""" + + observations: dict[str, SourceFreshnessObservation] + unknown: dict[str, SourceFreshnessUnknown] + def __init__( + self, + observations: dict[str, SourceFreshnessObservation] = ..., + unknown: dict[str, SourceFreshnessUnknown] = ..., + ) -> None: ... + +class SourceFreshnessIdentity: + """Stable identity for one source freshness stream in direct state.""" + + source_name: str + target_database: str | None + target_schema: str | None + target_name: str | None + def __init__( + self, + source_name: str, + target_database: str | None, + target_schema: str | None, + target_name: str | None, + ) -> None: ... + +class SourceFreshnessSet: + """Latest direct source freshness records for one target schema.""" + + schema: str + records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] + def __init__( + self, + schema: str, + records: dict[SourceFreshnessIdentity, SourceFreshnessRecord], + ) -> None: ... + +class DirectSourceFreshnessPropagationResult: + """Downstream model impact derived from direct source freshness roots.""" + + changed_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] + unknown_source_model_names: dict[str, frozenset[str]] + error_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] + stale_model_names: frozenset[str] + blocked_model_names: frozenset[str] + def __init__( + self, + changed_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = ..., + unknown_source_model_names: dict[str, frozenset[str]] = ..., + error_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = ..., + stale_model_names: frozenset[str] = ..., + blocked_model_names: frozenset[str] = ..., + ) -> None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py index 3741f1a35f..8d34692f81 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/source_freshness.py @@ -5,14 +5,17 @@ from dataclasses import replace from typing import Any -from sqlbuild._native import CostContext, write_source_freshness_records +from sqlbuild._native import ( + CostContext, + SourceFreshnessIdentity, + SourceFreshnessRenderers, + write_source_freshness_records, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, SourceFreshnessRecord, - SourceFreshnessRenderers, ) from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/src/sqlbuild/compiler/source_freshness/models.py b/src/sqlbuild/compiler/source_freshness/models.py index b12173a8a1..7cb94a2d42 100644 --- a/src/sqlbuild/compiler/source_freshness/models.py +++ b/src/sqlbuild/compiler/source_freshness/models.py @@ -2,79 +2,16 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import dataclass, field from datetime import datetime -from sqlbuild._native import require_aware_utc_datetime -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.compiler.source_freshness.types import ( +from sqlbuild._native import ( + DirectSourceFreshnessPropagationResult, SourceFreshnessAgeStatus, - SourceFreshnessUnknownReason, + SourceFreshnessIdentity, + SourceFreshnessUnknown, + require_aware_utc_datetime, ) -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind - - -@dataclass(frozen=True) -class SourceFreshnessRenderers: - """Adapter SQL renderers used when writing source freshness rows.""" - - render_qualified_name: Callable[..., str | None] - render_framework_type: Callable[[FrameworkType], str] - render_insert_records_sql: Callable[..., str] - render_create_table_sql: Callable[..., str] | None = None - render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None - - -@dataclass(frozen=True) -class SourceFreshnessObservation: - """One comparable observation whose datetime values are always aware UTC.""" - - source_name: str - strategy: SourceFreshnessStrategy - data_version: object - value_kind: SourceFreshnessValueKind - observed_at: datetime - - def __post_init__(self) -> None: - object.__setattr__( - self, - "observed_at", - require_aware_utc_datetime(value=self.observed_at, field_name="observed_at"), - ) - if isinstance(self.data_version, datetime): - object.__setattr__( - self, - "data_version", - require_aware_utc_datetime(value=self.data_version, field_name="data_version"), - ) - - -@dataclass(frozen=True) -class SourceFreshnessUnknown: - """One source whose freshness could not be observed, with the reason.""" - - source_name: str - reason: SourceFreshnessUnknownReason - message: str - - -@dataclass(frozen=True) -class AdapterSourceFreshnessBatch: - """Per-source outcomes of one batched adapter metadata observation.""" - - observations: dict[str, SourceFreshnessObservation] = field(default_factory=dict) - unknown: dict[str, SourceFreshnessUnknown] = field(default_factory=dict) - - -@dataclass(frozen=True) -class SourceFreshnessIdentity: - """Stable identity for one source freshness stream in direct state.""" - - source_name: str - target_database: str | None - target_schema: str | None - target_name: str | None @dataclass(frozen=True) @@ -109,29 +46,6 @@ def identity(self) -> SourceFreshnessIdentity: ) -@dataclass(frozen=True) -class SourceFreshnessSet: - """Latest direct source freshness records for one target schema.""" - - schema: str - records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] - - -@dataclass(frozen=True) -class DirectSourceFreshnessPropagationResult: - """Downstream model impact derived from direct source freshness roots.""" - - changed_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = field( - default_factory=dict - ) - unknown_source_model_names: dict[str, frozenset[str]] = field(default_factory=dict) - error_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] = field( - default_factory=dict - ) - stale_model_names: frozenset[str] = frozenset() - blocked_model_names: frozenset[str] = frozenset() - - @dataclass(frozen=True) class DirectSourceFreshnessPlanningResult: """Direct planning-time source freshness observations and comparisons.""" diff --git a/src/sqlbuild/compiler/source_freshness/types.py b/src/sqlbuild/compiler/source_freshness/types.py deleted file mode 100644 index dbb1ae0448..0000000000 --- a/src/sqlbuild/compiler/source_freshness/types.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Source freshness type declarations.""" - -from __future__ import annotations - -from enum import StrEnum -from typing import Protocol - - -class SourceFreshnessComparableRecord(Protocol): - @property - def data_version_hash(self) -> str: ... - - @property - def value_kind(self) -> str: ... - - @property - def data_version(self) -> str | None: ... - - -class SourceFreshnessAgeStatus(StrEnum): - PASS = "pass" - WARN = "warn" - ERROR = "error" - UNKNOWN = "unknown" - - -class SourceFreshnessUnknownReason(StrEnum): - """Why one source's freshness could not be observed.""" - - UNAVAILABLE = "unavailable" - ERROR = "error" - NO_FRESHNESS_CONFIG = "no_freshness_config" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py index b30fcfe4c8..f785980e47 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -9,13 +9,14 @@ from textwrap import dedent from types import MappingProxyType -from sqlbuild._native import write_fingerprint, write_source_freshness_records -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord, +from sqlbuild._native import ( SourceFreshnessRenderers, + write_fingerprint, + write_source_freshness_records, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, prepare_inline_project, diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py index dc7e13081d..488db3daf8 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/_test_types.py @@ -1,6 +1,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord +from sqlbuild._native import SourceFreshnessIdentity +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py index 5e7d43a15e..73e5d059a0 100644 --- a/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py +++ b/tests/integration/src/sqlbuild/compiler/source_freshness/main/test_read_write.py @@ -6,16 +6,17 @@ import pytest -from sqlbuild._native import read_latest_source_freshness, write_source_freshness_records -from sqlbuild.adapter.contract.types import FrameworkType -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.models import ( +from sqlbuild._native import ( SourceFreshnessIdentity, - SourceFreshnessRecord, SourceFreshnessRenderers, SourceFreshnessSet, + read_latest_source_freshness, + write_source_freshness_records, ) +from sqlbuild.adapter.contract.types import FrameworkType +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.integration.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessLatestResolutionTestCase, SourceFreshnessPruneHistoryTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py index 0a6a73f8e7..c69eeaf2d2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/freshness/_test_types.py @@ -3,7 +3,8 @@ from dataclasses import dataclass, field from datetime import UTC, datetime -from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord +from sqlbuild._native import SourceFreshnessIdentity +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index 5054627b09..bf07ffd0a2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -3,7 +3,11 @@ from datetime import UTC, datetime from pathlib import Path -from sqlbuild._native import CostContext, build_source_freshness_create_table_sql +from sqlbuild._native import ( + CostContext, + SourceFreshnessIdentity, + build_source_freshness_create_table_sql, +) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -13,10 +17,7 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ChainStep, ModelPlanEntry, SqlTestPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessIdentity, - SourceFreshnessRecord, -) +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py index 7360cdc360..ab85e3c358 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/test_source_freshness.py @@ -6,16 +6,13 @@ import pytest -from sqlbuild._native import CostContext +from sqlbuild._native import CostContext, DirectSourceFreshnessPropagationResult from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( append_eligible_direct_source_freshness_records, ) from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, -) +from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.build.types import BuildStatus diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index df1c67d96b..deab82b272 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -12,6 +12,7 @@ PlannerChangeResults, PlannerScopeResolution, PlanWarning, + SourceFreshnessIdentity, WarehouseFingerprints, WarehouseSnapshot, build_stale_out_of_selection_warnings, @@ -31,6 +32,7 @@ PlannerScope, PlannerSelection, ) +from sqlbuild.compiler.planner.types import ChangeKind from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessIdentity, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py index 8f08e90378..10db904107 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/_test_types.py @@ -2,8 +2,8 @@ from datetime import UTC, datetime from typing import Any +from sqlbuild._native import SourceFreshnessUnknownReason from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.compiler.source_freshness.types import SourceFreshnessUnknownReason from sqlbuild.spec.contracts.types import SourceFreshnessValueKind diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py index 11edc5e466..60a11450b1 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + SourceFreshnessObservation, SourceFreshnessObservationError, normalize_source_freshness_data_version, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main.configured_observation._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index f0ec34eb25..37cd5b2a76 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -6,6 +6,8 @@ from typing import Any from sqlbuild._native import ( + SourceFreshnessIdentity, + SourceFreshnessRenderers, build_source_freshness_read_latest_sql, source_freshness_data_version_hash, write_source_freshness_records, @@ -21,11 +23,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessIdentity, - SourceFreshnessRecord, - SourceFreshnessRenderers, -) +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py index 8c7a31b198..dfb9b335ad 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_adapter_observation.py @@ -6,15 +6,13 @@ import pytest from sqlbuild._native import ( + AdapterSourceFreshnessBatch, + SourceFreshnessUnknownReason, build_direct_source_freshness_planning_result, observe_adapter_sources_freshness, ) from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.compiler.source_freshness.models import ( - AdapterSourceFreshnessBatch, - DirectSourceFreshnessPlanningResult, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessUnknownReason +from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py index 639d906e1b..9d80b24ac6 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_age_policy.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import evaluate_source_freshness_age_policy -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus +from sqlbuild._native import SourceFreshnessAgeStatus, evaluate_source_freshness_age_policy from sqlbuild.spec.contracts.models import SourceFreshnessAgePolicy from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessAgeEvaluationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py index c82e8a782f..2373169f04 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_models.py @@ -4,10 +4,8 @@ import pytest -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessObservation, - SourceFreshnessRecord, -) +from sqlbuild._native import SourceFreshnessObservation +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( SourceFreshnessDatetimeNormalizationTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py index b81dbdfdb8..728bc319c4 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_observation.py @@ -6,11 +6,11 @@ import pytest from sqlbuild._native import ( + SourceFreshnessObservation, observe_configured_source_freshness, ) from sqlbuild.adapter.contract.models import QueryResult, TableFreshnessMetadata from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py index 06ec5f6358..cc450556d5 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_planning.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + SourceFreshnessRenderers, build_direct_source_freshness_planning_result, source_freshness_data_version_hash, write_source_freshness_records, @@ -17,7 +18,6 @@ from sqlbuild.compiler.source_freshness.models import ( DirectSourceFreshnessPlanningResult, SourceFreshnessRecord, - SourceFreshnessRenderers, ) from sqlbuild.spec.contracts.models import ( SourceEntry, diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py index 547a5375f9..9a8cf4f190 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_propagation.py @@ -3,14 +3,12 @@ import pytest from sqlbuild._native import ( + DirectSourceFreshnessPropagationResult, + SourceFreshnessAgeStatus, build_direct_source_freshness_propagation_result, ) from sqlbuild.compiler.planner.models import PlannerScope -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - DirectSourceFreshnessPropagationResult, -) -from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus +from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( DirectSourceFreshnessPropagationTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py index f889c13e35..b619eb4fed 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/test_read.py @@ -5,17 +5,15 @@ import pytest from sqlbuild._native import ( + SourceFreshnessIdentity, SourceFreshnessInputError, + SourceFreshnessRenderers, + SourceFreshnessSet, read_latest_source_freshness, write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessIdentity, - SourceFreshnessRecord, - SourceFreshnessRenderers, - SourceFreshnessSet, -) +from sqlbuild.compiler.source_freshness.models import SourceFreshnessRecord from tests.unit.src.sqlbuild.compiler.source_freshness.main._test_types import ( ReadLatestSourceFreshnessErrorTestCase, ReadLatestSourceFreshnessRendererTestCase, From 32f0c6b49b40cd7ac63ffc15161989b8104b9c15 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:37:59 +0100 Subject: [PATCH 13/23] refactor: build run-cost observations, records and ledger entries as native records --- .../bindings/_helpers/cost_command/refresh.rs | 3 +- .../_helpers/cost_report/degradation.rs | 6 +- .../bindings/_helpers/cost_report/records.rs | 5 +- .../_helpers/cost_telemetry/allocation.rs | 3 +- .../_helpers/cost_telemetry/ledger.rs | 8 +- .../_helpers/cost_telemetry/snowflake.rs | 4 +- .../_helpers/cost_telemetry/values.rs | 12 + .../bindings/_helpers/records/constants.rs | 77 +++++ src/sqlbuild/_native.pyi | 263 +++++++++++++----- src/sqlbuild/cost/models.py | 61 +--- src/sqlbuild/cost/types.py | 29 -- .../commands/_helpers/cost/test_collection.py | 4 +- .../cli/commands/_helpers/cost/test_output.py | 3 +- .../commands/_helpers/cost/test_refresh.py | 3 +- .../commands/main/inspection/cost/helpers.py | 4 +- tests/unit/src/sqlbuild/cost/helpers.py | 3 +- .../unit/src/sqlbuild/cost/test_allocation.py | 4 +- tests/unit/src/sqlbuild/cost/test_store.py | 2 +- .../pipeline/_helpers/test_auditing.py | 3 +- 19 files changed, 307 insertions(+), 190 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index cfee0e63c3..0f3bdfdcdd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -11,6 +11,7 @@ use sqlbuild_core::cost_command::models::CostCoverage; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cost_telemetry::values::cost_aware; const NATIVE_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; @@ -121,7 +122,7 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { ], )?; let adapter: Bound<'_, PyAny> = resolve_cli_adapter(&adapter_name, &project_dir)?; - if !adapter.is_instance(&imported(py, COST_TYPES, "CostAwareAdapter")?)? { + if !cost_aware(&adapter)? { return Ok(()); } let none_capability: Bound<'_, PyAny> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index 274a060f52..b3d7858934 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -14,7 +14,9 @@ use sqlbuild_core::cost_report::models::BreakdownOptions; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; +use crate::bindings::_helpers::cost_telemetry::values::cost_aware; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const LOGGER_NAME: &str = "sqlbuild.cli.commands._helpers.cost.collection"; const COLLECTION_LOGGER_NAME: &str = "sqlbuild.cost"; const COST_TYPES: &str = "sqlbuild.cost.types"; @@ -160,7 +162,7 @@ fn cost_record<'py>( "had_executable_work", finalization.getattr("had_executable_work")?, )); - call_with(&imported(py, COST_MODELS, "CostRunRecord")?, &fields) + call_with(&imported(py, NATIVE_RECORDS, "CostRunRecord")?, &fields) } fn summary<'py>( @@ -268,7 +270,7 @@ pub(crate) fn finalize_build_cost<'py>( ) -> PyResult> { let py: Python<'py> = finalization.py(); let adapter: Bound<'py, PyAny> = finalization.getattr("adapter")?; - if !adapter.is_instance(&imported(py, COST_TYPES, "CostAwareAdapter")?)? { + if !cost_aware(&adapter)? { return Ok(py.None().into_bound(py)); } let config: Bound<'py, PyAny> = finalization.getattr("config")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs index 93da8e9177..54d46b6fa2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs @@ -7,6 +7,7 @@ use sqlbuild_core::cost_report::models::{CostResource, CostRun, CostSummary}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const COST_MODELS: &str = "sqlbuild.cost.models"; const COST_TYPES: &str = "sqlbuild.cost.types"; const COST_EXCEPTIONS: &str = "sqlbuild._native"; @@ -154,7 +155,7 @@ fn resource_object<'py>( builders.decimal(&resource.estimated_compute_credits)?, )?; fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?; - imported(py, COST_MODELS, "ResourceCost")?.call((), Some(&fields)) + imported(py, NATIVE_RECORDS, "ResourceCost")?.call((), Some(&fields)) } fn summary_object<'py>( @@ -215,7 +216,7 @@ pub(crate) fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult( ) -> PyResult> { let decimal = |value: &PyDecimal| python_decimal(py, value); call_with( - &imported(py, COST_MODELS, "ResourceCost")?, + &imported(py, NATIVE_RECORDS, "ResourceCost")?, &[ ( "resource_type", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs index 25119a9b80..d013a6d08a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs @@ -19,11 +19,11 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; use crate::bindings::_helpers::cost_telemetry::health::mark_ledger_failure; -use crate::bindings::_helpers::cost_telemetry::values::{ - COST_EXCEPTIONS, COST_MODELS, process_object, -}; +use crate::bindings::_helpers::cost_telemetry::values::{COST_EXCEPTIONS, process_object}; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +const NATIVE_RECORDS: &str = "sqlbuild._native"; + static LEDGER_LOCK: PyOnceLock> = PyOnceLock::new(); /// The process-wide `threading.Lock` that orders ledger appends and snapshot reads. @@ -195,7 +195,7 @@ fn ledger_entry<'py>(line: &Bound<'py, PyAny>) -> PyResult> { ("completed_at", parsed("completed_at")?), ]; call_with( - &imported(py, COST_MODELS, "StatementLedgerEntry")?, + &imported(py, NATIVE_RECORDS, "StatementLedgerEntry")?, &entry_fields, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index a9dd18e9be..ebb850a906 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -21,9 +21,9 @@ use crate::bindings::_helpers::cost_telemetry::allocation::{ }; use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; -use crate::bindings::_helpers::cost_telemetry::values::COST_MODELS; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const DEADLINE_SECONDS: f64 = 10.0; const QUERY_TIMEOUT_SECONDS: &str = "5"; const RUNNING_STATUS: &str = "RUNNING"; @@ -258,7 +258,7 @@ fn row_observation<'py>( } let tag: QueryTag = tag.unwrap_or_default(); let observation: Bound<'py, PyAny> = call_with( - &imported(py, COST_MODELS, "QueryCostObservation")?, + &imported(py, NATIVE_RECORDS, "QueryCostObservation")?, &[ ("query_id", query_id.str()?.into_any()), ("warehouse_name", warehouse_name.str()?.into_any()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs index 30d073aa86..f45635d43d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs @@ -15,6 +15,18 @@ pub(crate) const COST_MODELS: &str = "sqlbuild.cost.models"; pub(crate) const COST_TYPES: &str = "sqlbuild.cost.types"; pub(crate) const COST_EXCEPTIONS: &str = "sqlbuild._native"; const DECIMAL_MODULE: &str = "decimal"; +/// The methods an adapter with warehouse cost telemetry implements. +const COST_AWARE_METHODS: [&str; 2] = ["cost_capability", "collect_run_cost"]; + +/// Whether `adapter` implements the cost telemetry methods. +pub(crate) fn cost_aware(adapter: &Bound<'_, PyAny>) -> PyResult { + for name in COST_AWARE_METHODS { + if !adapter.hasattr(name)? || adapter.getattr(name)?.is_none() { + return Ok(false); + } + } + Ok(true) +} /// A finite non-negative `Decimal` as its exact digits and exponent. pub(crate) fn native_decimal(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d361e71616..d5d2990a62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -27,6 +27,7 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ COMPILE_RECORDS, DBT_RECORDS, FRESHNESS_RECORDS, + COST_RECORDS, ]; /// Records whose bound values are normalized before the record is created. @@ -1266,3 +1267,79 @@ const SOURCE_FRESHNESS_UNKNOWN_REASON: EnumSpec = EnumSpec { /// Source freshness age statuses and unknown reasons. const FRESHNESS_ENUMS: &[EnumSpec] = &[SOURCE_FRESHNESS_AGE_STATUS, SOURCE_FRESHNESS_UNKNOWN_REASON]; + +const QUERY_COST_OBSERVATION: RecordSpec = RecordSpec { + name: "QueryCostObservation", + doc: "", + fields: &[ + FieldSpec::required("query_id"), + FieldSpec::required("warehouse_name"), + FieldSpec::required("warehouse_size"), + FieldSpec::required("warehouse_type"), + FieldSpec::required("cluster_number"), + FieldSpec::required("started_at"), + FieldSpec::required("completed_at"), + FieldSpec::required("execution_ms"), + FieldSpec::required("bytes_scanned"), + FieldSpec::required("execution_status"), + FieldSpec::new("run_id", FieldDefault::None), + FieldSpec::new("resource_type", FieldDefault::None), + FieldSpec::new("resource_name", FieldDefault::None), + ], +}; +const RESOURCE_COST: RecordSpec = RecordSpec { + name: "ResourceCost", + doc: "", + fields: &[ + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("warehouse_name"), + FieldSpec::required("warehouse_size"), + FieldSpec::required("warehouse_type"), + FieldSpec::required("cluster_number"), + FieldSpec::required("query_count"), + FieldSpec::required("attributed_seconds"), + FieldSpec::required("bytes_scanned"), + FieldSpec::required("estimated_compute_credits"), + FieldSpec::required("estimated_usd"), + ], +}; +const COST_RUN_RECORD: RecordSpec = RecordSpec { + name: "CostRunRecord", + doc: "", + fields: &[ + FieldSpec::required("run_id"), + FieldSpec::required("adapter_name"), + FieldSpec::required("target_name"), + FieldSpec::required("build_status"), + FieldSpec::required("started_at"), + FieldSpec::required("completed_at"), + FieldSpec::required("cost"), + FieldSpec::new("version", FieldDefault::Int(1)), + FieldSpec::new("had_executable_work", FieldDefault::None), + ], +}; +const STATEMENT_LEDGER_ENTRY: RecordSpec = RecordSpec { + name: "StatementLedgerEntry", + doc: "", + fields: &[ + FieldSpec::required("statement_id"), + FieldSpec::required("run_id"), + FieldSpec::required("resource_type"), + FieldSpec::required("resource_name"), + FieldSpec::required("phase"), + FieldSpec::required("attempt"), + FieldSpec::required("query_id"), + FieldSpec::required("status"), + FieldSpec::required("started_at"), + FieldSpec::required("completed_at"), + ], +}; + +/// Run-cost observations, allocations, run records and statement ledger entries. +const COST_RECORDS: &[RecordSpec] = &[ + QUERY_COST_OBSERVATION, + RESOURCE_COST, + COST_RUN_RECORD, + STATEMENT_LEDGER_ENTRY, +]; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 17cc6787ad..27c509824b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5,13 +5,13 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set from concurrent.futures import Future from contextlib import AbstractContextManager from contextvars import Token +from dataclasses import Field from datetime import date, datetime from decimal import Decimal from enum import StrEnum from pathlib import Path, PurePath from threading import Event from typing import Any, ClassVar, Literal, Protocol, TextIO, TypedDict -from typing import Any, Literal, Protocol, TextIO, TypedDict from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter @@ -287,11 +287,8 @@ from sqlbuild.compiler.sql_test_glue.models import ( ) from sqlbuild.cost.models import ( CostResourceContext, - CostRunRecord, - QueryCostObservation, RunCostSummary, StatementExecutionTelemetry, - StatementLedgerEntry, ) from sqlbuild.cost.types import CostStatus from sqlbuild.cursor_algebra.models import ( @@ -7374,7 +7371,7 @@ def render_snowflake_query_history_sql( def snowflake_statement_is_metadata_only(*, query_type: object, query_text: object) -> bool: ... def parse_sqlbuild_query_tag(value: object) -> dict[str, str | None] | None: ... -class NestedProgressChildRow: +class NestedProgressChildRow(NativeRecord): """One child row rendered below a completed nested progress item.""" label: str @@ -7389,7 +7386,7 @@ class NestedProgressChildRow: detail: str = "", ) -> None: ... -class AuditCommandRequest: +class AuditCommandRequest(NativeRecord): """CLI inputs for one audit command invocation.""" project_dir: Path | None @@ -7418,7 +7415,7 @@ class AuditCommandRequest: json_output_path: Path | None = None, ) -> None: ... -class AuditInvocation: +class AuditInvocation(NativeRecord): """Resolved project, adapter, and reporter context for the audit command.""" effective_project_dir: Path @@ -7443,7 +7440,7 @@ class AuditInvocation: planning_progress: PlanningProgressReporter, ) -> None: ... -class BuildCommandRequest: +class BuildCommandRequest(NativeRecord): """CLI inputs for one build command invocation.""" project_dir: Path | None @@ -7518,7 +7515,7 @@ class BuildCommandRequest: selector_files: tuple[SelectorFileSummary, ...] = (), ) -> None: ... -class BuildInvocation: +class BuildInvocation(NativeRecord): """Resolved project, adapter, and reporter context for the build command.""" effective_project_dir: Path @@ -7551,7 +7548,7 @@ class BuildInvocation: execution_limits: ExecutionLimitsConfig = ..., ) -> None: ... -class CheckCommandRequest: +class CheckCommandRequest(NativeRecord): """CLI inputs for one check command invocation.""" project_dir: Path | None @@ -7576,7 +7573,7 @@ class CheckCommandRequest: json_output_path: Path | None = None, ) -> None: ... -class CheckInvocation: +class CheckInvocation(NativeRecord): """Resolved project, adapter, and reporter context for the check command.""" effective_project_dir: Path @@ -7601,7 +7598,7 @@ class CheckInvocation: planning_progress: PlanningProgressReporter, ) -> None: ... -class CloneCommandRequest: +class CloneCommandRequest(NativeRecord): """CLI inputs for one clone command invocation.""" project_dir: Path | None @@ -7632,7 +7629,7 @@ class CloneCommandRequest: event_output_path: Path | None = None, ) -> None: ... -class DbtSqlbuildWorkContext: +class DbtSqlbuildWorkContext(NativeRecord): """Shared execution context for one dbt interop SQLBuild work run.""" plan_output: PlanOutput @@ -7655,7 +7652,7 @@ class DbtSqlbuildWorkContext: execution_limits: ExecutionLimitsConfig, ) -> None: ... -class PlanSafetyGates: +class PlanSafetyGates(NativeRecord): """Pre-confirmed plan safety gates and the guidance shown when one needs a terminal.""" allow_missing_migration_origin: bool @@ -7680,7 +7677,7 @@ class PlanSafetyGates: execution_limit_note: str, ) -> None: ... -class DbtInitCommandRequest: +class DbtInitCommandRequest(NativeRecord): """CLI inputs for one dbt init command invocation.""" cwd: Path @@ -7705,7 +7702,7 @@ class DbtInitCommandRequest: skip_dbt_debug: bool, ) -> None: ... -class DiffCommandRequest: +class DiffCommandRequest(NativeRecord): """CLI inputs for one diff command invocation.""" project_dir: Path | None @@ -7782,7 +7779,7 @@ class DiffCommandRequest: full_example_values: bool = False, ) -> None: ... -class QueryDiffPreparation: +class QueryDiffPreparation(NativeRecord): """Resolved raw-query diff adapter, connection, SQL, and limits.""" adapter: BaseAdapter @@ -7815,7 +7812,7 @@ class QueryDiffPreparation: effective_max_row_only_examples: int, ) -> None: ... -class FreshnessCommandRequest: +class FreshnessCommandRequest(NativeRecord): """CLI inputs for one `sqb freshness` invocation.""" project_dir: Path | None @@ -7846,7 +7843,7 @@ class FreshnessCommandRequest: fail_on_stale: bool = False, ) -> None: ... -class JanitorCommandRequest: +class JanitorCommandRequest(NativeRecord): """CLI inputs for one janitor command invocation.""" project_dir: Path | None @@ -7869,7 +7866,7 @@ class JanitorCommandRequest: as_target: str | None = None, ) -> None: ... -class JanitorInvocation: +class JanitorInvocation(NativeRecord): """Resolved project discovery and output context for janitor.""" effective_project_dir: Path @@ -7888,7 +7885,7 @@ class JanitorInvocation: as_target: str | None = None, ) -> None: ... -class JanitorSettings: +class JanitorSettings(NativeRecord): """Validated effective janitor settings.""" retention_days: int @@ -7903,7 +7900,7 @@ class JanitorSettings: drop_old_name_views: tuple[str, ...] = (), ) -> None: ... -class LoadCommandRequest: +class LoadCommandRequest(NativeRecord): """CLI inputs for one load command invocation.""" project_dir: Path | None @@ -7932,7 +7929,7 @@ class LoadCommandRequest: json_output_path: Path | None = None, ) -> None: ... -class PlanCommandRequest: +class PlanCommandRequest(NativeRecord): """CLI inputs for one plan command invocation.""" project_dir: Path | None @@ -7977,7 +7974,7 @@ class PlanCommandRequest: selection_diagnostics: bool = False, ) -> None: ... -class PlaygroundCommandRequest: +class PlaygroundCommandRequest(NativeRecord): """CLI inputs for one playground command invocation.""" project_dir: Path | None @@ -7990,7 +7987,7 @@ class PlaygroundCommandRequest: template: str = ..., ) -> None: ... -class AdapterConnectionContext: +class AdapterConnectionContext(NativeRecord): """Resolved adapter and connection configuration for one CLI command.""" adapter_name: str @@ -8003,7 +8000,7 @@ class AdapterConnectionContext: connection_config: dict[str, object], ) -> None: ... -class ScenarioSnapshotLimitInputs: +class ScenarioSnapshotLimitInputs(NativeRecord): """CLI snapshot capture-limit overrides for one scenario run.""" max_snapshot_rows: int | None @@ -8020,7 +8017,7 @@ class ScenarioSnapshotLimitInputs: force: bool = False, ) -> None: ... -class ScenarioTestCommandRequest: +class ScenarioTestCommandRequest(NativeRecord): """CLI inputs for one `sqb scenario test` invocation.""" project_dir: Path | None @@ -8057,7 +8054,7 @@ class ScenarioTestCommandRequest: json_output_path: Path | None = None, ) -> None: ... -class ScenarioCaptureCommandRequest: +class ScenarioCaptureCommandRequest(NativeRecord): """CLI inputs for one `sqb scenario capture` invocation.""" project_dir: Path | None @@ -8080,7 +8077,7 @@ class ScenarioCaptureCommandRequest: limit_inputs: ScenarioSnapshotLimitInputs = ..., ) -> None: ... -class SeedCommandRequest: +class SeedCommandRequest(NativeRecord): """CLI inputs for one seed command invocation.""" project_dir: Path | None @@ -8105,7 +8102,7 @@ class SeedCommandRequest: json_output_path: Path | None = None, ) -> None: ... -class SeedInvocation: +class SeedInvocation(NativeRecord): """Resolved project, adapter, connection, and output context for seed.""" effective_project_dir: Path @@ -8126,7 +8123,7 @@ class SeedInvocation: progress_stream: TextIO, ) -> None: ... -class TestCommandRequest: +class TestCommandRequest(NativeRecord): """CLI inputs for one test command invocation.""" project_dir: Path | None @@ -8157,7 +8154,7 @@ class TestCommandRequest: inspect: bool = False, ) -> None: ... -class RefactorCommandRequest: +class RefactorCommandRequest(NativeRecord): """CLI inputs for one `sqb rename` or `sqb mv` invocation.""" command: str @@ -8182,7 +8179,7 @@ class RefactorCommandRequest: no_color: bool = False, ) -> None: ... -class ScopeCommandRequest: +class ScopeCommandRequest(NativeRecord): """CLI inputs for one offline declaration-scope inspection.""" project_dir: Path | None @@ -8233,7 +8230,7 @@ class ScopeCommandRequest: no_color: bool = False, ) -> None: ... -class TestInvocation: +class TestInvocation(NativeRecord): """Resolved project, adapter, and reporter context for the test command.""" effective_project_dir: Path @@ -8258,7 +8255,7 @@ class TestInvocation: planning_progress: PlanningProgressReporter, ) -> None: ... -class RulesCommandRequest: +class RulesCommandRequest(NativeRecord): """Inputs for focused rule execution, inspection, or skill generation.""" project_dir: Path | None @@ -8279,7 +8276,7 @@ class RulesCommandRequest: exclude: tuple[str, ...] = (), ) -> None: ... -class CostCommandRequest: +class CostCommandRequest(NativeRecord): """Inputs for local run-cost history and drill-down.""" project_dir: Path | None @@ -8308,7 +8305,7 @@ class CostCommandRequest: json_output_path: Path | None, ) -> None: ... -class BuildCostFinalization: +class BuildCostFinalization(NativeRecord): """Inputs for nonfatal automatic build-cost finalization.""" project_dir: Path @@ -8351,7 +8348,7 @@ class BuildCostFinalization: had_executable_work: bool | None = True, ) -> None: ... -class ContractCommandRequest: +class ContractCommandRequest(NativeRecord): """CLI inputs for physical contract comparison and repository adoption.""" action: str @@ -8378,7 +8375,7 @@ class ContractCommandRequest: cli_vars: dict[str, object] | None = None, ) -> None: ... -class LineageCommandRequest: +class LineageCommandRequest(NativeRecord): """CLI inputs for dependency and column lineage inspection.""" project_dir: Path | None @@ -8407,7 +8404,7 @@ class LineageCommandRequest: cli_vars: dict[str, object] | None = None, ) -> None: ... -class CompileProfileFlags: +class CompileProfileFlags(NativeRecord): """Profiling toggles that skip compile phases for benchmarking.""" skip_discovery_sql_analysis: bool @@ -8422,7 +8419,7 @@ class CompileProfileFlags: skip_write: bool = False, ) -> None: ... -class CompileCommandRequest: +class CompileCommandRequest(NativeRecord): """CLI inputs for one `sqb compile` invocation.""" project_dir: Path | None @@ -8459,7 +8456,7 @@ class CompileCommandRequest: debug: bool = False, ) -> None: ... -class SelectorFileSummary: +class SelectorFileSummary(NativeRecord): """Non-expanded provenance for one selector file.""" path: Path @@ -8470,7 +8467,7 @@ class SelectorFileSummary: selector_count: int, ) -> None: ... -class ParsedCliInvocation: +class ParsedCliInvocation(NativeRecord): """Outcome of parsing CLI arguments: either a namespace or an exit code.""" args: CliNamespace | None @@ -8481,7 +8478,7 @@ class ParsedCliInvocation: exit_code: int | None, ) -> None: ... -class CliEntrypointHandlers: +class CliEntrypointHandlers(NativeRecord): """Injected command handlers for the CLI entrypoint.""" run_compile: Callable[[CompileCommandRequest], int] @@ -8552,7 +8549,7 @@ class CliEntrypointHandlers: run_refactor: Callable[..., int] | None = None, ) -> None: ... -class CommandWarehouseScope: +class CommandWarehouseScope(NativeRecord): """The warehouse group and CLI override active for one command invocation.""" group: WarehouseGroup | None @@ -8565,7 +8562,7 @@ class CommandWarehouseScope: cli_vars: dict[str, object] = ..., ) -> None: ... -class ResolvedWarehouse: +class ResolvedWarehouse(NativeRecord): """The warehouse one command connects with and where that value came from.""" warehouse: str | None @@ -8576,7 +8573,7 @@ class ResolvedWarehouse: source: str, ) -> None: ... -class WrittenTarget: +class WrittenTarget(NativeRecord): """Result of writing compiled output to target/.""" model_count: int @@ -8597,7 +8594,7 @@ class WrittenTarget: diagnostics: tuple[CompilerDiagnostic, ...] = ..., ) -> None: ... -class SkillMaintenanceResult: +class SkillMaintenanceResult(NativeRecord): """Non-blocking generated-skill maintenance result.""" message: str @@ -8606,7 +8603,7 @@ class SkillMaintenanceResult: message: str = "", ) -> None: ... -class SkillUpdateResult: +class SkillUpdateResult(NativeRecord): """Result of installing or updating skill files.""" written_paths: tuple[Path, ...] @@ -8615,7 +8612,7 @@ class SkillUpdateResult: written_paths: tuple[Path, ...], ) -> None: ... -class CompileAnalysis: +class CompileAnalysis(NativeRecord): """Compiled project analysis shared by compile output phases.""" discovered_inputs: DiscoveredProjectInputs @@ -8658,7 +8655,7 @@ class CompileAnalysis: skipped_type_proof_rules: tuple[str, ...] = (), ) -> None: ... -class DbtManifestModel: +class DbtManifestModel(NativeRecord): """One dbt model node needed for SQLBuild dbt_ref resolution.""" unique_id: str @@ -8689,7 +8686,7 @@ class DbtManifestModel: payload: dict[str, object] = ..., ) -> None: ... -class DbtManifestSource: +class DbtManifestSource(NativeRecord): """One dbt source node needed for graph and lineage resolution.""" unique_id: str @@ -8714,7 +8711,7 @@ class DbtManifestSource: payload: dict[str, object] = ..., ) -> None: ... -class DbtManifestSeed: +class DbtManifestSeed(NativeRecord): """One dbt seed node, a leaf data relation used for column lineage.""" unique_id: str @@ -8737,7 +8734,7 @@ class DbtManifestSeed: payload: dict[str, object] = ..., ) -> None: ... -class DbtManifestIndex: +class DbtManifestIndex(NativeRecord): """Lookup indexes for dbt model and source nodes in a manifest.""" models_by_unique_id: dict[str, DbtManifestModel] @@ -8756,7 +8753,7 @@ class DbtManifestIndex: validation_depends_on_nodes_by_unique_id: dict[str, tuple[str, ...]] = ..., ) -> None: ... -class DbtCliConfigOverrides: +class DbtCliConfigOverrides(NativeRecord): """dbt config values supplied by CLI flags.""" project_dir: str | None @@ -8771,7 +8768,7 @@ class DbtCliConfigOverrides: target_path: str | None = None, ) -> None: ... -class ResolvedDbtConfig: +class ResolvedDbtConfig(NativeRecord): """Resolved dbt config after applying CLI overrides.""" project_dir: Path | None @@ -8786,7 +8783,7 @@ class ResolvedDbtConfig: target_path: Path | None, ) -> None: ... -class DbtNodeMessage: +class DbtNodeMessage(NativeRecord): """One dbt log message attached to a dbt node.""" level: str @@ -8797,7 +8794,7 @@ class DbtNodeMessage: message: str, ) -> None: ... -class DbtCommandExecutionResult: +class DbtCommandExecutionResult(NativeRecord): """dbt command execution output from streamed JSON events.""" returncode: int @@ -8808,7 +8805,7 @@ class DbtCommandExecutionResult: node_results: tuple[DbtNodeExecutionResult, ...] = ..., ) -> None: ... -class DbtLsNode: +class DbtLsNode(NativeRecord): """One node returned by `dbt ls --output json`.""" unique_id: str @@ -8827,7 +8824,7 @@ class DbtLsNode: original_file_path: str | None = None, ) -> None: ... -class DbtInteropParsedArgs: +class DbtInteropParsedArgs(NativeRecord): """Declared `sqb dbt` flags parsed by argparse, before routing to buckets.""" select: tuple[str, ...] @@ -8880,7 +8877,7 @@ class DbtInteropParsedArgs: dbt_passthrough: tuple[str, ...] = ..., ) -> None: ... -class DbtInteropRoutedArgs: +class DbtInteropRoutedArgs(NativeRecord): """Arguments split for a future `sqb dbt` command execution.""" command: DbtInteropCommand @@ -8897,7 +8894,7 @@ class DbtInteropRoutedArgs: sqlbuild_args: tuple[str, ...] = ..., ) -> None: ... -class DbtInteropSelectionResult: +class DbtInteropSelectionResult(NativeRecord): """SQLBuild-side selection produced from `sqb dbt` selectors.""" sqlbuild_model_names: tuple[str, ...] @@ -8914,7 +8911,7 @@ class DbtInteropSelectionResult: path_translations: tuple[tuple[str, str], ...] = ..., ) -> None: ... -class DbtInteropPlan: +class DbtInteropPlan(NativeRecord): """Plan output for one future `sqb dbt` command.""" command: DbtInteropCommand @@ -8945,7 +8942,7 @@ class DbtInteropPlan: warnings: tuple[str, ...] = ..., ) -> None: ... -class DbtInteropExecutionRequest: +class DbtInteropExecutionRequest(NativeRecord): """Caller inputs for one dbt interop execution command.""" command: DbtInteropCommand @@ -8980,7 +8977,7 @@ class DbtInteropExecutionRequest: json_output: bool = False, ) -> None: ... -class DbtInteropInvocation: +class DbtInteropInvocation(NativeRecord): """Resolved execution environment for one dbt interop command.""" dbt_executable: str @@ -9005,7 +9002,7 @@ class DbtInteropInvocation: runner: Any, ) -> None: ... -class DbtInteropCompiledProject: +class DbtInteropCompiledProject(NativeRecord): """Adapter and compiled SQLBuild project for dbt interop planning.""" adapter_name: str @@ -9018,7 +9015,7 @@ class DbtInteropCompiledProject: project: CompiledProjectView, ) -> None: ... -class DbtInteropPlanResolution: +class DbtInteropPlanResolution(NativeRecord): """Combined dbt graph and the resolved interop plan.""" graph: DbtCombinedGraph @@ -9029,7 +9026,7 @@ class DbtInteropPlanResolution: plan: DbtInteropPlan, ) -> None: ... -class DbtInteropCommandArgs: +class DbtInteropCommandArgs(NativeRecord): """Selection, passthrough args, and executables for a dbt interop command.""" select: tuple[str, ...] @@ -9048,7 +9045,7 @@ class DbtInteropCommandArgs: sqlbuild_executable: str = "sqb", ) -> None: ... -class DbtPlanEnvironment: +class DbtPlanEnvironment(NativeRecord): """Compiled project and adapter environment for dbt plan-output builds.""" project_dir: Path @@ -9065,7 +9062,7 @@ class DbtPlanEnvironment: adapter_name: str, ) -> None: ... -class DbtSqlbuildPlanRequest: +class DbtSqlbuildPlanRequest(NativeRecord): """Selection and native planner inputs for a SQLBuild plan-output build.""" selected_model_names: tuple[str, ...] @@ -9099,7 +9096,7 @@ class SourceFreshnessUnknownReason(StrEnum): ERROR = "error" NO_FRESHNESS_CONFIG = "no_freshness_config" -class SourceFreshnessRenderers: +class SourceFreshnessRenderers(NativeRecord): """Adapter SQL renderers used when writing source freshness rows.""" render_qualified_name: Callable[..., str | None] @@ -9116,7 +9113,7 @@ class SourceFreshnessRenderers: render_create_index_sqls: Callable[..., tuple[str, ...]] | None = None, ) -> None: ... -class SourceFreshnessObservation: +class SourceFreshnessObservation(NativeRecord): """One comparable observation whose datetime values are always aware UTC.""" source_name: str @@ -9133,7 +9130,7 @@ class SourceFreshnessObservation: observed_at: datetime, ) -> None: ... -class SourceFreshnessUnknown: +class SourceFreshnessUnknown(NativeRecord): """One source whose freshness could not be observed, with the reason.""" source_name: str @@ -9146,7 +9143,7 @@ class SourceFreshnessUnknown: message: str, ) -> None: ... -class AdapterSourceFreshnessBatch: +class AdapterSourceFreshnessBatch(NativeRecord): """Per-source outcomes of one batched adapter metadata observation.""" observations: dict[str, SourceFreshnessObservation] @@ -9157,7 +9154,7 @@ class AdapterSourceFreshnessBatch: unknown: dict[str, SourceFreshnessUnknown] = ..., ) -> None: ... -class SourceFreshnessIdentity: +class SourceFreshnessIdentity(NativeRecord): """Stable identity for one source freshness stream in direct state.""" source_name: str @@ -9172,7 +9169,7 @@ class SourceFreshnessIdentity: target_name: str | None, ) -> None: ... -class SourceFreshnessSet: +class SourceFreshnessSet(NativeRecord): """Latest direct source freshness records for one target schema.""" schema: str @@ -9183,7 +9180,7 @@ class SourceFreshnessSet: records: dict[SourceFreshnessIdentity, SourceFreshnessRecord], ) -> None: ... -class DirectSourceFreshnessPropagationResult: +class DirectSourceFreshnessPropagationResult(NativeRecord): """Downstream model impact derived from direct source freshness roots.""" changed_source_model_names: dict[SourceFreshnessIdentity, frozenset[str]] @@ -9199,3 +9196,115 @@ class DirectSourceFreshnessPropagationResult: stale_model_names: frozenset[str] = ..., blocked_model_names: frozenset[str] = ..., ) -> None: ... + +class QueryCostObservation(NativeRecord): + query_id: str + warehouse_name: str + warehouse_size: str + warehouse_type: str | None + cluster_number: int | None + started_at: datetime + completed_at: datetime + execution_ms: int + bytes_scanned: int + execution_status: str + run_id: str | None + resource_type: str | None + resource_name: str | None + def __init__( + self, + query_id: str, + warehouse_name: str, + warehouse_size: str, + warehouse_type: str | None, + cluster_number: int | None, + started_at: datetime, + completed_at: datetime, + execution_ms: int, + bytes_scanned: int, + execution_status: str, + run_id: str | None = None, + resource_type: str | None = None, + resource_name: str | None = None, + ) -> None: ... + +class ResourceCost(NativeRecord): + resource_type: str + resource_name: str + warehouse_name: str + warehouse_size: str + warehouse_type: str | None + cluster_number: int | None + query_count: int + attributed_seconds: Decimal + bytes_scanned: int + estimated_compute_credits: Decimal + estimated_usd: Decimal + def __init__( + self, + resource_type: str, + resource_name: str, + warehouse_name: str, + warehouse_size: str, + warehouse_type: str | None, + cluster_number: int | None, + query_count: int, + attributed_seconds: Decimal, + bytes_scanned: int, + estimated_compute_credits: Decimal, + estimated_usd: Decimal, + ) -> None: ... + +class CostRunRecord(NativeRecord): + run_id: str + adapter_name: str + target_name: str | None + build_status: str + started_at: datetime + completed_at: datetime + cost: RunCostSummary + version: int + had_executable_work: bool | None + def __init__( + self, + run_id: str, + adapter_name: str, + target_name: str | None, + build_status: str, + started_at: datetime, + completed_at: datetime, + cost: RunCostSummary, + version: int = 1, + had_executable_work: bool | None = None, + ) -> None: ... + +class StatementLedgerEntry(NativeRecord): + statement_id: str + run_id: str + resource_type: str + resource_name: str + phase: str + attempt: int + query_id: str | None + status: str + started_at: datetime + completed_at: datetime + def __init__( + self, + statement_id: str, + run_id: str, + resource_type: str, + resource_name: str, + phase: str, + attempt: int, + query_id: str | None, + status: str, + started_at: datetime, + completed_at: datetime, + ) -> None: ... + +class NativeRecord: + """Base of every native frozen record, built, compared, shown and pickled as a dataclass.""" + + __dataclass_fields__: ClassVar[dict[str, Field[Any]]] + __match_args__: ClassVar[tuple[str, ...]] diff --git a/src/sqlbuild/cost/models.py b/src/sqlbuild/cost/models.py index e816fa6077..53e60f87c7 100644 --- a/src/sqlbuild/cost/models.py +++ b/src/sqlbuild/cost/models.py @@ -4,30 +4,13 @@ from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime from decimal import Decimal from pathlib import Path +from sqlbuild._native import ResourceCost from sqlbuild.cost.types import CostStatus -@dataclass(frozen=True) -class QueryCostObservation: - query_id: str - warehouse_name: str - warehouse_size: str - warehouse_type: str | None - cluster_number: int | None - started_at: datetime - completed_at: datetime - execution_ms: int - bytes_scanned: int - execution_status: str - run_id: str | None = None - resource_type: str | None = None - resource_name: str | None = None - - @dataclass(frozen=True) class StatementExecutionTelemetry: """Completed Snowflake statement identity and local elapsed time.""" @@ -51,21 +34,6 @@ class CostResourceContext: on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None -@dataclass(frozen=True) -class ResourceCost: - resource_type: str - resource_name: str - warehouse_name: str - warehouse_size: str - warehouse_type: str | None - cluster_number: int | None - query_count: int - attributed_seconds: Decimal - bytes_scanned: int - estimated_compute_credits: Decimal - estimated_usd: Decimal - - @dataclass(frozen=True) class RunCostSummary: status: CostStatus @@ -85,30 +53,3 @@ class RunCostSummary: method: str = "equal_share_visible_query_busy_time_v1" limitations: tuple[str, ...] = () message: str | None = None - - -@dataclass(frozen=True) -class CostRunRecord: - run_id: str - adapter_name: str - target_name: str | None - build_status: str - started_at: datetime - completed_at: datetime - cost: RunCostSummary - version: int = 1 - had_executable_work: bool | None = None - - -@dataclass(frozen=True) -class StatementLedgerEntry: - statement_id: str - run_id: str - resource_type: str - resource_name: str - phase: str - attempt: int - query_id: str | None - status: str - started_at: datetime - completed_at: datetime diff --git a/src/sqlbuild/cost/types.py b/src/sqlbuild/cost/types.py index c1cd69624a..d265e28792 100644 --- a/src/sqlbuild/cost/types.py +++ b/src/sqlbuild/cost/types.py @@ -1,15 +1,6 @@ """Cost observability type declarations.""" -from __future__ import annotations - -from datetime import datetime -from decimal import Decimal from enum import StrEnum -from pathlib import Path -from typing import TYPE_CHECKING, Protocol, runtime_checkable - -if TYPE_CHECKING: - from sqlbuild.cost.models import RunCostSummary class CostCapability(StrEnum): @@ -23,23 +14,3 @@ class CostStatus(StrEnum): PENDING = "pending" UNAVAILABLE = "unavailable" COLLECTION_FAILED = "collection_failed" - - -@runtime_checkable -class CostAwareAdapter(Protocol): - """Adapter extension implemented only by warehouses with cost telemetry.""" - - def cost_capability(self) -> CostCapability: ... - - def collect_run_cost( - self, - *, - connection_config: dict[str, object], - target_database: str | None, - run_id: str, - started_at: datetime, - completed_at: datetime, - statement_ledger_path: Path, - usd_per_credit: Decimal, - rate_source: str, - ) -> RunCostSummary: ... diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py index 7576891f3d..b078e34ce3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_collection.py @@ -7,8 +7,8 @@ import pytest -from sqlbuild._native import finalize_build_cost, read_cost_run -from sqlbuild.cost.models import CostRunRecord, RunCostSummary +from sqlbuild._native import CostRunRecord, finalize_build_cost, read_cost_run +from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( CostCollectionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py index 99e0dcefed..a6ffad2750 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_output.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import format_cost_breakdown -from sqlbuild.cost.models import CostRunRecord +from sqlbuild._native import CostRunRecord, format_cost_breakdown from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( CostOutputTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index 4988768b0c..9a86d50d88 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -11,11 +11,12 @@ from sqlbuild._native import ( CostCommandRequest, + CostRunRecord, read_cost_run, refresh_pending_cost_run, write_cost_run, ) -from sqlbuild.cost.models import CostRunRecord, RunCostSummary +from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostCapability, CostStatus from tests.unit.src.sqlbuild.cli.commands._helpers.cost._test_types import ( CostRefreshDegradationTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py index fa0aeba44d..ad202b2db1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/helpers.py @@ -2,8 +2,8 @@ from decimal import Decimal from pathlib import Path -from sqlbuild._native import CostCommandRequest -from sqlbuild.cost.models import CostRunRecord, ResourceCost, RunCostSummary +from sqlbuild._native import CostCommandRequest, CostRunRecord, ResourceCost +from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus diff --git a/tests/unit/src/sqlbuild/cost/helpers.py b/tests/unit/src/sqlbuild/cost/helpers.py index 50fb65b781..6cb346cdb9 100644 --- a/tests/unit/src/sqlbuild/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cost/helpers.py @@ -1,7 +1,8 @@ from datetime import UTC, datetime, timedelta from decimal import Decimal -from sqlbuild.cost.models import CostRunRecord, QueryCostObservation, ResourceCost, RunCostSummary +from sqlbuild._native import CostRunRecord, QueryCostObservation, ResourceCost +from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus diff --git a/tests/unit/src/sqlbuild/cost/test_allocation.py b/tests/unit/src/sqlbuild/cost/test_allocation.py index b6e2c87032..593175ec6b 100644 --- a/tests/unit/src/sqlbuild/cost/test_allocation.py +++ b/tests/unit/src/sqlbuild/cost/test_allocation.py @@ -4,8 +4,8 @@ import pytest -from sqlbuild._native import allocate_run_cost -from sqlbuild.cost.models import QueryCostObservation, RunCostSummary +from sqlbuild._native import QueryCostObservation, allocate_run_cost +from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.cost._test_types import AllocationTestCase from tests.unit.src.sqlbuild.cost.helpers import build_query_cost_observation diff --git a/tests/unit/src/sqlbuild/cost/test_store.py b/tests/unit/src/sqlbuild/cost/test_store.py index 00fdc8ae77..e2b3d67f4a 100644 --- a/tests/unit/src/sqlbuild/cost/test_store.py +++ b/tests/unit/src/sqlbuild/cost/test_store.py @@ -8,12 +8,12 @@ from sqlbuild._native import ( CostArtifactError, + CostRunRecord, list_cost_runs, read_cost_run, resolve_cost_run, write_cost_run, ) -from sqlbuild.cost.models import CostRunRecord from tests.unit.src.sqlbuild.cost._test_types import ( CostStoreTestCase, InvalidCostArtifactTestCase, diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 21daaa917a..50b8999d87 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -21,6 +21,7 @@ from sqlbuild._native import ( AuditProgressReporter, CostContext, + StatementLedgerEntry, read_statement_ledger, record_statement, ) @@ -35,7 +36,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput -from sqlbuild.cost.models import CostResourceContext, StatementLedgerEntry +from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing From be7553dde6ed383fd0bdbb68e685652cdb63ea2b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:43:13 +0100 Subject: [PATCH 14/23] refactor: declare CLI command and dbt interop enums natively --- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../src/bindings/_helpers/cli/namespace.rs | 2 +- .../command_runtime/warehouse_scope.rs | 2 +- .../src/bindings/_helpers/dbt_interop/cli.rs | 2 +- .../_helpers/dbt_interop/orchestration.rs | 2 +- .../bindings/_helpers/dbt_interop/pipeline.rs | 5 +- .../src/bindings/_helpers/dbt_interop/plan.rs | 2 +- .../_helpers/dbt_interop/sqlbuild_work.rs | 2 +- .../bindings/_helpers/records/constants.rs | 80 +++++++++- .../_helpers/refactor/offline_compile.rs | 2 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 141 ++++++++++++++++-- src/sqlbuild/cli/commands/types.py | 136 ----------------- src/sqlbuild/integrations/dbt/types.py | 17 --- .../_helpers/refactor/test_command.py | 3 +- .../sqlbuild/integrations/dbt/_test_types.py | 2 +- .../sqlbuild/integrations/dbt/test_plan.py | 3 +- .../commands/_helpers/entry/test_parsing.py | 3 +- .../commands/_helpers/runtime/_test_types.py | 2 +- .../_helpers/runtime/test_warehouses.py | 2 +- .../commands/classes/test_cli_namespace.py | 3 +- .../cli/commands/main/compile/_test_types.py | 2 +- .../main/compile/test_compile_command.py | 3 +- .../main/compile/test_compile_output.py | 2 +- .../sqlbuild/cli/commands/main/dbt/helpers.py | 8 +- .../cli/commands/main/dbt/test_dbt.py | 8 +- .../cli/commands/main/entry/_test_types.py | 2 +- .../cli/commands/main/entry/test_main.py | 2 +- .../main/skills/test_skill_directory.py | 2 +- .../sqlbuild/integrations/dbt/test_args.py | 2 +- .../sqlbuild/integrations/dbt/test_execute.py | 2 +- 31 files changed, 246 insertions(+), 204 deletions(-) delete mode 100644 src/sqlbuild/cli/commands/types.py diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 7ecac4974c..1847867028 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -599,7 +599,7 @@ const COMPILE_FIELDS: [Field; 15] = [ field( "lineage_mode", Source::Member( - class("sqlbuild.cli.commands.types", "CompileLineageMode"), + class("sqlbuild._native", "CompileLineageMode"), "compile_lineage_mode", ), ), @@ -731,7 +731,7 @@ const DBT_ARGUMENTS: [Field; 3] = [ field("args", Source::TupleOf("dbt_args")), NO_COLOR, ]; -const DBT_COMMAND: ClassRef = class("sqlbuild.integrations.dbt.types", "DbtInteropCommand"); +const DBT_COMMAND: ClassRef = class("sqlbuild._native", "DbtInteropCommand"); pub(crate) const DBT_PLAN_ROUTE: Route = dbt_route("run_dbt_plan", "plan", &DBT_ARGUMENTS, DBT_COMMAND); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs index 50472bef31..37eceaccbb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs @@ -9,7 +9,7 @@ use sqlbuild_cli::argv::models::NamespaceDefault; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const COMPILE_LINEAGE_MODULE: &str = "sqlbuild.cli.commands.types"; +const COMPILE_LINEAGE_MODULE: &str = "sqlbuild._native"; const COLUMN_LINEAGE_MODULE: &str = "sqlbuild.compiler.lineage.types"; /// Typed namespace for all CLI arguments across all commands. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs index b7023de7f4..208e5184f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const NATIVE_MODULE: &str = "sqlbuild._native"; const ENTRY_MODELS: &str = "sqlbuild._native"; const WAREHOUSE_GROUP: (&str, &str) = ("sqlbuild.spec.contracts.types", "WarehouseGroup"); -const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; +const COMMAND_TYPES: &str = "sqlbuild._native"; const CONTEXT_ATTRIBUTE: &str = "_command_warehouse_scope_context"; const CONTEXT_NAME: &str = "sqlbuild_command_warehouse_scope"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs index 9748a99b29..6dfcaed30e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_RECORDS: &str = "sqlbuild._native"; -const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const DBT_TYPES: &str = "sqlbuild._native"; const DBT_EXCEPTIONS: &str = "sqlbuild.integrations.dbt.exceptions"; const DBT_EXECUTABLE_ENV_VAR: &str = "DBT_EXECUTABLE"; const COMMAND_ARGV_OPTIONS: (&[&str], &[&str]) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs index 7ca16f2b71..43226da479 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/orchestration.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::dbt_interop::graph::{ use crate::bindings::_helpers::dbt_interop::manifest::resolve_sqlbuild_model_dbt_refs; use crate::bindings::_helpers::dbt_interop::plan::build_dbt_interop_plan; -const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const DBT_TYPES: &str = "sqlbuild._native"; const PLAN_OPTIONS: (&[&str], &[&str]) = ( &[ "command", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index fd0bdf7336..9fa36ef698 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -104,7 +104,7 @@ fn resolve_plan<'py>( fn execute_dbt_interop_from_project(request: &Bound<'_, PyAny>) -> PyResult { let py = request.py(); let command = request.getattr("command")?; - let kind = imported(py, "sqlbuild.integrations.dbt.types", "DbtInteropCommand")?; + let kind = imported(py, "sqlbuild._native", "DbtInteropCommand")?; if !command.eq(kind.getattr("RUN")?)? && !command.eq(kind.getattr("BUILD")?)? { return Err(PyErr::from_value( imported( @@ -245,8 +245,7 @@ fn plan_dbt_interop_from_project<'py>( &[ ( "command", - imported(py, "sqlbuild.integrations.dbt.types", "DbtInteropCommand")? - .getattr("PLAN")?, + imported(py, "sqlbuild._native", "DbtInteropCommand")?.getattr("PLAN")?, ), ("project_dir", args.value("project_dir")?), ("args", args.value("args")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs index f36212e5ac..764aa3e0ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const DBT_TYPES: &str = "sqlbuild.integrations.dbt.types"; +const DBT_TYPES: &str = "sqlbuild._native"; const PRESENTATION_MODELS: &str = "sqlbuild.presentation.models"; const BUILD_OPTIONS: (&[&str], &[&str]) = ( &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 5fe7f76bc2..2a12498023 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -100,7 +100,7 @@ fn execute_dbt_sqlbuild_work<'py>( ], )?; let command = args.value("command")?; - let kinds = imported(py, "sqlbuild.integrations.dbt.types", "DbtInteropCommand")?; + let kinds = imported(py, "sqlbuild._native", "DbtInteropCommand")?; let display = if command.eq(kinds.getattr("RUN")?)? { "sqb build --no-tests --no-audits".to_owned() } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d5d2990a62..6eeeda01af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -17,7 +17,7 @@ pub(crate) const ENUM_MODULE: &str = "enum"; pub(crate) const STR_ENUM: &str = "StrEnum"; /// Every native string enum group, registered before the records. -pub(crate) const ENUMS: &[&[EnumSpec]] = &[FRESHNESS_ENUMS]; +pub(crate) const ENUMS: &[&[EnumSpec]] = &[FRESHNESS_ENUMS, CLI_ENUMS, DBT_ENUMS]; /// Every native record group, in registration order. pub(crate) const RECORDS: &[&[RecordSpec]] = &[ @@ -727,7 +727,7 @@ const COMPILE_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_color", FieldDefault::False), FieldSpec::new( "lineage_mode", - FieldDefault::Member("sqlbuild.cli.commands.types", "CompileLineageMode", "FAST"), + FieldDefault::Member("sqlbuild._native", "CompileLineageMode", "FAST"), ), FieldSpec::new("select", FieldDefault::EmptyTuple), FieldSpec::new("exclude", FieldDefault::EmptyTuple), @@ -1343,3 +1343,79 @@ const COST_RECORDS: &[RecordSpec] = &[ COST_RUN_RECORD, STATEMENT_LEDGER_ENTRY, ]; + +const COMPILE_LINEAGE_MODE: EnumSpec = EnumSpec { + name: "CompileLineageMode", + doc: "Column lineage mode for compile output.", + members: &[("FAST", "fast"), ("RICH", "rich"), ("NONE", "none")], +}; +const CLI_COMMAND: EnumSpec = EnumSpec { + name: "CliCommand", + doc: "", + members: &[ + ("COMPILE", "compile"), + ("DAG", "dag"), + ("PLAN", "plan"), + ("FRESHNESS", "freshness"), + ("BUILD", "build"), + ("TEST", "test"), + ("CHECK", "check"), + ("AUDIT", "audit"), + ("LOAD", "load"), + ("SEED", "seed"), + ("CLONE", "clone"), + ("DIFF", "diff"), + ("DEBUG", "debug"), + ("LINEAGE", "lineage"), + ("QUERY", "query"), + ("COST", "cost"), + ("CLEAN", "clean"), + ("JANITOR", "janitor"), + ("INIT", "init"), + ("PLAYGROUND", "playground"), + ("SCENARIO", "scenario"), + ("DBT", "dbt"), + ("SKILLS", "skills"), + ("FORMAT", "format"), + ("RULES", "rules"), + ("SCOPE", "scope"), + ("CONTRACT", "contract"), + ("RENAME", "rename"), + ("MV", "mv"), + ], +}; +const PLAYGROUND_TEMPLATE: EnumSpec = EnumSpec { + name: "PlaygroundTemplate", + doc: "Available `sqb playground` templates.", + members: &[ + ("WAFFLE_SHOP", "waffle_shop"), + ("LOADER_WAFFLE_SHOP", "loader_waffle_shop"), + ("DAGSTER", "dagster"), + ("RIVERS", "rivers"), + ("PYTHON_NODES", "python_nodes"), + ], +}; + +/// `sqb` commands, compile lineage modes and playground templates. +const CLI_ENUMS: &[EnumSpec] = &[COMPILE_LINEAGE_MODE, CLI_COMMAND, PLAYGROUND_TEMPLATE]; +const DBT_INTEROP_COMMAND: EnumSpec = EnumSpec { + name: "DbtInteropCommand", + doc: "dbt interop commands with SQLBuild participation.", + members: &[ + ("PLAN", "plan"), + ("RUN", "run"), + ("BUILD", "build"), + ("DEBUG", "debug"), + ], +}; +const DBT_INTEROP_SKIP_REASON: EnumSpec = EnumSpec { + name: "DbtInteropSkipReason", + doc: "Reason one side of a dbt interop plan has no work.", + members: &[ + ("NO_DBT_WORK", "no_dbt_work"), + ("NO_SQLBUILD_WORK", "no_sqlbuild_work"), + ], +}; + +/// dbt interop commands and skip reasons. +const DBT_ENUMS: &[EnumSpec] = &[DBT_INTEROP_COMMAND, DBT_INTEROP_SKIP_REASON]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs index b2adf6af7e..6bb3e8ecb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/offline_compile.rs @@ -16,7 +16,7 @@ const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; const RULES_ERRORS: &str = "sqlbuild.rule_engine.exceptions"; const SPEC_ERRORS: &str = "sqlbuild.spec.contracts.exceptions"; const ENTRY_MODELS: &str = "sqlbuild._native"; -const COMMAND_TYPES: &str = "sqlbuild.cli.commands.types"; +const COMMAND_TYPES: &str = "sqlbuild._native"; const REFACTOR_MODELS: &str = "sqlbuild.compiler.refactoring.models"; const DEFAULT_ERROR_CODE: &str = "E001"; diff --git a/python-boundary.toml b/python-boundary.toml index 1edcd598b8..94d5a99c4d 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 10007 +remaining_lines = 9766 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 27c509824b..a78439df9d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -28,18 +28,6 @@ from sqlbuild.adapter.contract.types import ( TypeDialect, ) from sqlbuild.adapter.relations.models import InspectionQueryRecord -from sqlbuild.cli.commands.types import ( - CliCommand, - CompileLineageMode, - DagCommandHandler, - DebugCommandHandler, - FormatCommandHandler, - LineageCommandHandler, - QueryCommandHandler, - ScopeCommandHandler, - ScopeIndexLoader, - SkillsUpdateCommandHandler, -) from sqlbuild.cli.compile_reuse.models import CompileReuseRequest from sqlbuild.compiler.auditing.models import ( AuditGateIdentity, @@ -323,7 +311,6 @@ from sqlbuild.integrations.dbt.models import ( NormalizedDbtProfileConnection, ResolvedDbtProfileOutput, ) -from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason from sqlbuild.lint.models import ( CompileFacts, ExpandedLintFile, @@ -9308,3 +9295,131 @@ class NativeRecord: __dataclass_fields__: ClassVar[dict[str, Field[Any]]] __match_args__: ClassVar[tuple[str, ...]] + +class CompileLineageMode(StrEnum): + """Column lineage mode for compile output.""" + + FAST = "fast" + RICH = "rich" + NONE = "none" + +class CliCommand(StrEnum): + COMPILE = "compile" + DAG = "dag" + PLAN = "plan" + FRESHNESS = "freshness" + BUILD = "build" + TEST = "test" + CHECK = "check" + AUDIT = "audit" + LOAD = "load" + SEED = "seed" + CLONE = "clone" + DIFF = "diff" + DEBUG = "debug" + LINEAGE = "lineage" + QUERY = "query" + COST = "cost" + CLEAN = "clean" + JANITOR = "janitor" + INIT = "init" + PLAYGROUND = "playground" + SCENARIO = "scenario" + DBT = "dbt" + SKILLS = "skills" + FORMAT = "format" + RULES = "rules" + SCOPE = "scope" + CONTRACT = "contract" + RENAME = "rename" + MV = "mv" + +class ScopeCommandHandler(Protocol): + def __call__(self, *, request: ScopeCommandRequest) -> int: ... + +class ScopeIndexLoader(Protocol): + def __call__(self, *, project_dir: Path, no_cache: bool = False) -> ScopeIndex: ... + +class DagCommandHandler(Protocol): + def __call__( + self, + project_dir: Path | None, + *, + no_sql_validation: bool, + json_output: bool, + cli_vars: dict[str, object] | None, + ) -> int: ... + +class QueryCommandHandler(Protocol): + def __call__( + self, + project_dir: Path | None, + *, + sql: str | None, + query_file: Path | None, + selected_target: str | None, + output_format: str, + limit: int | None, + ) -> int: ... + +class DebugCommandHandler(Protocol): + def __call__( + self, + project_dir: Path | None, + *, + no_color: bool, + no_connection: bool, + selected_target: str | None, + json_output: bool, + ) -> int: ... + +class LineageCommandHandler(Protocol): + def __call__(self, request: LineageCommandRequest) -> int: ... + +class SkillsUpdateCommandHandler(Protocol): + def __call__( + self, + project_dir: Path | None, + *, + global_install: bool, + targets: tuple[str, ...], + force: bool, + ) -> int: ... + +class FormatCommandHandler(Protocol): + def __call__( + self, + project_dir: Path | None, + *, + select: tuple[str, ...], + exclude: tuple[str, ...], + check: bool, + diff: bool, + fixtures_only: bool, + fix: bool, + json_output: bool, + no_color: bool, + ) -> int: ... + +class PlaygroundTemplate(StrEnum): + """Available `sqb playground` templates.""" + + WAFFLE_SHOP = "waffle_shop" + LOADER_WAFFLE_SHOP = "loader_waffle_shop" + DAGSTER = "dagster" + RIVERS = "rivers" + PYTHON_NODES = "python_nodes" + +class DbtInteropCommand(StrEnum): + """dbt interop commands with SQLBuild participation.""" + + PLAN = "plan" + RUN = "run" + BUILD = "build" + DEBUG = "debug" + +class DbtInteropSkipReason(StrEnum): + """Reason one side of a dbt interop plan has no work.""" + + NO_DBT_WORK = "no_dbt_work" + NO_SQLBUILD_WORK = "no_sqlbuild_work" diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py deleted file mode 100644 index 855c6ebc9c..0000000000 --- a/src/sqlbuild/cli/commands/types.py +++ /dev/null @@ -1,136 +0,0 @@ -"""CLI command type-layer declarations.""" - -from __future__ import annotations - -from enum import StrEnum -from pathlib import Path -from typing import TYPE_CHECKING, Protocol - -if TYPE_CHECKING: - from sqlbuild._native import LineageCommandRequest, ScopeCommandRequest - from sqlbuild.compiler.scopes.models import ScopeIndex - - -class CompileLineageMode(StrEnum): - """Column lineage mode for compile output.""" - - FAST = "fast" - RICH = "rich" - NONE = "none" - - -class CliCommand(StrEnum): - COMPILE = "compile" - DAG = "dag" - PLAN = "plan" - FRESHNESS = "freshness" - BUILD = "build" - TEST = "test" - CHECK = "check" - AUDIT = "audit" - LOAD = "load" - SEED = "seed" - CLONE = "clone" - DIFF = "diff" - DEBUG = "debug" - LINEAGE = "lineage" - QUERY = "query" - COST = "cost" - CLEAN = "clean" - JANITOR = "janitor" - INIT = "init" - PLAYGROUND = "playground" - SCENARIO = "scenario" - DBT = "dbt" - SKILLS = "skills" - FORMAT = "format" - RULES = "rules" - SCOPE = "scope" - CONTRACT = "contract" - RENAME = "rename" - MV = "mv" - - -class ScopeCommandHandler(Protocol): - def __call__(self, *, request: ScopeCommandRequest) -> int: ... - - -class ScopeIndexLoader(Protocol): - def __call__(self, *, project_dir: Path, no_cache: bool = False) -> ScopeIndex: ... - - -class DagCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - no_sql_validation: bool, - json_output: bool, - cli_vars: dict[str, object] | None, - ) -> int: ... - - -class QueryCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - sql: str | None, - query_file: Path | None, - selected_target: str | None, - output_format: str, - limit: int | None, - ) -> int: ... - - -class DebugCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - no_color: bool, - no_connection: bool, - selected_target: str | None, - json_output: bool, - ) -> int: ... - - -class LineageCommandHandler(Protocol): - def __call__(self, request: LineageCommandRequest) -> int: ... - - -class SkillsUpdateCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - global_install: bool, - targets: tuple[str, ...], - force: bool, - ) -> int: ... - - -class FormatCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - check: bool, - diff: bool, - fixtures_only: bool, - fix: bool, - json_output: bool, - no_color: bool, - ) -> int: ... - - -class PlaygroundTemplate(StrEnum): - """Available `sqb playground` templates.""" - - WAFFLE_SHOP = "waffle_shop" - LOADER_WAFFLE_SHOP = "loader_waffle_shop" - DAGSTER = "dagster" - RIVERS = "rivers" - PYTHON_NODES = "python_nodes" diff --git a/src/sqlbuild/integrations/dbt/types.py b/src/sqlbuild/integrations/dbt/types.py index d1de84b07e..dc4da7b53d 100644 --- a/src/sqlbuild/integrations/dbt/types.py +++ b/src/sqlbuild/integrations/dbt/types.py @@ -2,7 +2,6 @@ from __future__ import annotations -from enum import StrEnum from pathlib import Path from typing import Protocol @@ -11,19 +10,3 @@ class DbtInvoker(Protocol): """Callable dbt process invocation contract.""" def __call__(self, *, argv: tuple[str, ...], cwd: Path | None) -> object: ... - - -class DbtInteropCommand(StrEnum): - """dbt interop commands with SQLBuild participation.""" - - PLAN = "plan" - RUN = "run" - BUILD = "build" - DEBUG = "debug" - - -class DbtInteropSkipReason(StrEnum): - """Reason one side of a dbt interop plan has no work.""" - - NO_DBT_WORK = "no_dbt_work" - NO_SQLBUILD_WORK = "no_sqlbuild_work" diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py index 400a9b170f..1e39018122 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import RefactorCommandRequest, run_refactor_command -from sqlbuild.cli.commands.types import CliCommand +from sqlbuild._native import CliCommand, RefactorCommandRequest, run_refactor_command from tests.integration.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( BareTargetRefactorTestCase, RefactorStatusMessagesTestCase, diff --git a/tests/integration/src/sqlbuild/integrations/dbt/_test_types.py b/tests/integration/src/sqlbuild/integrations/dbt/_test_types.py index 5ecf408935..62d2f294d8 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/_test_types.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.integrations.dbt.types import DbtInteropSkipReason +from sqlbuild._native import DbtInteropSkipReason @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py index dbb431e2b0..9e671055ec 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_plan.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import DbtInteropPlan, plan_dbt_interop_from_project -from sqlbuild.integrations.dbt.types import DbtInteropSkipReason +from sqlbuild._native import DbtInteropPlan, DbtInteropSkipReason, plan_dbt_interop_from_project from tests.integration.src.sqlbuild.integrations.dbt._test_types import ( RealDbtInteropPlanTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index e770071e95..1f9a593afd 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -8,10 +8,11 @@ import pytest from sqlbuild._native import ( + CompileLineageMode, ParsedCliInvocation, + PlaygroundTemplate, parse_cli_invocation, ) -from sqlbuild.cli.commands.types import CompileLineageMode, PlaygroundTemplate from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py index b7cfa9bd13..f324319fe9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field -from sqlbuild.cli.commands.types import CliCommand +from sqlbuild._native import CliCommand from sqlbuild.spec.contracts.models import ( LocalConfig, LocalTargetConfig, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index a50bfb0da6..a34cecec9a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CliCommand, ParsedCliInvocation, ResolvedWarehouse, command_warehouse_scope, @@ -16,7 +17,6 @@ resolve_project_connection_config, ) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.types import CliCommand from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig from sqlbuild.spec.contracts.types import WarehouseGroup diff --git a/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py b/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py index d30b53075d..9332c5a01e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py +++ b/tests/unit/src/sqlbuild/cli/commands/classes/test_cli_namespace.py @@ -2,8 +2,7 @@ import pytest -from sqlbuild._native import CliNamespace -from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild._native import CliNamespace, CompileLineageMode from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.unit.src.sqlbuild.cli.commands.classes._test_types import ( CliNamespaceAssignmentTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py index 4999e39e4e..233a5308e1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field -from sqlbuild.cli.commands.types import CompileLineageMode +from sqlbuild._native import CompileLineageMode @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 1fb53e8c4f..38d398bb1d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -12,9 +12,8 @@ import sqlbuild._native as discovery_entry import sqlbuild._native as plan_artifacts_module -from sqlbuild._native import CompileCommandRequest, run_compile +from sqlbuild._native import CompileCommandRequest, CompileLineageMode, run_compile from sqlbuild.adapter.discovery.main import resolve_adapter as runtime_adapters -from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.compiler.lineage.types import ColumnLineageMode from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 5f8746aa9c..d75ae8e2a6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ( + CompileLineageMode, ProjectColumnLineage, WrittenTarget, compiled_model_record, @@ -14,7 +15,6 @@ format_compile_text, project_with, ) -from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileModelConfig from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.discovery.models import SqlHookEntry diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py index 57327532d8..41f49a2dd9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/helpers.py @@ -2,8 +2,12 @@ from pathlib import Path -from sqlbuild._native import DbtInteropPlan, DbtInteropSelectionResult, build_dbt_interop_plan -from sqlbuild.integrations.dbt.types import DbtInteropCommand +from sqlbuild._native import ( + DbtInteropCommand, + DbtInteropPlan, + DbtInteropSelectionResult, + build_dbt_interop_plan, +) from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import DbtAutoInitTestCase diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index 6d16407fa3..737749767a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -7,9 +7,13 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import DbtInteropExecutionRequest, DbtInteropPlan, run_dbt_command +from sqlbuild._native import ( + DbtInteropCommand, + DbtInteropExecutionRequest, + DbtInteropPlan, + run_dbt_command, +) from sqlbuild.integrations.dbt.models import DbtInitRequest, DbtInitResult -from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.cli.commands.main.dbt._test_types import ( DbtAutoInitTestCase, DbtDebugWrapperTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index f2cd5b643f..fb53295fa0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -2,8 +2,8 @@ from dataclasses import dataclass, field from pathlib import Path +from sqlbuild._native import CompileLineageMode from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.lineage.types import ColumnLineageMode diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 57a5178f85..1049478c6f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -14,6 +14,7 @@ BuildCommandRequest, CloneCommandRequest, CompileCommandRequest, + CompileLineageMode, CostCommandRequest, DbtInitCommandRequest, DiffCommandRequest, @@ -30,7 +31,6 @@ from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.lineage.types import ColumnLineageMode diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py index ef86465c73..4b9ec30aa1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py @@ -6,13 +6,13 @@ import pytest from sqlbuild._native import ( + CliCommand, SkillMaintenanceResult, load_packaged_skill_files, maintain_sqlbuild_skills, update_sqlbuild_skills, ) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.types import CliCommand from tests.unit.src.sqlbuild.cli.commands.main.skills._test_types import ( SkillContentTestCase, SkillDirectoryCollisionTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py index dbd1174d51..bc2ac4a2e9 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_args.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_args.py @@ -3,13 +3,13 @@ import pytest from sqlbuild._native import ( + DbtInteropCommand, DbtInteropParsedArgs, DbtInteropRoutedArgs, parse_dbt_execution_args, route_dbt_interop_args, ) from sqlbuild.integrations.dbt.exceptions import DbtInteropArgumentError -from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtArgParseErrorTestCase, DbtArgParseTestCase, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py index 9e2d338288..261c56e695 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_execute.py @@ -14,6 +14,7 @@ from sqlbuild._native import ( DbtCombinedGraph, DbtCommandExecutionResult, + DbtInteropCommand, DbtInteropCompiledProject, DbtInteropExecutionRequest, DbtInteropInvocation, @@ -34,7 +35,6 @@ from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.main.runtime import _execute_json_event_stream as event_module from sqlbuild.integrations.dbt.models import DbtCliOptions, DbtNodeExecutionResult -from sqlbuild.integrations.dbt.types import DbtInteropCommand from tests.unit.src.sqlbuild.integrations.dbt._test_types import ( DbtCompileFullRefreshPipelineTestCase, DbtExecutionSelectionStatusTestCase, From 7a2d24d34195e68dab1fd674e319f1d77b025c14 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:48:22 +0100 Subject: [PATCH 15/23] refactor: build authored SQL values and limits natively and raise validation errors natively --- .../_helpers/compiled_project/attachment.rs | 3 +- .../_helpers/contracts/exception_types.rs | 19 ++++++++++ .../src/bindings/_helpers/contracts/mod.rs | 1 + .../_helpers/contracts/registration.rs | 5 +-- .../bindings/_helpers/contracts/sql_values.rs | 11 +++--- .../bindings/_helpers/records/constants.rs | 25 +++++++++++++ .../_helpers/render/declaration_values.rs | 2 +- .../_helpers/sqltext/authored_values.rs | 2 +- fensu.toml | 10 +++--- src/sqlbuild/_native.pyi | 36 ++++++++++++++++++- src/sqlbuild/sql_values/constants.py | 5 --- src/sqlbuild/sql_values/exceptions.py | 4 --- src/sqlbuild/sql_values/models.py | 28 --------------- .../sqlbuild/compiler/discovery/helpers.py | 3 +- .../_helpers/test_sql_declarations.py | 3 +- .../discovery/_helpers/test_sql_models.py | 3 +- .../src/sqlbuild/sql_values/_test_types.py | 2 +- .../sqlbuild/sql_values/test_normalization.py | 10 ++++-- 18 files changed, 112 insertions(+), 60 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contracts/exception_types.rs delete mode 100644 src/sqlbuild/sql_values/constants.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index edbc5f2c56..4d937e7809 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -21,6 +21,7 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ }; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; +const NATIVE_RECORDS: &str = "sqlbuild._native"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -296,7 +297,7 @@ fn parameter_renderer<'py>(value_renderer: &Bound<'py, PyAny>) -> PyResult) -> PyResult<()> { + module.add( + "SqlValueValidationError", + module.py().get_type::(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index bd7003c946..2b178eeadc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod contract_diagnostics; pub(crate) mod effective_config; +pub(crate) mod exception_types; pub(crate) mod model_contracts; pub(crate) mod project_config; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index cc4f591a2b..c4774e6584 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::contracts::{ - contract_diagnostics, effective_config, project_config, resource_identities, spec_resolution, - sql_values, state_literals, + contract_diagnostics, effective_config, exception_types, project_config, resource_identities, + spec_resolution, sql_values, state_literals, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { contract_diagnostics::register(module)?; effective_config::register(module)?; + exception_types::register(module)?; project_config::register(module)?; resource_identities::register(module)?; spec_resolution::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs index aea1abc5b4..88b128ab63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs @@ -15,7 +15,7 @@ use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLim const MODELS_MODULE: &str = "sqlbuild.sql_values.models"; const TYPES_MODULE: &str = "sqlbuild.sql_values.types"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const NATIVE_MODULE: &str = "sqlbuild._native"; /// The Python classes authored values are told apart by and normalized values are built from. pub(crate) struct SqlValueClasses<'py> { @@ -30,10 +30,11 @@ pub(crate) struct SqlValueClasses<'py> { impl<'py> SqlValueClasses<'py> { pub(crate) fn load(py: Python<'py>) -> PyResult { let models = py.import(MODELS_MODULE)?; + let native = py.import(NATIVE_MODULE)?; Ok(Self { decimal: py.import("decimal")?.getattr("Decimal")?, - authored_set: models.getattr("AuthoredSqlSet")?, - value_call: models.getattr("AuthoredSqlValueCall")?, + authored_set: native.getattr("AuthoredSqlSet")?, + value_call: native.getattr("AuthoredSqlValueCall")?, sql_value: models.getattr("SqlValue")?, logical_type: models.getattr("SqlLogicalType")?, kind: py.import(TYPES_MODULE)?.getattr("SqlValueKind")?, @@ -177,7 +178,7 @@ pub(crate) fn sql_value<'py>( pub(crate) fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { Ok(match failure { SqlValueFailure::Invalid(message) => PyErr::from_value( - py.import(EXCEPTIONS_MODULE)? + py.import(NATIVE_MODULE)? .getattr("SqlValueValidationError")? .call1((message,))?, ), @@ -204,7 +205,7 @@ fn normalize_sql_value<'py>( let applied: Bound<'py, PyAny> = match limits { Some(limits) if !limits.is_none() => limits.clone(), _ => py - .import(MODELS_MODULE)? + .import(NATIVE_MODULE)? .getattr("SqlValueLimits")? .call0()?, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 6eeeda01af..b48199c51e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -28,6 +28,7 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ DBT_RECORDS, FRESHNESS_RECORDS, COST_RECORDS, + SQL_VALUE_RECORDS, ]; /// Records whose bound values are normalized before the record is created. @@ -1396,6 +1397,30 @@ const PLAYGROUND_TEMPLATE: EnumSpec = EnumSpec { ], }; +const AUTHORED_SQL_SET: RecordSpec = RecordSpec { + name: "AuthoredSqlSet", + doc: "Set literal entries retained in authored order until validation.", + fields: &[FieldSpec::required("values")], +}; +const AUTHORED_SQL_VALUE_CALL: RecordSpec = RecordSpec { + name: "AuthoredSqlValueCall", + doc: "Source-preserving representation of an authored ``constant(...)`` call.", + fields: &[FieldSpec::required("arguments")], +}; +const SQL_VALUE_LIMITS: RecordSpec = RecordSpec { + name: "SqlValueLimits", + doc: "Safety limits applied while normalizing recursively authored values.", + fields: &[ + FieldSpec::new("max_depth", FieldDefault::Int(32)), + FieldSpec::new("max_elements", FieldDefault::Int(10000)), + FieldSpec::new("max_size", FieldDefault::Int(1000000)), + ], +}; + +/// Authored typed SQL values and normalization limits. +const SQL_VALUE_RECORDS: &[RecordSpec] = + &[AUTHORED_SQL_SET, AUTHORED_SQL_VALUE_CALL, SQL_VALUE_LIMITS]; + /// `sqb` commands, compile lineage modes and playground templates. const CLI_ENUMS: &[EnumSpec] = &[COMPILE_LINEAGE_MODE, CLI_COMMAND, PLAYGROUND_TEMPLATE]; const DBT_INTEROP_COMMAND: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index d3972c62b6..133f79650a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -401,7 +401,7 @@ pub(crate) fn constant_declaration_sql<'py>( let rendering_error: Bound<'py, PyAny> = imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueRenderingError")?; let validation_error: Bound<'py, PyAny> = - imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueValidationError")?; + imported(py, NATIVE_MODULE, "SqlValueValidationError")?; let raised: Bound<'py, PyAny> = error.value(py).clone().into_any(); if !(raised.is_instance(&rendering_error)? || raised.is_instance(&validation_error)?) { return Err(error); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index 77dba08be1..7fa8ceaa39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; use sqlbuild_sqltext::compiler::models::HeaderWord; -const SQL_VALUES_MODULE: &str = "sqlbuild.sql_values.models"; +const SQL_VALUES_MODULE: &str = "sqlbuild._native"; const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; fn word_to_python(py: Python<'_>, word: String) -> PyResult> { diff --git a/fensu.toml b/fensu.toml index b207ddeddd..e86a350f96 100644 --- a/fensu.toml +++ b/fensu.toml @@ -28,11 +28,6 @@ rule = "FFR309" path = "src/sqlbuild/lint/__init__.py" reason = "Native bindings own lint entry points, expansion preparation and compiler-verified fix planning; this package retains boundary models, custom-rule and macro callbacks." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/sql_values/constants.py" -reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types, limits and errors the native bindings build and raise." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" @@ -178,6 +173,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/graph/__init__.py" reason = "Graph algorithms and views run natively; the package keeps only the edge and view models their callers read." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/sql_values/exceptions.py" +reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types and the rendering error adapters raise." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/output/__init__.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a78439df9d..587fdeffec 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -379,7 +379,7 @@ from sqlbuild.spec.contracts.types import ( SourceFreshnessValueKind, WarehouseGroup, ) -from sqlbuild.sql_values.models import SqlValue, SqlValueLimits +from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import CollectionRendering, StateSqlValueType BUILD_IDENTITY: str @@ -9423,3 +9423,37 @@ class DbtInteropSkipReason(StrEnum): NO_DBT_WORK = "no_dbt_work" NO_SQLBUILD_WORK = "no_sqlbuild_work" + +class AuthoredSqlSet(NativeRecord): + """Set literal entries retained in authored order until validation.""" + + values: tuple[object, ...] + def __init__( + self, + values: tuple[object, ...], + ) -> None: ... + +class AuthoredSqlValueCall(NativeRecord): + """Source-preserving representation of an authored ``constant(...)`` call.""" + + arguments: tuple[tuple[str, object], ...] + def __init__( + self, + arguments: tuple[tuple[str, object], ...], + ) -> None: ... + +class SqlValueLimits(NativeRecord): + """Safety limits applied while normalizing recursively authored values.""" + + max_depth: int + max_elements: int + max_size: int + def __init__( + self, + max_depth: int = ..., + max_elements: int = ..., + max_size: int = ..., + ) -> None: ... + +class SqlValueValidationError(ValueError): + """An authored value cannot be represented by the typed SQL value domain.""" diff --git a/src/sqlbuild/sql_values/constants.py b/src/sqlbuild/sql_values/constants.py deleted file mode 100644 index 6fee16de1b..0000000000 --- a/src/sqlbuild/sql_values/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Constants for typed SQL value validation.""" - -DEFAULT_MAX_SQL_VALUE_DEPTH: int = 32 -DEFAULT_MAX_SQL_VALUE_ELEMENTS: int = 10_000 -DEFAULT_MAX_SQL_VALUE_SIZE: int = 1_000_000 diff --git a/src/sqlbuild/sql_values/exceptions.py b/src/sqlbuild/sql_values/exceptions.py index 31b30de258..893e8fd63f 100644 --- a/src/sqlbuild/sql_values/exceptions.py +++ b/src/sqlbuild/sql_values/exceptions.py @@ -1,9 +1,5 @@ """Typed SQL value errors.""" -class SqlValueValidationError(ValueError): - """An authored value cannot be represented by the typed SQL value domain.""" - - class SqlValueRenderingError(ValueError): """An adapter cannot represent a validated typed SQL value.""" diff --git a/src/sqlbuild/sql_values/models.py b/src/sqlbuild/sql_values/models.py index c3fc732b49..e548e25395 100644 --- a/src/sqlbuild/sql_values/models.py +++ b/src/sqlbuild/sql_values/models.py @@ -4,11 +4,6 @@ from dataclasses import dataclass -from sqlbuild.sql_values.constants import ( - DEFAULT_MAX_SQL_VALUE_DEPTH, - DEFAULT_MAX_SQL_VALUE_ELEMENTS, - DEFAULT_MAX_SQL_VALUE_SIZE, -) from sqlbuild.sql_values.types import SqlValueKind, SqlValuePayload @@ -40,26 +35,3 @@ def kind(self) -> SqlValueKind: """Return the outer logical kind.""" return self.logical_type.kind - - -@dataclass(frozen=True) -class AuthoredSqlSet: - """Set literal entries retained in authored order until validation.""" - - values: tuple[object, ...] - - -@dataclass(frozen=True) -class AuthoredSqlValueCall: - """Source-preserving representation of an authored ``constant(...)`` call.""" - - arguments: tuple[tuple[str, object], ...] - - -@dataclass(frozen=True) -class SqlValueLimits: - """Safety limits applied while normalizing recursively authored values.""" - - max_depth: int = DEFAULT_MAX_SQL_VALUE_DEPTH - max_elements: int = DEFAULT_MAX_SQL_VALUE_ELEMENTS - max_size: int = DEFAULT_MAX_SQL_VALUE_SIZE diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index fde1966b25..7a011212c2 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -21,6 +21,8 @@ from sqlbuild import _native from sqlbuild._native import ( + AuthoredSqlSet, + AuthoredSqlValueCall, discover_audit_files, discover_constant_files, discover_enum_files, @@ -42,7 +44,6 @@ SqlHookEntry, TolerantScopeDiscovery, ) -from sqlbuild.sql_values.models import AuthoredSqlSet, AuthoredSqlValueCall from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( GeneratedDeclarationFileTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index 9dda0fe2b5..418dbfd95a 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -7,6 +7,7 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import AuthoredSqlValueCall from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, @@ -17,7 +18,7 @@ ModelSchemaDeclaration, ) from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError -from sqlbuild.sql_values.models import AuthoredSqlValueCall, SqlValue +from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeclarationResourceIdentityErrorTestCase, ParseDeclarationFileErrorTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index ad13db280f..b97195f39b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -6,6 +6,8 @@ import sqlbuild._native as native_module from sqlbuild._native import ( + AuthoredSqlSet, + AuthoredSqlValueCall, discover_model_files, model_output_column_locations, ) @@ -17,7 +19,6 @@ SqlHookEntry, ) from sqlbuild.spec.contracts.models import SourceLocation -from sqlbuild.sql_values.models import AuthoredSqlSet, AuthoredSqlValueCall from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DeferredModelOutputLocationTestCase, ExpectedBooleanTestCase, diff --git a/tests/unit/src/sqlbuild/sql_values/_test_types.py b/tests/unit/src/sqlbuild/sql_values/_test_types.py index f022acce9c..b79e8082f9 100644 --- a/tests/unit/src/sqlbuild/sql_values/_test_types.py +++ b/tests/unit/src/sqlbuild/sql_values/_test_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from sqlbuild.sql_values.models import SqlValueLimits +from sqlbuild._native import SqlValueLimits @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/sql_values/test_normalization.py b/tests/unit/src/sqlbuild/sql_values/test_normalization.py index d42e56aa9b..3796531f8b 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_normalization.py +++ b/tests/unit/src/sqlbuild/sql_values/test_normalization.py @@ -5,9 +5,13 @@ import pytest -from sqlbuild._native import normalize_sql_value -from sqlbuild.sql_values.exceptions import SqlValueValidationError -from sqlbuild.sql_values.models import AuthoredSqlSet, SqlValue, SqlValueLimits +from sqlbuild._native import ( + AuthoredSqlSet, + SqlValueLimits, + SqlValueValidationError, + normalize_sql_value, +) +from sqlbuild.sql_values.models import SqlValue from tests.unit.src.sqlbuild.sql_values._test_types import ( NormalizeSqlValueErrorTestCase, NormalizeSqlValueTestCase, From 757521d4e2c73b95e3dfa4e32f9a00c418c5adbe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:52:41 +0100 Subject: [PATCH 16/23] refactor: declare completion states, partial build timings and cost window units natively --- .../bindings/_helpers/records/constants.rs | 30 ++++++++++++++++++- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 28 +++++++++++++++++ src/sqlbuild/diagnostics/models.py | 13 -------- src/sqlbuild/presentation/types.py | 9 ------ .../main/test_compile_phase_timings.py | 3 +- 6 files changed, 59 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index b48199c51e..af60f2cbf7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -17,7 +17,8 @@ pub(crate) const ENUM_MODULE: &str = "enum"; pub(crate) const STR_ENUM: &str = "StrEnum"; /// Every native string enum group, registered before the records. -pub(crate) const ENUMS: &[&[EnumSpec]] = &[FRESHNESS_ENUMS, CLI_ENUMS, DBT_ENUMS]; +pub(crate) const ENUMS: &[&[EnumSpec]] = + &[FRESHNESS_ENUMS, CLI_ENUMS, DBT_ENUMS, PRESENTATION_ENUMS]; /// Every native record group, in registration order. pub(crate) const RECORDS: &[&[RecordSpec]] = &[ @@ -29,6 +30,7 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ FRESHNESS_RECORDS, COST_RECORDS, SQL_VALUE_RECORDS, + DIAGNOSTIC_RECORDS, ]; /// Records whose bound values are normalized before the record is created. @@ -1421,6 +1423,23 @@ const SQL_VALUE_LIMITS: RecordSpec = RecordSpec { const SQL_VALUE_RECORDS: &[RecordSpec] = &[AUTHORED_SQL_SET, AUTHORED_SQL_VALUE_CALL, SQL_VALUE_LIMITS]; +const PARTIAL_BUILD_PHASE_TIMINGS: RecordSpec = RecordSpec { + name: "PartialBuildPhaseTimings", + doc: "Available disjoint phase durations after an exceptional build.", + fields: &[ + FieldSpec::new("compile_seconds", FieldDefault::None), + FieldSpec::new("planning_seconds", FieldDefault::None), + FieldSpec::new("connection_preparation_seconds", FieldDefault::None), + FieldSpec::new("schema_preparation_seconds", FieldDefault::None), + FieldSpec::new("execution_seconds", FieldDefault::None), + FieldSpec::new("cost_collection_seconds", FieldDefault::None), + FieldSpec::new("total_seconds", FieldDefault::None), + ], +}; + +/// Phase durations available after an exceptional build. +const DIAGNOSTIC_RECORDS: &[RecordSpec] = &[PARTIAL_BUILD_PHASE_TIMINGS]; + /// `sqb` commands, compile lineage modes and playground templates. const CLI_ENUMS: &[EnumSpec] = &[COMPILE_LINEAGE_MODE, CLI_COMMAND, PLAYGROUND_TEMPLATE]; const DBT_INTEROP_COMMAND: EnumSpec = EnumSpec { @@ -1444,3 +1463,12 @@ const DBT_INTEROP_SKIP_REASON: EnumSpec = EnumSpec { /// dbt interop commands and skip reasons. const DBT_ENUMS: &[EnumSpec] = &[DBT_INTEROP_COMMAND, DBT_INTEROP_SKIP_REASON]; + +const COMPLETION_STATE: EnumSpec = EnumSpec { + name: "CompletionState", + doc: "Overall outcome state for a completion summary line.", + members: &[("OK", "ok"), ("WARN", "warn"), ("FAIL", "fail")], +}; + +/// Completion summary outcomes. +const PRESENTATION_ENUMS: &[EnumSpec] = &[COMPLETION_STATE]; diff --git a/python-boundary.toml b/python-boundary.toml index 94d5a99c4d..06498d11dc 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 9766 +remaining_lines = 9707 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 587fdeffec..6668ae1557 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -9457,3 +9457,31 @@ class SqlValueLimits(NativeRecord): class SqlValueValidationError(ValueError): """An authored value cannot be represented by the typed SQL value domain.""" + +class CompletionState(StrEnum): + """Overall outcome state for a completion summary line.""" + + OK = "ok" + WARN = "warn" + FAIL = "fail" + +class PartialBuildPhaseTimings(NativeRecord): + """Available disjoint phase durations after an exceptional build.""" + + compile_seconds: float | None + planning_seconds: float | None + connection_preparation_seconds: float | None + schema_preparation_seconds: float | None + execution_seconds: float | None + cost_collection_seconds: float | None + total_seconds: float | None + def __init__( + self, + compile_seconds: float | None = None, + planning_seconds: float | None = None, + connection_preparation_seconds: float | None = None, + schema_preparation_seconds: float | None = None, + execution_seconds: float | None = None, + cost_collection_seconds: float | None = None, + total_seconds: float | None = None, + ) -> None: ... diff --git a/src/sqlbuild/diagnostics/models.py b/src/sqlbuild/diagnostics/models.py index 646a29b5e5..8459b36f3d 100644 --- a/src/sqlbuild/diagnostics/models.py +++ b/src/sqlbuild/diagnostics/models.py @@ -15,19 +15,6 @@ class ProcessResourceUsage: max_rss_bytes: int | None -@dataclass(frozen=True) -class PartialBuildPhaseTimings: - """Available disjoint phase durations after an exceptional build.""" - - compile_seconds: float | None = None - planning_seconds: float | None = None - connection_preparation_seconds: float | None = None - schema_preparation_seconds: float | None = None - execution_seconds: float | None = None - cost_collection_seconds: float | None = None - total_seconds: float | None = None - - @dataclass(frozen=True) class DiagnosticRoutingOptions: """Policy for one invocation's diagnostic destinations.""" diff --git a/src/sqlbuild/presentation/types.py b/src/sqlbuild/presentation/types.py index f0e598c6cc..467d0c0cfb 100644 --- a/src/sqlbuild/presentation/types.py +++ b/src/sqlbuild/presentation/types.py @@ -1,17 +1,8 @@ """Type-layer declarations for presentation.""" -from enum import StrEnum from typing import Protocol -class CompletionState(StrEnum): - """Overall outcome state for a completion summary line.""" - - OK = "ok" - WARN = "warn" - FAIL = "fail" - - class TransientLineOwner(Protocol): """A renderer that owns the transient line currently drawn in place on a terminal.""" diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 87815a8441..f580a17fd0 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -9,13 +9,12 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import BuildPhaseTimingTracker +from sqlbuild._native import BuildPhaseTimingTracker, PartialBuildPhaseTimings from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from sqlbuild.rule_engine._helpers.run import rule_threads from sqlbuild.rule_engine.models import RulesRunResult from sqlbuild.runtime.contracts.models import ConnectionHooks From b0ec1374cb8154cc661fc90e484509897c653685 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:10:55 +0100 Subject: [PATCH 17/23] refactor: raise expected CLI and migration state errors from native classes --- .../_helpers/build_command/policies.rs | 2 +- .../bindings/_helpers/check/check_command.rs | 2 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../src/bindings/_helpers/cli/entry.rs | 2 +- .../bindings/_helpers/clone/clone_command.rs | 2 +- .../_helpers/command_runtime/adapters.rs | 2 +- .../command_runtime/runtime_errors.rs | 2 +- .../_helpers/compile_command/target_writer.rs | 2 +- .../_helpers/contract_command/command.rs | 2 +- .../bindings/_helpers/cost_command/refusal.rs | 2 +- .../src/bindings/_helpers/dbt_command/init.rs | 2 +- .../bindings/_helpers/diff_report/support.rs | 2 +- .../_helpers/freshness/freshness_command.rs | 2 +- .../bindings/_helpers/init_command/command.rs | 2 +- .../_helpers/janitor/janitor_command.rs | 2 +- .../_helpers/janitor/janitor_phases.rs | 2 +- .../_helpers/lineage/lineage_command.rs | 2 +- .../bindings/_helpers/load/load_command.rs | 2 +- .../bindings/_helpers/load/load_selection.rs | 2 +- .../bindings/_helpers/query_command/query.rs | 2 +- .../bindings/_helpers/records/constants.rs | 91 +++++++++- .../_helpers/records/error_classes.rs | 165 ++++++++++++++++++ .../src/bindings/_helpers/records/mod.rs | 1 + .../src/bindings/_helpers/records/models.rs | 21 +++ .../bindings/_helpers/records/registration.rs | 39 ++++- .../_helpers/refactor/refactor_command.rs | 2 +- .../_helpers/scenario_command/setup.rs | 2 +- .../bindings/_helpers/scopes/scope_command.rs | 2 +- .../bindings/_helpers/state_rows/constants.rs | 2 +- .../_helpers/workspace_commands/commands.rs | 2 +- src/sqlbuild/_native.pyi | 72 ++++++++ src/sqlbuild/cli/commands/exceptions.py | 90 ---------- .../compiler/migrations/exceptions.py | 15 -- .../cli/commands/main/test_load_command.py | 2 +- .../_helpers/test_module_paths.py | 2 +- .../build_planning/test_execution_limits.py | 3 +- .../build_planning/test_full_refresh.py | 3 +- .../build_planning/test_plan_policies.py | 7 +- .../build_planning/test_retention_decrease.py | 3 +- .../build_planning/test_table_type.py | 3 +- .../commands/_helpers/check/test_planning.py | 3 +- .../_helpers/runtime/test_warehouses.py | 2 +- .../commands/_helpers/scope/test_target.py | 3 +- .../commands/main/compile/test_adapters.py | 3 +- .../test_staged_artifact_publication.py | 3 +- .../cli/commands/main/entry/_test_types.py | 3 +- .../cli/commands/main/entry/test_main.py | 2 +- .../execution/test_build_process_resources.py | 3 +- .../main/inspection/cost/test_cost.py | 3 +- .../main/inspection/query/test_query.py | 3 +- .../main/playground/test_playground.py | 2 +- .../main/skills/test_skill_directory.py | 2 +- .../cli/commands/main/skills/test_skills.py | 2 +- 53 files changed, 435 insertions(+), 166 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs delete mode 100644 src/sqlbuild/cli/commands/exceptions.py delete mode 100644 src/sqlbuild/compiler/migrations/exceptions.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs index a63ccadc3a..3b0714a9e5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs @@ -18,7 +18,7 @@ use sqlbuild_core::build_command::models::{ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const SNAPSHOT_VALUE: &str = "snapshot"; const FULL_REFRESH_REASON: &str = "full_refresh"; const SKIP_ACTION: &str = "skip"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 0179390a01..eb7034d6d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_che use crate::bindings::_helpers::graph::python_selection::python_node_names; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const STATIC_COMMAND_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 5732239d8e..531979fcc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -20,7 +20,7 @@ use sqlbuild_core::text::main::python_strip::python_strip; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; -const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const ENTRY_MODELS_MODULE: &str = "sqlbuild._native"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; const PATHLIB_MODULE: &str = "pathlib"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 483a944ba9..f16b619ccb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -23,7 +23,7 @@ use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; use sqlbuild_core::cli_text::models::CliStyle; const NO_COLOR_OPTION: &str = "--no-color"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPUTE_LOGS_MODULE: &str = "sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"; const LIFECYCLE_EVENT_MODULE: &str = "sqlbuild.runtime.observability.main.create_lifecycle_event"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index b7d33d8a59..faae3454c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const TARGET_CONFIG_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs index c1ba684318..3edb6eb057 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/adapters.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe const REGISTRY_MODULE: &str = "sqlbuild.adapter.discovery.main.resolve_adapter"; const ADAPTER_ERRORS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; -const CLI_ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const CLI_ERRORS_MODULE: &str = "sqlbuild._native"; const ADAPTER_CONFIGURATION_CODE: &str = "C601"; /// `CliUserError(error.message, code="C601", help=error.help)` raised from `error`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs index a20f23f817..441cc01201 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/runtime_errors.rs @@ -6,7 +6,7 @@ use sqlbuild_config::command_connection::errors::CommandConnectionError; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const CLI_ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const CLI_ERRORS_MODULE: &str = "sqlbuild._native"; const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index d8df2df8e3..3e6257e344 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -45,7 +45,7 @@ const LANGUAGE_DIR_DEPTH: usize = 2; const PUBLISHED_STAGING_CHANGED: &str = "changed"; const PUBLISHED_TREE: &str = "tree"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild._native"; -const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; const MODEL_CHAIN_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index 0659bfa7f6..ef2324bcad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_co use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs index 093784f8c8..13b791cb68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs @@ -6,7 +6,7 @@ use sqlbuild_core::cost_command::errors::CostCommandError; use crate::bindings::_helpers::compile_driver::python_calls::imported; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; /// The `CliUserError` for a refused cost request. pub(crate) fn cost_error(py: Python<'_>, error: &CostCommandError) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs index 17cf8c3bcc..2c21464486 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs @@ -14,7 +14,7 @@ use sqlbuild_core::dbt_command::models::DbtInitSummary; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::dbt_command::support::{color_enabled, flag, path, stream}; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const PROGRESS_REPORTER_MODULE: &str = "sqlbuild._native"; const PROFILE_INIT_MODULE: &str = "sqlbuild.integrations.dbt._helpers.profile.init"; const DBT_MODELS: &str = "sqlbuild.integrations.dbt.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs index 77f7cfe9b4..5d8f4c05a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs @@ -8,7 +8,7 @@ use sqlbuild_cli::diff_report::models::DiffRequestFacts; use crate::bindings::_helpers::compile_driver::python_calls::imported; -pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +pub(crate) const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; /// `name(**pairs)` with keyword arguments. pub(crate) fn call<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index bf3216e726..bfaff6a081 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -21,7 +21,7 @@ use crate::bindings::_helpers::freshness::state_io::{StateRead, read_latest}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const CONNECTION_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs index dad2d061af..4b276a57b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::init_command::scaffold::scaffold_blank_project; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const EXISTS_MESSAGE: &str = "sqlbuild_project.toml already exists in this directory"; fn exists_error(py: Python<'_>) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs index a293f77af0..483a823167 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_command.rs @@ -27,7 +27,7 @@ use crate::bindings::_helpers::janitor::janitor_plan::{janitor_plan_text, janito use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const PREVIEW_TARGET_MODULE: &str = "sqlbuild._native"; const TARGET_NAME_MODULE: &str = "sqlbuild._native"; const RETENTION_MODULE: &str = "sqlbuild.executor.janitor.main.resolve_retention_days"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index 06deeaaf9b..e3ce1b35b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -14,7 +14,7 @@ const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation const JANITOR_PLAN_MODULE: &str = "sqlbuild.executor.janitor.main.plan"; const JANITOR_EXECUTE_MODULE: &str = "sqlbuild.executor.janitor.main.execute"; const JANITOR_MODELS: &str = "sqlbuild.executor.janitor.models"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const UNKNOWN_VIEW_CODE: &str = "C503"; const UNKNOWN_VIEW_HELP: &str = "Run sqb janitor without --drop-old-name-view to list compatibility views."; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index a8cfdc1f67..f644e44552 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -40,7 +40,7 @@ use crate::bindings::_helpers::lineage::lineage_index::{graph_index, index_views use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const GRAPH_SELECTION_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 1565112d7f..8f35985120 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -28,7 +28,7 @@ use crate::bindings::_helpers::load::load_selection::{ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild._native"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs index 7b66a5d255..70299b2a80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; const DESTINATION_MODULE: &str = "sqlbuild._native"; const CONNECTION_CONFIG_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index 32ff9db1b3..f8eb8af746 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -22,7 +22,7 @@ use crate::bindings::_helpers::query_command::query_values::{ }; const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const ADAPTER_ERRORS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; const LONG_FORMAT: &str = "long"; const TABLE_FORMAT: &str = "table"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index af60f2cbf7..28796ee67a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -1,7 +1,9 @@ //! The layouts of every native record class. use crate::bindings::_helpers::freshness::observation::normalize_observation; -use crate::bindings::_helpers::records::models::{EnumSpec, FieldDefault, FieldSpec, RecordSpec}; +use crate::bindings::_helpers::records::models::{ + ClassValue, EnumSpec, ErrorSpec, FieldDefault, FieldSpec, RecordSpec, +}; use crate::bindings::_helpers::records::types::RecordFinish; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; @@ -16,6 +18,93 @@ pub(crate) const DATACLASS_FIELD_KIND: &str = "_FIELD"; pub(crate) const ENUM_MODULE: &str = "enum"; pub(crate) const STR_ENUM: &str = "StrEnum"; +/// The class attribute holding an error class's index into `ERRORS`. +pub(crate) const ERROR_SPEC_ATTRIBUTE: &str = "__native_error__"; +/// The `parent` of an error class deriving directly from `Exception`. +pub(crate) const EXCEPTION_PARENT: &str = "Exception"; +/// The `parent` of an error class deriving directly from `ValueError`. +pub(crate) const VALUE_ERROR_PARENT: &str = "ValueError"; + +/// Every native expected-error class, each after its parent. +pub(crate) const ERRORS: &[ErrorSpec] = &[ + ErrorSpec { + name: "CliUserError", + parent: EXCEPTION_PARENT, + doc: "Expected CLI-facing error that should be rendered without a traceback.", + class_attributes: &[("code", ClassValue::Text("C000"))], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "StagedArtifactsChangedError", + parent: "CliUserError", + doc: "Staged compile artifacts changed before publication, so nothing stale is deleted.", + class_attributes: &[], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "QueryExecutionError", + parent: "CliUserError", + doc: "The warehouse rejected or failed an ad hoc query.", + class_attributes: &[ + ("code", ClassValue::Text("C108")), + ("exit_code", ClassValue::Int(2)), + ], + required: &[], + fixed: &[], + }, + ErrorSpec { + name: "QueryDiffOutcomeError", + parent: "CliUserError", + doc: "Expected query-diff failure with an explicit outcome and exit code.", + class_attributes: &[], + required: &["status", "exit_code"], + fixed: &[], + }, + ErrorSpec { + name: "QueryDiffIncompleteError", + parent: "QueryDiffOutcomeError", + doc: "Safety or preflight prevented complete value evidence.", + class_attributes: &[], + required: &[], + fixed: &[ + ("status", ClassValue::Text("incomplete")), + ("exit_code", ClassValue::Int(2)), + ], + }, + ErrorSpec { + name: "QueryDiffExecutionError", + parent: "QueryDiffOutcomeError", + doc: "Setup, execution, ownership publication, or cleanup failed.", + class_attributes: &[], + required: &[], + fixed: &[ + ("status", ClassValue::Text("execution_failed")), + ("exit_code", ClassValue::Int(3)), + ], + }, + ErrorSpec { + name: "DiffSizeGuardError", + parent: "CliUserError", + doc: "A default full model diff stopped before reading data because of table sizes.", + class_attributes: &[ + ("status", ClassValue::Text("incomplete")), + ("exit_code", ClassValue::Int(2)), + ], + required: &["details"], + fixed: &[], + }, + ErrorSpec { + name: "MigrationStateError", + parent: VALUE_ERROR_PARENT, + doc: "Raised when model migration state inputs or rows are invalid.", + class_attributes: &[("code", ClassValue::Text("M000"))], + required: &[], + fixed: &[], + }, +]; + /// Every native string enum group, registered before the records. pub(crate) const ENUMS: &[&[EnumSpec]] = &[FRESHNESS_ENUMS, CLI_ENUMS, DBT_ENUMS, PRESENTATION_ENUMS]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs new file mode 100644 index 0000000000..6c95b0f2a7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs @@ -0,0 +1,165 @@ +//! The native constructor shared by the expected CLI and migration-state error classes. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods, PyTypeMethods}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::records::constants::{ERROR_SPEC_ATTRIBUTE, ERRORS}; +use crate::bindings::_helpers::records::models::{ClassValue, ErrorSpec}; + +/// The `__init__` of the expected-error classes: `Error(message, *, code=None, help=None, **fields)`. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ErrorInitializer {} + +#[pymethods] +impl ErrorInitializer { + fn __get__( + slf: &Bound<'_, Self>, + instance: &Bound<'_, PyAny>, + _owner: &Bound<'_, PyAny>, + ) -> PyResult> { + if instance.is_none() { + return Ok(slf.clone().into_any().unbind()); + } + Ok(Py::new( + slf.py(), + BoundErrorInitializer { + error: instance.clone().unbind(), + }, + )? + .into_any()) + } +} + +/// `ErrorInitializer` bound to one error instance. +#[derive(Debug)] +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct BoundErrorInitializer { + error: Py, +} + +#[pymethods] +impl BoundErrorInitializer { + #[pyo3(signature = (*args, **kwargs))] + fn __call__( + &self, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let error: &Bound<'_, PyAny> = self.error.bind(args.py()); + let name: String = error.get_type().qualname()?.to_string(); + let fields: Bound<'_, PyDict> = match kwargs { + Some(kwargs) => kwargs.copy()?, + None => PyDict::new(args.py()), + }; + let message: Bound<'_, PyAny> = match (args.len(), fields.get_item("message")?) { + (1, None) => args.get_item(0)?, + (0, Some(message)) => { + fields.del_item("message")?; + message + } + (0, None) => { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() missing 1 required positional argument: 'message'" + ))); + } + (1, Some(_)) => { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() got multiple values for argument 'message'" + ))); + } + (count, _) => { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() takes 2 positional arguments but {} were given", + count + 1 + ))); + } + }; + let code: Option> = taken(&fields, "code")?; + let help: Option> = taken(&fields, "help")?; + initialize( + error, + (&message, code.as_ref(), help.as_ref()), + Some(&fields), + ) + } +} + +/// Remove and return the keyword `name`, if it was passed. +fn taken<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult>> { + let value: Option> = fields.get_item(name)?; + if value.is_some() { + fields.del_item(name)?; + } + Ok(value) +} + +type ErrorArguments<'a, 'py> = ( + &'a Bound<'py, PyAny>, + Option<&'a Bound<'py, PyAny>>, + Option<&'a Bound<'py, PyAny>>, +); + +/// Set `args`, `message`, `code`, `help` and the class's keyword fields on `error`. +fn initialize( + error: &Bound<'_, PyAny>, + (message, code, help): ErrorArguments<'_, '_>, + fields: Option<&Bound<'_, PyDict>>, +) -> PyResult<()> { + let py = error.py(); + let class = error.get_type(); + let spec: &ErrorSpec = error_spec(&class.getattr(ERROR_SPEC_ATTRIBUTE)?.extract()?)?; + let name: String = class.qualname()?.to_string(); + if let Some(fields) = fields { + for key in fields.keys() { + let key: String = key.extract()?; + if !spec.required.contains(&key.as_str()) { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() got an unexpected keyword argument '{key}'" + ))); + } + } + } + error.setattr("args", PyTuple::new(py, [message])?)?; + error.setattr("message", message)?; + match code { + Some(code) if !code.is_none() => error.setattr("code", code)?, + _ => error.setattr("code", class.getattr("code")?)?, + } + error.setattr( + "help", + help.map_or_else(|| py.None().into_bound(py), Clone::clone), + )?; + for required in spec.required { + let value = match fields { + Some(fields) => fields.get_item(required)?, + None => None, + }; + let Some(value) = value else { + return Err(PyTypeError::new_err(format!( + "{name}.__init__() missing 1 required keyword-only argument: '{required}'" + ))); + }; + error.setattr(*required, value)?; + } + for (attribute, value) in spec.fixed { + error.setattr(*attribute, class_value(py, *value)?)?; + } + Ok(()) +} + +fn error_spec(index: &usize) -> PyResult<&'static ErrorSpec> { + ERRORS + .get(*index) + .ok_or_else(|| PyTypeError::new_err("unknown native error class")) +} + +/// The Python value of one class-level constant. +pub(crate) fn class_value(py: Python<'_>, value: ClassValue) -> PyResult> { + match value { + ClassValue::Text(text) => text.into_bound_py_any(py), + ClassValue::Int(number) => number.into_bound_py_any(py), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs index 21ee303956..2bc36c8a91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod constants; +pub(crate) mod error_classes; pub(crate) mod fields; pub(crate) mod models; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index beffb44b1b..4d5df04eb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -51,6 +51,27 @@ pub(crate) struct EnumSpec { pub(crate) members: &'static [(&'static str, &'static str)], } +/// A class-level constant of a native error class. +#[derive(Debug, Clone, Copy)] +pub(crate) enum ClassValue { + Text(&'static str), + Int(i64), +} + +/// One native expected-error class: its parent, class constants and keyword fields. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ErrorSpec { + pub(crate) name: &'static str, + /// A native error base or an earlier error in `ERRORS`. + pub(crate) parent: &'static str, + pub(crate) doc: &'static str, + pub(crate) class_attributes: &'static [(&'static str, ClassValue)], + /// Keyword-only constructor fields stored as attributes. + pub(crate) required: &'static [&'static str], + /// Attributes every instance is given. + pub(crate) fixed: &'static [(&'static str, ClassValue)], +} + /// Base of every native frozen record, built, compared, shown and pickled as a dataclass. #[derive(Debug)] #[pyclass(module = "sqlbuild._native", subclass, frozen)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 3942d7e3d7..6bd817393d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -1,15 +1,20 @@ //! Register the native record base, its field descriptor and every record class. +use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::records::constants::{ DATACLASS_FIELD_KIND, DATACLASS_FIELDS_ATTRIBUTE, DATACLASSES_MODULE, ENUM_MODULE, ENUMS, - NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, + ERROR_SPEC_ATTRIBUTE, ERRORS, EXCEPTION_PARENT, NATIVE_MODULE, RECORD_SPEC_ATTRIBUTE, STR_ENUM, + VALUE_ERROR_PARENT, +}; +use crate::bindings::_helpers::records::error_classes::{ + BoundErrorInitializer, ErrorInitializer, class_value, }; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::models::{ - EnumSpec, NativeRecord, RecordAttribute, RecordSpec, + EnumSpec, ErrorSpec, NativeRecord, RecordAttribute, RecordSpec, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -22,6 +27,16 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { for spec in ENUMS.iter().flat_map(|group| group.iter()) { module.add(spec.name, string_enum(&str_enum, spec)?)?; } + module.add_class::()?; + module.add_class::()?; + for (index, spec) in ERRORS.iter().enumerate() { + let parent: Bound<'_, PyAny> = match spec.parent { + EXCEPTION_PARENT => py.get_type::().into_any(), + VALUE_ERROR_PARENT => py.get_type::().into_any(), + parent => module.getattr(parent)?, + }; + module.add(spec.name, error_class(&parent, (index, spec))?)?; + } let dataclasses: Bound<'_, PyModule> = py.import(DATACLASSES_MODULE)?; for (index, spec) in all_specs().enumerate() { let class: Bound<'_, PyAny> = record_class(&metaclass, &base, (index, spec))?; @@ -34,6 +49,26 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { Ok(()) } +/// A subclass of `parent` laid out as `spec`. +fn error_class<'py>( + parent: &Bound<'py, PyAny>, + (index, spec): (usize, &ErrorSpec), +) -> PyResult> { + let py: Python<'py> = parent.py(); + let namespace: Bound<'py, PyDict> = PyDict::new(py); + namespace.set_item("__module__", NATIVE_MODULE)?; + namespace.set_item("__qualname__", spec.name)?; + namespace.set_item("__doc__", spec.doc)?; + namespace.set_item(ERROR_SPEC_ATTRIBUTE, index)?; + for (name, value) in spec.class_attributes { + namespace.set_item(*name, class_value(py, *value)?)?; + } + if matches!(spec.parent, EXCEPTION_PARENT | VALUE_ERROR_PARENT) { + namespace.set_item("__init__", Py::new(py, ErrorInitializer {})?)?; + } + parent.get_type().call1((spec.name, (parent,), namespace)) +} + /// `StrEnum(name, members)` living in `sqlbuild._native`. fn string_enum<'py>(str_enum: &Bound<'py, PyAny>, spec: &EnumSpec) -> PyResult> { let py: Python<'py> = str_enum.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs index 1f2117bf89..69b2026012 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_command.rs @@ -41,7 +41,7 @@ use crate::bindings::_helpers::refactor::refactor_facts::{ declaration_placement, model_facts, refactor_facts, }; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const REFACTOR_ERRORS_MODULE: &str = "sqlbuild.compiler.refactoring.exceptions"; const STAGING_PREFIX: &str = "sqb-refactor-"; const MOVE_COMMAND: &str = "mv"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index fe9d2e0d21..aee03fa27c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -26,7 +26,7 @@ use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild._native"; const SCENARIO_CONFIG_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index a9a4e1b0a7..b73006f8a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::scopes::scope_reports::{ const NATIVE_MODULE: &str = "sqlbuild._native"; const MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; -const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ERRORS_MODULE: &str = "sqlbuild._native"; const TARGET_ERROR_CODE: &str = "C960"; const BARE_USAGE: &str = "bare names work for models, sources, seeds and functions; write macros, \ enums, constants, tests and audits with their prefix, for example macro:"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs index 79a3b60291..7bdc577590 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/state_rows/constants.rs @@ -2,7 +2,7 @@ pub(crate) const MIGRATION_MODELS: &str = "sqlbuild.compiler.migrations.models"; pub(crate) const MIGRATION_TYPES: &str = "sqlbuild.compiler.migrations.types"; -pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild.compiler.migrations.exceptions"; +pub(crate) const MIGRATION_EXCEPTIONS: &str = "sqlbuild._native"; pub(crate) const STATE_TABLE_DDL: &str = "sqlbuild.adapter.state_sql.main.render_state_table_create_sql"; pub(crate) const SQL_VALUE_TYPES: &str = "sqlbuild.sql_values.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index 58b50f56b3..bae5bed561 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; const TEMPLATES_DIRECTORY: &str = "playground_templates"; -const EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const EXCEPTIONS_MODULE: &str = "sqlbuild._native"; const OUTPUT_MODELS: &str = "sqlbuild._native"; const SKILLS_OPTIONS: (&[&str], &[&str]) = (&["project_dir"], &["global_install", "targets", "force"]); diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6668ae1557..12d44ec6fa 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -9485,3 +9485,75 @@ class PartialBuildPhaseTimings(NativeRecord): cost_collection_seconds: float | None = None, total_seconds: float | None = None, ) -> None: ... + +class CliUserError(Exception): + """Expected CLI-facing error that should be rendered without a traceback.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class StagedArtifactsChangedError(CliUserError): + """Staged compile artifacts changed before publication, so nothing stale is deleted.""" + +class QueryExecutionError(CliUserError): + """The warehouse rejected or failed an ad hoc query.""" + + exit_code: int + +class QueryDiffOutcomeError(CliUserError): + """Expected query-diff failure with an explicit outcome and exit code.""" + + status: str + exit_code: int + def __init__( + self, + message: str, + *, + status: str, + exit_code: int, + code: str | None = None, + help: str | None = None, + ) -> None: ... + +class QueryDiffIncompleteError(QueryDiffOutcomeError): + """Safety or preflight prevented complete value evidence.""" + + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class QueryDiffExecutionError(QueryDiffOutcomeError): + """Setup, execution, ownership publication, or cleanup failed.""" + + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... + +class DiffSizeGuardError(CliUserError): + """A default full model diff stopped before reading data because of table sizes.""" + + status: str + exit_code: int + details: dict[str, object] + def __init__( + self, + message: str, + *, + details: dict[str, object], + code: str | None = None, + help: str | None = None, + ) -> None: ... + +class MigrationStateError(ValueError): + """Raised when model migration state inputs or rows are invalid.""" + + code: str + message: str + help: str | None + def __init__( + self, message: str, *, code: str | None = None, help: str | None = None + ) -> None: ... diff --git a/src/sqlbuild/cli/commands/exceptions.py b/src/sqlbuild/cli/commands/exceptions.py deleted file mode 100644 index db0843f5fc..0000000000 --- a/src/sqlbuild/cli/commands/exceptions.py +++ /dev/null @@ -1,90 +0,0 @@ -"""CLI-specific expected exception types.""" - -from __future__ import annotations - - -class CliUserError(Exception): - """Expected CLI-facing error that should be rendered without a traceback.""" - - code: str = "C000" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help - - -class StagedArtifactsChangedError(CliUserError): - """Staged compile artifacts changed before publication, so nothing stale is deleted.""" - - -class QueryExecutionError(CliUserError): - """The warehouse rejected or failed an ad hoc query.""" - - code: str = "C108" - exit_code: int = 2 - - -class QueryDiffOutcomeError(CliUserError): - """Expected query-diff failure with an explicit outcome and exit code.""" - - status: str - exit_code: int - - def __init__( - self, - message: str, - *, - status: str, - exit_code: int, - code: str | None = None, - help: str | None = None, - ) -> None: - super().__init__(message, code=code, help=help) - self.status = status - self.exit_code = exit_code - - -class QueryDiffIncompleteError(QueryDiffOutcomeError): - """Safety or preflight prevented complete value evidence.""" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__( - message, - status="incomplete", - exit_code=2, - code=code, - help=help, - ) - - -class QueryDiffExecutionError(QueryDiffOutcomeError): - """Setup, execution, ownership publication, or cleanup failed.""" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__( - message, - status="execution_failed", - exit_code=3, - code=code, - help=help, - ) - - -class DiffSizeGuardError(CliUserError): - """A default full model diff stopped before reading data because of table sizes.""" - - status: str = "incomplete" - exit_code: int = 2 - - def __init__( - self, - message: str, - *, - details: dict[str, object], - code: str | None = None, - help: str | None = None, - ) -> None: - super().__init__(message, code=code, help=help) - self.details = details diff --git a/src/sqlbuild/compiler/migrations/exceptions.py b/src/sqlbuild/compiler/migrations/exceptions.py deleted file mode 100644 index faca42ec60..0000000000 --- a/src/sqlbuild/compiler/migrations/exceptions.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Expected model migration state exception types.""" - -from __future__ import annotations - - -class MigrationStateError(ValueError): - """Raised when model migration state inputs or rows are invalid.""" - - code: str = "M000" - - def __init__(self, message: str, *, code: str | None = None, help: str | None = None) -> None: - super().__init__(message) - self.message = message - self.code = code if code is not None else self.code - self.help = help diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 012838e072..cf4ca69fc0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -16,6 +16,7 @@ from sqlbuild._native import ( AuditCommandRequest, BuildCommandRequest, + CliUserError, LoadCommandRequest, PlanCommandRequest, ScenarioTestCommandRequest, @@ -32,7 +33,6 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import CursorOverrides diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index 5f6df47ca4..fb7ea24f5d 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -13,7 +13,7 @@ ) from tests.unit.scripts.native_module_paths._helpers.helpers import native_source_errors -EXCEPTIONS_MODULE: str = "sqlbuild.cli.commands.exceptions" +EXCEPTIONS_MODULE: str = "sqlbuild._native" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py index b0cc3de61b..9295e1c946 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py @@ -1,7 +1,6 @@ import pytest -from sqlbuild._native import enforce_model_execution_limit -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, enforce_model_execution_limit from sqlbuild.spec.contracts.models import ExecutionLimitsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( ModelExecutionLimitTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py index 2d40c76480..cb653ebbde 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import enforce_plan_safety_policies -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, enforce_plan_safety_policies from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py index 0a1e83037c..5dfa1f0206 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py @@ -4,8 +4,11 @@ import pytest -from sqlbuild._native import dbt_interop_plan_safety_gates, enforce_plan_safety_policies -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import ( + CliUserError, + dbt_interop_plan_safety_gates, + enforce_plan_safety_policies, +) from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py index c97cf962e7..1355aec214 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import enforce_plan_safety_policies -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, enforce_plan_safety_policies from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.planner.types import RetentionDirection from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py index d4f97d6e22..cb78756d38 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import enforce_plan_safety_policies -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, enforce_plan_safety_policies from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py index 6f0764caff..394064a12b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/check/test_planning.py @@ -6,10 +6,9 @@ import pytest -from sqlbuild._native import run_check_read_side_dependencies +from sqlbuild._native import CliUserError, run_check_read_side_dependencies from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main import relation_lookup as relation_lookup_module -from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index a34cecec9a..e732a3ad32 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CliCommand, + CliUserError, ParsedCliInvocation, ResolvedWarehouse, command_warehouse_scope, @@ -16,7 +17,6 @@ parse_cli_invocation, resolve_project_connection_config, ) -from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import LocalTargetConfig, TargetWarehousesConfig from sqlbuild.spec.contracts.types import WarehouseGroup diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py index cbd0d288d2..ea1025ef75 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scope/test_target.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild._native import ScopeCommandRequest, run_scope_command -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, ScopeCommandRequest, run_scope_command from tests.unit.src.sqlbuild.cli.commands._helpers.scope._test_types import ( ScopeTargetCase, ScopeTargetErrorCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py index a1ca166e58..d3ea7889a5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_adapters.py @@ -5,9 +5,8 @@ import pytest -from sqlbuild._native import resolve_adapter, resolve_effective_adapter_name +from sqlbuild._native import CliUserError, resolve_adapter, resolve_effective_adapter_name from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py index 7d31f329cb..b7d34af1d4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py @@ -11,9 +11,8 @@ import pytest from filelock import FileLock, Timeout -from sqlbuild._native import PreparedCompileArtifacts, WrittenTarget +from sqlbuild._native import PreparedCompileArtifacts, StagedArtifactsChangedError, WrittenTarget from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( StagedPublishTestCase, StagingInterleavingTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index fb53295fa0..84f2efce91 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -2,8 +2,7 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild._native import CompileLineageMode -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, CompileLineageMode from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.lineage.types import ColumnLineageMode diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 1049478c6f..0e86e4037c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -12,6 +12,7 @@ from sqlbuild._native import ( AuditCommandRequest, BuildCommandRequest, + CliUserError, CloneCommandRequest, CompileCommandRequest, CompileLineageMode, @@ -30,7 +31,6 @@ ) from sqlbuild._native import run_cli as _main_with_dependencies from sqlbuild._native import run_cli as main -from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.lineage.types import ColumnLineageMode diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py index 5a3cf41bf1..429bd1358e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import BuildCommandRequest, run_build_command -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import BuildCommandRequest, CliUserError, run_build_command from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( BuildDurationLimitTestCase, BuildProcessReportTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py index b4c4ec7dbc..688de57f20 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py @@ -8,8 +8,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import parse_cost_since_bound, run_cost_command, write_cost_run -from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild._native import CliUserError, parse_cost_since_bound, run_cost_command, write_cost_run from tests.unit.src.sqlbuild.cli.commands.main.inspection.cost._test_types import ( CostCommandTestCase, CostRelativeSinceTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py index 3b1024e00d..429f5783d1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py @@ -5,8 +5,7 @@ import duckdb import pytest -from sqlbuild._native import run_query_command -from sqlbuild.cli.commands.exceptions import QueryExecutionError +from sqlbuild._native import QueryExecutionError, run_query_command from tests.unit.src.sqlbuild.cli.commands.main.inspection.query._test_types import ( QueryExecutionErrorTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 4980c30bad..091232dcd6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -6,12 +6,12 @@ from _pytest.capture import CaptureResult from sqlbuild._native import ( + CliUserError, PlaygroundCommandRequest, SkillMaintenanceResult, maintain_sqlbuild_skills, run_playground_command, ) -from sqlbuild.cli.commands.exceptions import CliUserError from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, RunPlaygroundTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py index 4b9ec30aa1..3b6043df78 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skill_directory.py @@ -7,12 +7,12 @@ from sqlbuild._native import ( CliCommand, + CliUserError, SkillMaintenanceResult, load_packaged_skill_files, maintain_sqlbuild_skills, update_sqlbuild_skills, ) -from sqlbuild.cli.commands.exceptions import CliUserError from tests.unit.src.sqlbuild.cli.commands.main.skills._test_types import ( SkillContentTestCase, SkillDirectoryCollisionTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py index 7e3f608426..8a63fbd274 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/skills/test_skills.py @@ -5,13 +5,13 @@ import pytest from sqlbuild._native import ( + CliUserError, SkillMaintenanceResult, SkillUpdateResult, load_packaged_skill_files, maintain_sqlbuild_skills, update_sqlbuild_skills, ) -from sqlbuild.cli.commands.exceptions import CliUserError from tests.unit.src.sqlbuild.cli.commands.main.skills._test_types import ( SkillMaintenanceTestCase, SkillUpdateErrorTestCase, From eec1f5f4f295c893284a1cc2fd53810e21f86928 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:33:37 +0100 Subject: [PATCH 18/23] chore: reconcile integrated data-model batch --- .../src/bindings/_helpers/diagnostics/trackers.rs | 2 +- python-boundary.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs index 33f860fdac..3eb497510d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -9,7 +9,7 @@ use pyo3::{pyclass, pymethods}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; -const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild.diagnostics.models"; +const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild._native"; const PROCESS_RESOURCES_MODULE: &str = "sqlbuild.diagnostics._helpers.process_resources"; const TRACKER_VARIABLE: &str = "sqlbuild_build_phase_timing_tracker"; const PHASES: [&str; 6] = [ diff --git a/python-boundary.toml b/python-boundary.toml index 06498d11dc..493b802784 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 9707 +remaining_lines = 9602 [[stays]] path = "executor" From 3e0209ebc6b4543e8251b98da35c765946366407 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:34:23 +0100 Subject: [PATCH 19/23] test: compare planner differential types by name, not module --- .../_helpers/baseline/baseline.py | 23 ++++++ .../_helpers/baseline/__init__.py | 0 .../_helpers/baseline/_test_types.py | 15 ++++ .../_helpers/baseline/test_baseline.py | 78 +++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 tests/unit/scripts/planner_differential/_helpers/baseline/__init__.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/baseline/_test_types.py create mode 100644 tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py diff --git a/scripts/planner_differential/_helpers/baseline/baseline.py b/scripts/planner_differential/_helpers/baseline/baseline.py index 49556e77e7..d107a0a0bb 100644 --- a/scripts/planner_differential/_helpers/baseline/baseline.py +++ b/scripts/planner_differential/_helpers/baseline/baseline.py @@ -13,6 +13,7 @@ _STEP_KEY: str = "step" _STEPS_KEY: str = "steps" _MISSING: str = "" +_TYPE_MARKERS: frozenset[str] = frozenset({"__type__", "__enum__"}) def write_document(*, directory: Path, slug: str, document: dict[str, object]) -> Path: @@ -45,6 +46,8 @@ def compare_documents( left: dict[str, object] = expected_steps[index] if index < len(expected_steps) else {} right: dict[str, object] = actual_steps[index] if index < len(actual_steps) else {} step: str = str(right.get(_STEP_KEY, left.get(_STEP_KEY, index))) + left = _without_type_modules(left) + right = _without_type_modules(right) for artifact in (*left, *(key for key in right if key not in left)): if left.get(artifact, _MISSING) == right.get(artifact, _MISSING): continue @@ -87,5 +90,25 @@ def _steps(document: dict[str, object]) -> list[dict[str, object]]: return [mapping for step in steps if (mapping := as_json_object(step)) is not None] +def _without_type_modules(value: dict[str, object]) -> dict[str, object]: + """Compare captured class and enum names without their module, which moves when a type is + hosted natively.""" + + return {key: _type_names_only(key=key, value=item) for key, item in value.items()} + + +def _type_names_only(*, key: str, value: object) -> object: + if key in _TYPE_MARKERS and isinstance(value, str): + return value.rpartition(":")[2] + if isinstance(value, dict): + return { + str(inner_key): _type_names_only(key=str(inner_key), value=item) + for inner_key, item in value.items() + } + if isinstance(value, list): + return [_type_names_only(key="", value=item) for item in value] + return value + + def _rendered(value: object) -> str: return json.dumps(value, indent=1, ensure_ascii=False) diff --git a/tests/unit/scripts/planner_differential/_helpers/baseline/__init__.py b/tests/unit/scripts/planner_differential/_helpers/baseline/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/planner_differential/_helpers/baseline/_test_types.py b/tests/unit/scripts/planner_differential/_helpers/baseline/_test_types.py new file mode 100644 index 0000000000..e68606072f --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/baseline/_test_types.py @@ -0,0 +1,15 @@ +"""Test case types for the planner differential baseline comparison.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CompareDocumentsTestCase: + """A baseline and an actual run, and the artifacts reported as different.""" + + description: str + expected: dict[str, object] + actual: dict[str, object] + expected_artifacts: tuple[str, ...] diff --git a/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py b/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py new file mode 100644 index 0000000000..7456fcbead --- /dev/null +++ b/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py @@ -0,0 +1,78 @@ +"""Baseline comparison reports real plan differences, not where a captured type is defined.""" + +from __future__ import annotations + +import pytest + +from scripts.planner_differential._helpers.baseline.baseline import compare_documents +from tests.unit.scripts.planner_differential._helpers.baseline._test_types import ( + CompareDocumentsTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompareDocumentsTestCase( + description="type_moved_to_native_module_matches", + expected={ + "steps": [ + { + "step": "plan", + "plan": { + "__type__": "sqlbuild.compiler.planner.models:PlanOutput", + "action": { + "__enum__": "sqlbuild.compiler.planner.types:PlanAction.CREATE" + }, + }, + } + ] + }, + actual={ + "steps": [ + { + "step": "plan", + "plan": { + "__type__": "sqlbuild._native:PlanOutput", + "action": {"__enum__": "sqlbuild._native:PlanAction.CREATE"}, + }, + } + ] + }, + expected_artifacts=(), + ), + CompareDocumentsTestCase( + description="different_type_or_member_still_differs", + expected={ + "steps": [ + { + "step": "plan", + "plan": { + "action": { + "__enum__": "sqlbuild.compiler.planner.types:PlanAction.CREATE" + } + }, + "entry": {"__type__": "sqlbuild.compiler.planner.models:ModelPlanEntry"}, + } + ] + }, + actual={ + "steps": [ + { + "step": "plan", + "plan": {"action": {"__enum__": "sqlbuild._native:PlanAction.SKIP"}}, + "entry": {"__type__": "sqlbuild._native:SeedPlanEntry"}, + } + ] + }, + expected_artifacts=("plan", "entry"), + ), + ], + ids=lambda case: case.description, +) +def test_given_runs_when_comparing_then_reports_only_real_differences( + test_case: CompareDocumentsTestCase, +) -> None: + differences = compare_documents(expected=test_case.expected, actual=test_case.actual) + + assert tuple(difference.artifact for difference in differences) == test_case.expected_artifacts From 87f9d7b2313325b91ffe5798fb0c7f69665d8330 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:38:34 +0100 Subject: [PATCH 20/23] chore: drop stale imports after integrating native data models --- src/sqlbuild/_native.pyi | 4 ++-- .../compiler/planner/_helpers/test_selection_staleness.py | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 12d44ec6fa..93df92569d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -286,7 +286,7 @@ from sqlbuild.cursor_algebra.models import ( TimestampValue, ) from sqlbuild.cursor_algebra.types import Bound, BoundSentinel, CursorScalar -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings, ProcessResourceUsage +from sqlbuild.diagnostics.models import ProcessResourceUsage from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildExecutionResult, @@ -329,7 +329,7 @@ from sqlbuild.lint.types import ( RelationKeyRow, ) from sqlbuild.presentation.models import DisplayOptions -from sqlbuild.presentation.types import CompletionState, TransientLineOwner +from sqlbuild.presentation.types import TransientLineOwner from sqlbuild.provider.classes.container import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.rule_engine.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index deab82b272..d8da9b9ed7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -32,11 +32,7 @@ PlannerScope, PlannerSelection, ) -from sqlbuild.compiler.planner.types import ChangeKind -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, -) +from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, PlannerStaleWarningScopeTestCase, From 636f484c1a02a4fc11de11c0241c973a4bc69fbe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:39:41 +0100 Subject: [PATCH 21/23] chore: drop stale fensu exception and tidy differential comparison --- fensu.toml | 5 ----- scripts/planner_differential/_helpers/baseline/baseline.py | 3 +-- .../planner_differential/_helpers/baseline/test_baseline.py | 5 ++++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/fensu.toml b/fensu.toml index e86a350f96..15a559828f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -58,11 +58,6 @@ rule = "FFR309" path = "src/sqlbuild/cost/constants.py" reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models, types and constants they use." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/project_assembly/__init__.py" -reason = "Native bindings own project assembly and SQL syntax checks; this package only declares the resource fact models the native assembly builds." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/contract_adoption/__init__.py" diff --git a/scripts/planner_differential/_helpers/baseline/baseline.py b/scripts/planner_differential/_helpers/baseline/baseline.py index d107a0a0bb..65ac5b1a55 100644 --- a/scripts/planner_differential/_helpers/baseline/baseline.py +++ b/scripts/planner_differential/_helpers/baseline/baseline.py @@ -91,8 +91,7 @@ def _steps(document: dict[str, object]) -> list[dict[str, object]]: def _without_type_modules(value: dict[str, object]) -> dict[str, object]: - """Compare captured class and enum names without their module, which moves when a type is - hosted natively.""" + """Drop module paths from captured class and enum names so natively hosted types still match.""" return {key: _type_names_only(key=key, value=item) for key, item in value.items()} diff --git a/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py b/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py index 7456fcbead..3c0b4eb975 100644 --- a/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py +++ b/tests/unit/scripts/planner_differential/_helpers/baseline/test_baseline.py @@ -5,6 +5,7 @@ import pytest from scripts.planner_differential._helpers.baseline.baseline import compare_documents +from scripts.planner_differential.models import ProjectDifference from tests.unit.scripts.planner_differential._helpers.baseline._test_types import ( CompareDocumentsTestCase, ) @@ -73,6 +74,8 @@ def test_given_runs_when_comparing_then_reports_only_real_differences( test_case: CompareDocumentsTestCase, ) -> None: - differences = compare_documents(expected=test_case.expected, actual=test_case.actual) + differences: tuple[ProjectDifference, ...] = compare_documents( + expected=test_case.expected, actual=test_case.actual + ) assert tuple(difference.artifact for difference in differences) == test_case.expected_artifacts From 14e03cd9314fb05be5c408019fef8e693d103f39 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:06:34 +0100 Subject: [PATCH 22/23] fix: keep process resource usage in Python and account render-raised reference codes --- .../src/bindings/_helpers/diagnostics/trackers.rs | 6 +++--- scripts/compiler_differential/constants.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs index 3eb497510d..1ca3d76d6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -9,7 +9,8 @@ use pyo3::{pyclass, pymethods}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; -const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild._native"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DIAGNOSTICS_MODELS_MODULE: &str = "sqlbuild.diagnostics.models"; const PROCESS_RESOURCES_MODULE: &str = "sqlbuild.diagnostics._helpers.process_resources"; const TRACKER_VARIABLE: &str = "sqlbuild_build_phase_timing_tracker"; const PHASES: [&str; 6] = [ @@ -100,8 +101,7 @@ impl BuildPhaseTimingTracker { keywords.set_item(phase, value)?; } keywords.set_item("total_seconds", non_negative(now, self.started_at))?; - imported(py, DIAGNOSTICS_MODELS_MODULE, "PartialBuildPhaseTimings")? - .call((), Some(&keywords)) + imported(py, NATIVE_MODULE, "PartialBuildPhaseTimings")?.call((), Some(&keywords)) } /// Return the active build timing tracker. diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 06cd15e799..3d97b3a040 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -542,7 +542,9 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: } RENDER_CONFIG_ONLY_KINDS: dict[str, str] = {} DIAGNOSTIC_CODE_PATTERN: re.Pattern[str] = re.compile(r"[A-Z][0-9]{3}") -RENDER_RAISED_CODES: frozenset[str] = frozenset({"C214", "C216", "K012", "P004", "P010", "P011"}) +RENDER_RAISED_CODES: frozenset[str] = frozenset( + {"C214", "C216", "K012", "P004", "P010", "P011", "P012", "P013"} +) RENDER_CODE_SCAN_EXCLUDED: dict[str, str] = { "_helpers/analysis": "SQL analysis of rendered SQL, after the CompileProjectInputs frontier", } From 3d322cf60469125092034bf2ba8739c3adf79ea1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:23:33 +0100 Subject: [PATCH 23/23] fix: report oversized rendered constants as compile errors --- .../src/bindings/_helpers/render/declaration_values.rs | 6 +----- .../compiler/compile/_helpers/test_declarations.py | 8 ++++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 133f79650a..9396908efc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -368,11 +368,7 @@ fn rendered_constant<'py>( declaration.getattr("name")?.str()? ); return Err(PyErr::from_value( - imported( - declaration.py(), - SQL_VALUE_ERRORS_MODULE, - "SqlValueValidationError", - )? + imported(declaration.py(), NATIVE_MODULE, "SqlValueValidationError")? .call1((format!( "{context} rendered value is {size} bytes; maximum is {MAX_SQL_VALUE_SIZE} bytes" ),))?, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index 761be7aae8..53f1b2665e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -610,6 +610,14 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity "constants/groups.sql constant 'groups'.*adapter 'duckdb'.*value_list" ), ), + CompileDeclarationsErrorTestCase( + description="constant rendered over the SQL value size limit", + repo_files={ + "constants/big.sql": f'CONSTANT (name big, value "{"'" * 600_000}");', + "models/orders.sql": 'MODEL (description "Test model orders.");\nSELECT @const("big") AS big\n', + }, + expected_error_fragment="constants/big.sql constant 'big'.*maximum is 1000000 bytes", + ), ], ids=lambda case: case.description, )